3.G09_combination.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "050a59a7-2bce-4f43-b9c7-07cade54fb9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing modules\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import numba\n",
    "import pyarrow.dataset as ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7e1e1254-22ac-4187-a587-4f0c84614d56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pandas display options (personal preference)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3a70dc83-6609-409f-ba91-75e86468a0ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the G09 parquet file\n",
    "dataset = ds.dataset(\"../data/G09_1p1_Z22_unmasked.parquet\", format=\"parquet\")\n",
    "table = dataset.to_table()\n",
    "\n",
    "# Setting uberID as index\n",
    "df = table.to_pandas().set_index('uberID')\n",
    "df.index=df.index.astype('int64')\n",
    "\n",
    "del dataset,table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f694d01a-8563-4870-88ab-380716bc8f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting rid of weird spaces in the class column\n",
    "df=df.replace({'class': {'ambiguous ': 'ambiguous',\n",
    "                         'star      ': 'star', \n",
    "                         'galaxy    ': 'galaxy', \n",
    "                         'artefact  ':'artefact'}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "21482372-1866-4498-bb1f-d8ff812aca8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making a number of cuts: magnitude, mask, starmask and removing artefacts\n",
    "df=df[(df['mag_Zt']<21.2) & \n",
    "      (df['mask']==0) & \n",
    "      (df['starmask']==0) & \n",
    "      (df['class']!='artefact')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "285908ed-087c-463d-8145-dabd52b207b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting a 1/10 subset of the data (for the purpose of this tutorial)\n",
    "df=df[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "07b091d1-78fa-4e85-ad66-e5dd161ff47a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading the u and no u labels from notebooks 1 and 2, and creating new dataframes\n",
    "df_u=pd.read_csv('../data/G09_u_labels.csv').set_index('uberID')\n",
    "df_no_u=pd.read_csv('../data/G09_no_u_labels.csv').set_index('uberID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "776becd4-2f03-4b26-a068-6b7c53213f58",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating new columns in the main dataframe with u and no u labels\n",
    "df['cluster_label']=np.zeros(len(df))\n",
    "df['cluster_label_no_u']=np.zeros(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7442cf2c-cf2f-4db7-8b9a-f1d1a67b4d83",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filling the new columns with u and no u labels\n",
    "df.loc[df_u.index,'cluster_label']=df_u['cluster_label'].apply(str)\n",
    "df.loc[df_no_u.index,'cluster_label_no_u']=df_no_u['cluster_label'].apply(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2531971f-066d-4fc3-b182-f0dc067c2c9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=['star','galaxy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "24bc10e7-7261-4683-b343-e5eb5c3efdf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a 'final label' column full of 1's to combine the two labels\n",
    "df['final_label']=np.ones(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3656e442-b3b4-4895-be2e-6aee5b55643c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# First applying the no u labels to the final label\n",
    "for label in labels:\n",
    "    df.loc[df[df['cluster_label_no_u']==label].index,'final_label']=label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b6807c05-5dcd-4e83-b0b7-359ac13be915",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Then applying the u labels to the final label, overwriting any existing no u labels\n",
    "for label in labels:\n",
    "    df.loc[df[df['cluster_label']==label].index,'final_label']=label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1ba13952-de7c-4ec6-9cd9-79e7051f94fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convering final label to string\n",
    "df['final_label']=df['final_label'].apply(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "82568496-d116-470f-84f0-ad58b3621ef6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a new column to indicate whether a source has any missing bands.\n",
    "# If final label still == 1, that indicates a label hasn't been added, so it has a missing band\n",
    "df['missing_bands']=True\n",
    "df.loc[df[df['final_label']!='1.0'].index,'missing_bands']=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f3a02437-80e8-42b8-a70b-b471e9f62e06",
   "metadata": {},
   "outputs": [],
   "source": [
    "# For the sources with missing labels we then overwrite with class\n",
    "df.loc[df[df['final_label']=='1.0'].index,'final_label'] = df[df['final_label']=='1.0']['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "75829e18-2f54-43bb-b926-6d1c4a7ecd15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01640050945429607"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sources with missing bands are only 1.5% of the sample\n",
    "len(df[df['missing_bands']==True])/len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3b8fd59f-3c3f-433e-8360-404568ef7378",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving\n",
    "df[['class','missing_bands','final_label']].to_csv('../data/G09_labels.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}