eval_plots.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import opensoundscape\n",
    "import glob\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import librosa\n",
    "import torch\n",
    "import wandb\n",
    "import random\n",
    "from  apply_transfer_function import TransferFunction\n",
    "from convert_audio_to_bits import convert_audio_to_bits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model\n",
    "model = opensoundscape.ml.cnn.load_model('/home/michaela/CV4E/model_states/best.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load training data \n",
    "train_clips = pd.read_csv('/home/michaela/CV4E/labeled_data/train_balanced_1500.csv', index_col=[0,1,2]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_scores = model.predict(train_clips, num_workers=16,batch_size=128) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_scores.columns = ['pred_D','pred_A','pred_B']\n",
    "train_all = train_clips.join(train_scores)\n",
    "train_evaluation = train_all.reset_index()\n",
    "# train_scores_normal = train_scores.reset_index()\n",
    "# train_clips_normal = train_clips.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_path = '/mnt/ssd-cluster/michaela/data/transfer_functions/688_130415_B_HARP_DCPP01A.tf'\n",
    "TF = pd.read_csv(tf_path,delim_whitespace=True,header=None)\n",
    "TF.columns=['frequency','calibration']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_clips_normal['end_time']= (train_clips_normal['end_time']).astype(int)\n",
    "# train_clips_normal['start_time']= (train_clips_normal['start_time']).astype(int)\n",
    "\n",
    "# train_scores_normal['end_time']= (train_scores_normal['end_time']).astype(int)\n",
    "# train_scores_normal['start_time']= (train_scores_normal['start_time']).astype(int)\n",
    "# train_evaluation = pd.merge(train_scores_normal, train_clips_normal, on =['file','start_time','end_time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# D call train\n",
    "D_eval_index = train_evaluation.index[train_evaluation['D']==1]\n",
    "D_eval = train_evaluation.loc[D_eval_index]\n",
    "D_noise_index = train_evaluation.index[train_evaluation['D']==0]\n",
    "D_noise = train_evaluation.loc[D_noise_index]\n",
    "\n",
    "plt.hist(D_noise['pred_D'],bins=40,edgecolor='black',alpha=0.5,color='blue',label='Noise prediction score')\n",
    "plt.hist(D_eval['pred_D'],bins=40,edgecolor='black',alpha=0.5,color='orange',label='D call prediction score')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.semilogy()\n",
    "plt.title('D call prediction scores train')\n",
    "plt.legend(loc='upper right') # look at low scoring D calls and high scoring negatives! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# investigate false postives for D call train\n",
    "# threshold of zero for these (before sigmoid function applyed to activation layer)\n",
    "# D_noise\n",
    "# D_noise_fp = D_noise[D_noise['pred_D'] > -0]\n",
    "# D_noise_fp = D_noise_fp.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_path = '/mnt/ssd-cluster/michaela/data/transfer_functions/688_130415_B_HARP_DCPP01A.tf'\n",
    "TF = pd.read_csv(tf_path,delim_whitespace=True,header=None)\n",
    "TF.columns=['frequency','calibration']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate false positives in training data (D calls)\n",
    "# D_fp =opensoundscape.Audio.from_file(D_noise_fp['file'][39],sample_rate=3200,offset=D_noise_fp['start_time'][39],duration=15) # is this the first 30 sec?\n",
    "# bits = 16 \n",
    "# abs_max = 2 ** (bits - 1) \n",
    "# D_fp.samples = np.float64(D_fp.samples)*abs_max\n",
    "# D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming',window_samples=1600,overlap_samples=1400,fft_size=3200,decibel_limits=(-200,200),scaling='density')\n",
    "# D_fp1_TF = apply_transfer_function(D_fp1,TF,decibel_limits=(40,140))\n",
    "# D_fp1_TF.bandpass(10,150).to_image() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate D call false postives with high prediction scores\n",
    "# shocker they're all D calls that were incorrectly labeled. \n",
    "\n",
    "D_noise_fp = D_noise[D_noise['pred_D'] > 0]\n",
    "D_noise_fp = D_noise_fp.reset_index(drop=True)\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive\n",
    "for index, row in D_noise_fp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    annotations = [row['D'],row['A NE Pacific'], row['B NE Pacific']]\n",
    "    predictions = [row['pred_D'],row['pred_A'], row['pred_B']]\n",
    "    rounded_predictions = [round(p, 2) for p in predictions]\n",
    "\n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=3200, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "\n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()\n",
    "\n",
    "    print(f'Annotations: D={annotations[0]},A NE Pacific={annotations[1]}, B NE Pacific={annotations[2]}')\n",
    "    print(f'Predictions: D={rounded_predictions[0]},A NE Pacific={rounded_predictions[1]}, B NE Pacific={rounded_predictions[2]}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate D call true postives with low prediction scores\n",
    "# shocker they're all D calls that were incorrectly labeled. \n",
    "\n",
    "D_eval_tp = D_eval[D_eval['pred_D'] < 0]\n",
    "D_eval_tp = D_eval_tp.reset_index()\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive\n",
    "for index, row in D_eval_tp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    \n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=sample_rate, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "    \n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A call train\n",
    "A_eval_index = train_evaluation.index[train_evaluation['A NE Pacific']==1]\n",
    "A_eval = train_evaluation.loc[A_eval_index]\n",
    "A_noise_index = train_evaluation.index[train_evaluation['A NE Pacific']==0]\n",
    "A_noise = train_evaluation.loc[A_noise_index]\n",
    "\n",
    "plt.hist(A_noise['pred_A'],edgecolor='black',bins=40,alpha=0.5,color='blue',label='Noise prediction score')\n",
    "plt.hist(A_eval['pred_A'],edgecolor='black',bins=40,alpha=0.5,color='orange',label='A call prediction score')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.semilogy()\n",
    "plt.title('A call prediction scores train')\n",
    "plt.legend(loc='upper right') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A_noise_fp = A_noise[A_noise['pred_A'] > -17]\n",
    "A_noise_fp = A_noise_fp.reset_index(drop=True) # Cant plot too many cause its slowwwww"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(A_noise_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate A call false postives with high prediction scores\n",
    "# Looking at false positve A calls with a score greater than -20 on infite scale\n",
    "A_noise_fp = A_noise[A_noise['pred_A'] > -17]\n",
    "A_noise_fp = A_noise_fp.reset_index(drop=True)\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive\n",
    "for index, row in A_noise_fp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    annotations = [row['D'],row['A NE Pacific'], row['B NE Pacific']]\n",
    "    predictions = [row['pred_D'],row['pred_A'], row['pred_B']]\n",
    "    rounded_predictions = [round(p, 2) for p in predictions]\n",
    "\n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=3200, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "\n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()\n",
    "\n",
    "    print(f'Annotations: D={annotations[0]},A NE Pacific={annotations[1]}, B NE Pacific={annotations[2]}')\n",
    "    print(f'Predictions: D={rounded_predictions[0]},A NE Pacific={rounded_predictions[1]}, B NE Pacific={rounded_predictions[2]}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B call train\n",
    "B_eval_index = train_evaluation.index[train_evaluation['B NE Pacific']==1]\n",
    "B_eval = train_evaluation.loc[B_eval_index]\n",
    "B_noise_index = train_evaluation.index[train_evaluation['B NE Pacific']==0]\n",
    "B_noise = train_evaluation.loc[B_noise_index]\n",
    "\n",
    "plt.hist(B_noise['pred_B'],bins=40,edgecolor='black',alpha=0.5,color='blue',label='Noise prediction score')\n",
    "plt.hist(B_eval['pred_B'],bins=40,edgecolor='black',alpha=0.5,color='orange',label='B call prediction score')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.semilogy()\n",
    "plt.title('B call prediction scores train')\n",
    "plt.legend(loc='upper right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate B call false postives with high prediction scores\n",
    "# Looking at false positve A calls with a score greater than -20 on infite scale\n",
    "B_noise_fp = B_noise[B_noise['pred_B'] > -20]\n",
    "B_noise_fp = B_noise_fp.reset_index(drop=True)\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive\n",
    "for index, row in B_noise_fp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    annotations = [row['D'],row['A NE Pacific'], row['B NE Pacific']]\n",
    "    predictions = [row['pred_D'],row['pred_A'], row['pred_B']]\n",
    "    rounded_predictions = [round(p, 2) for p in predictions]\n",
    "\n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=3200, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "\n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()\n",
    "\n",
    "    print(f'Annotations: D={annotations[0]},A NE Pacific={annotations[1]}, B NE Pacific={annotations[2]}')\n",
    "    print(f'Predictions: D={rounded_predictions[0]},A NE Pacific={rounded_predictions[1]}, B NE Pacific={rounded_predictions[2]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B_noise=opensoundscape.Audio.from_file(B_noise.iloc[0]['file'],sample_rate=2000,offset=B_noise['start_time'][0],duration=15) # is this the first 30 sec?\n",
    "\n",
    "# bits = 16 \n",
    "# abs_max = 2 ** (bits - 1)\n",
    "# B_noise.samples = np.float64(B_noise.samples)*abs_max\n",
    "\n",
    "# #test_spec = Spectrogram.from_audio(test_wav, window_type='hamming',window_samples=1000,overlap_samples=900,fft_size=2000,decibel_limits=(-200,200),scaling='density')\n",
    "# test_spec = opensoundscape.Spectrogram.from_audio(B_noise, window_type='hamming',window_samples=1000,overlap_samples=900,fft_size=2000,decibel_limits=(-200,200),scaling='density')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# VALIDATE \n",
    "# run model on validation dataset\n",
    "validate_clips = pd.read_csv('/home/michaela/CV4E/labeled_data/validate_clips.csv', index_col=[0,1,2])\n",
    "validate_scores = model.predict(validate_clips, num_workers=16,batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validate_scores.columns = ['pred_D','pred_A','pred_B']\n",
    "validate_all = validate_clips.join(validate_scores)\n",
    "validate_evaluation = validate_all.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# D call validate\n",
    "D_eval_index = validate_evaluation.index[validate_evaluation['D']==1]\n",
    "D_eval = validate_evaluation.loc[D_eval_index]\n",
    "D_noise_index = validate_evaluation.index[validate_evaluation['D']==0]\n",
    "D_noise = validate_evaluation.loc[D_noise_index]\n",
    "\n",
    "plt.hist(D_noise['pred_D'],bins=40,alpha=0.5,edgecolor='black',color='blue',label='Noise prediction score')\n",
    "plt.hist(D_eval['pred_D'],bins=40,alpha=0.5,edgecolor='black',color='orange',label='D call prediction score')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.semilogy()\n",
    "plt.title('D call prediction scores validate')\n",
    "plt.legend(loc='upper right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate D call false postives with high prediction scores\n",
    "# Looking at false positve A calls with a score greater than -20 on infite scale\n",
    "D_noise_fp = D_noise[D_noise['pred_D'] > -3]\n",
    "D_noise_fp = D_noise_fp.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(D_noise_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate D call false postives with high prediction scores\n",
    "# Looking at false positve D calls with a score greater than 0 on infite scale\n",
    "D_noise_fp = D_noise[D_noise['pred_D'] > -3]\n",
    "D_noise_fp = D_noise_fp.reset_index(drop=True)\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive (HIGH SCORING NEGATIVE)\n",
    "for index, row in D_noise_fp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    annotations = [row['D'],row['A NE Pacific'], row['B NE Pacific']]\n",
    "    predictions = [row['pred_D'],row['pred_A'], row['pred_B']]\n",
    "    rounded_predictions = [round(p, 2) for p in predictions]\n",
    "\n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=3200, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "\n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()\n",
    "\n",
    "    print(f'Annotations: D={annotations[0]},A NE Pacific={annotations[1]}, B NE Pacific={annotations[2]}')\n",
    "    print(f'Predictions: D={rounded_predictions[0]},A NE Pacific={rounded_predictions[1]}, B NE Pacific={rounded_predictions[2]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B call validate\n",
    "B_eval_index = validate_evaluation.index[validate_evaluation['B NE Pacific']==1]\n",
    "B_eval = validate_evaluation.loc[B_eval_index]\n",
    "B_noise_index = validate_evaluation.index[validate_evaluation['B NE Pacific']==0]\n",
    "B_noise = validate_evaluation.loc[B_noise_index]\n",
    "\n",
    "plt.hist(B_noise['pred_B'],bins=40,edgecolor='black',alpha=0.5,color='blue',label='Noise prediction score')\n",
    "plt.hist(B_eval['pred_B'],bins=40,edgecolor='black',alpha=0.5,color='orange',label='B call prediction score')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.semilogy()\n",
    "plt.title('B call prediction scores validate')\n",
    "plt.legend(loc='upper right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "B_noise_fp = B_noise[B_noise['pred_B'] > -20]\n",
    "B_noise_fp = B_noise_fp.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(B_noise_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate B call false postives with high prediction scores\n",
    "# Looking at false positve B calls with a score greater than -20 on infinite scale\n",
    "B_noise_fp = B_noise[B_noise['pred_B'] > -20]\n",
    "B_noise_fp = B_noise_fp.reset_index(drop=True)\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive (HIGH SCORING NEGATIVE)\n",
    "for index, row in B_noise_fp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    annotations = [row['D'],row['A NE Pacific'], row['B NE Pacific']]\n",
    "    predictions = [row['pred_D'],row['pred_A'], row['pred_B']]\n",
    "    rounded_predictions = [round(p, 2) for p in predictions]\n",
    "\n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=3200, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "\n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()\n",
    "\n",
    "    print(f'Annotations: D={annotations[0]},A NE Pacific={annotations[1]}, B NE Pacific={annotations[2]}')\n",
    "    print(f'Predictions: D={rounded_predictions[0]},A NE Pacific={rounded_predictions[1]}, B NE Pacific={rounded_predictions[2]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A call validate\n",
    "A_eval_index = validate_evaluation.index[validate_evaluation['A NE Pacific']==1]\n",
    "A_eval = validate_evaluation.loc[A_eval_index]\n",
    "A_noise_index = validate_evaluation.index[validate_evaluation['A NE Pacific']==0]\n",
    "A_noise = validate_evaluation.loc[A_noise_index]\n",
    "\n",
    "plt.hist(A_noise['pred_A'],bins=40,edgecolor='black',alpha=0.5,color='blue',label='Noise prediction score')\n",
    "plt.hist(A_eval['pred_A'],bins=40,edgecolor='black',alpha=0.5,color='orange',label='A call prediction score')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.semilogy()\n",
    "plt.title('A call prediction scores validate')\n",
    "plt.legend(loc='upper right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A_noise_fp = A_noise[A_noise['pred_A'] > -20]\n",
    "A_noise_fp = A_noise_fp.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(A_noise_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate A call false postives with high prediction scores\n",
    "# Looking at false positve A calls with a score greater than -20 on infinite scale\n",
    "A_noise_fp = A_noise[A_noise['pred_A'] > -20]\n",
    "A_noise_fp = A_noise_fp.reset_index(drop=True)\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive (HIGH SCORING NEGATIVE)\n",
    "for index, row in A_noise_fp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    annotations = [row['D'],row['A NE Pacific'], row['B NE Pacific']]\n",
    "    predictions = [row['pred_D'],row['pred_A'], row['pred_B']]\n",
    "    rounded_predictions = [round(p, 2) for p in predictions]\n",
    "\n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=3200, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "\n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()\n",
    "\n",
    "    print(f'Annotations: D={annotations[0]},A NE Pacific={annotations[1]}, B NE Pacific={annotations[2]}')\n",
    "    print(f'Predictions: D={rounded_predictions[0]},A NE Pacific={rounded_predictions[1]}, B NE Pacific={rounded_predictions[2]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST Model on CINMS18B ###\n",
    "test_clips = pd.read_csv('/home/michaela/CV4E/labeled_data/CINMS18B_one_hot_clips.csv', index_col=[0,1,2]) # read in test data\n",
    "test_clips.sum() \n",
    "# model settings for test data\n",
    "model.preprocessor.pipeline.to_spec.params.window_samples = 1000 # 100 window samples\n",
    "model.preprocessor.pipeline.to_spec.params.overlap_samples = 900 # 90% overlap, for 2000 Fs this means 900 samples, and 0.05 sec bins\n",
    "model.preprocessor.pipeline.to_spec.params.fft_size = 2000 # FFT = Fs, 1 Hz bins\n",
    "test_scores = model.predict(test_clips, num_workers=16,batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_scores.columns = ['pred_D','pred_A','pred_B']\n",
    "test_all = test_clips.join(test_scores)\n",
    "test_evaluation = test_all.reset_index()\n",
    "\n",
    "#test_clips_new.iloc[0]['file']\n",
    "#test_clips_new = test_clips.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.preprocessor.forward(test_clips.iloc[0].name[0]) # preprocess the sample\n",
    "# fails when trying to apply the transfer function. seems like its something in the naming scheme...\n",
    "# going to have to adjust the way I apply my transfer function for the model. I can't give it every single model. Needs to be input in the beginning somehow... \n",
    "# or going to have to figure out how to load it as the model runs - in the function. Idk. Wish I had more time. \n",
    "# can't just have my nice transfer function saved in my model. need config file. need to make config file. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# D call test\n",
    "D_eval_index = test_evaluation.index[test_evaluation['D']==1]\n",
    "D_eval = test_evaluation.loc[D_eval_index]\n",
    "D_noise_index = test_evaluation.index[test_evaluation['D']==0]\n",
    "D_noise = test_evaluation.loc[D_noise_index]\n",
    "\n",
    "plt.hist(D_noise['pred_D'],bins=40,alpha=0.5,edgecolor='black',color='blue',label='Noise prediction score')\n",
    "plt.hist(D_eval['pred_D'],bins=40,alpha=0.5,edgecolor='black',color='orange',label='D call prediction score')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.semilogy()\n",
    "plt.title('D call prediction scores test')\n",
    "plt.legend(loc='upper right') # this is progress. look at those high scoring missed detections\n",
    "# blue is all of the examples in the D call column that did not actually contain a D call. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate D call false postives with high prediction scores\n",
    "# Looking at false positve D calls with a score greater than 0 on infite scale\n",
    "D_noise_fp = D_noise[D_noise['pred_D'] > 10]\n",
    "D_noise_fp = D_noise_fp.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(D_noise_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate D call false postives with high prediction scores\n",
    "# Looking at false positve D calls with a score greater than 0 on infite scale\n",
    "D_noise_fp = D_noise[D_noise['pred_D'] > 10]\n",
    "D_noise_fp = D_noise_fp.reset_index(drop=True)\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive (HIGH SCORING NEGATIVE)\n",
    "for index, row in D_noise_fp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    annotations = [row['D'],row['A NE Pacific'], row['B NE Pacific']]\n",
    "    predictions = [row['pred_D'],row['pred_A'], row['pred_B']]\n",
    "    rounded_predictions = [round(p, 2) for p in predictions]\n",
    "\n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=3200, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "\n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()\n",
    "\n",
    "    print(f'Annotations: D={annotations[0]},A NE Pacific={annotations[1]}, B NE Pacific={annotations[2]}')\n",
    "    print(f'Predictions: D={rounded_predictions[0]},A NE Pacific={rounded_predictions[1]}, B NE Pacific={rounded_predictions[2]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B call test\n",
    "B_eval_index = test_evaluation.index[test_evaluation['B NE Pacific']==1]\n",
    "B_eval = test_evaluation.loc[B_eval_index]\n",
    "B_noise_index = test_evaluation.index[test_evaluation['B NE Pacific']==0] \n",
    "B_noise = test_evaluation.loc[B_noise_index]\n",
    "\n",
    "plt.hist(B_noise['pred_B'],bins=40,alpha=0.5,edgecolor='black',color='blue',label='Noise prediction score')\n",
    "plt.hist(B_eval['pred_B'],bins=40,alpha=0.5,edgecolor='black',color='orange',label='B call prediction score')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.semilogy()\n",
    "plt.title('B call prediction scores test')\n",
    "plt.legend(loc='upper right') # Ok this is a start! could be worse... needs more training examples. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate B call false postives with high prediction scores\n",
    "# Looking at false positve D calls with a score greater than 0 on infite scale\n",
    "B_noise_fp = B_noise[B_noise['pred_B'] > -15]\n",
    "B_noise_fp = B_noise_fp.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(B_noise_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate D call false postives with high prediction scores\n",
    "# Looking at false positve D calls with a score greater than 0 on infite scale\n",
    "B_noise_fp = B_noise[B_noise['pred_B'] > -15]\n",
    "B_noise_fp = B_noise_fp.reset_index(drop=True)\n",
    "\n",
    "# bit transform\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "\n",
    "# Create a for loop to process each false positive (HIGH SCORING NEGATIVE)\n",
    "for index, row in B_noise_fp.iterrows():\n",
    "    file_path = row['file']\n",
    "    start_time = row['start_time']\n",
    "    annotations = [row['D'],row['A NE Pacific'], row['B NE Pacific']]\n",
    "    predictions = [row['pred_D'],row['pred_A'], row['pred_B']]\n",
    "    rounded_predictions = [round(p, 2) for p in predictions]\n",
    "\n",
    "    # Load the audio\n",
    "    D_fp = opensoundscape.Audio.from_file(file_path, sample_rate=3200, offset=start_time, duration=15)\n",
    "    \n",
    "    # Scale the audio\n",
    "    D_fp.samples = np.float64(D_fp.samples) * abs_max\n",
    "    \n",
    "    # Create a spectrogram\n",
    "    D_fp1 = opensoundscape.Spectrogram.from_audio(D_fp, window_type='hamming', window_samples=1600, \n",
    "                                                  overlap_samples=1400, fft_size=3200, \n",
    "                                                  decibel_limits=(-200,200), scaling='density')\n",
    "    \n",
    "    # Apply the transfer function\n",
    "    D_fp1_TF = apply_transfer_function(D_fp1, TF, decibel_limits=(40, 140))\n",
    "    \n",
    "    # Bandpass filter and plot\n",
    "    # \n",
    "    filtered_image = D_fp1_TF.bandpass(10, 150).to_image()\n",
    "\n",
    "    # Display the image using matplotlib\n",
    "    plt.imshow(filtered_image)\n",
    "    plt.axis('off')  # Turn off axis labels and ticks\n",
    "    plt.show()\n",
    "\n",
    "    print(f'Annotations: D={annotations[0]},A NE Pacific={annotations[1]}, B NE Pacific={annotations[2]}')\n",
    "    print(f'Predictions: D={rounded_predictions[0]},A NE Pacific={rounded_predictions[1]}, B NE Pacific={rounded_predictions[2]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extra/random/backup code "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(D_eval_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_transfer_function(spec,tf_dataframe,decibel_limits=None):\n",
    "    \"\"\"\n",
    "    apply transfer function to opensoundscape.Spectrogram object\n",
    "    \n",
    "    helper function to apply transfer function to Spectrogram\n",
    "    transfer function is list of | freq | dB offset |\n",
    "    we should interpolate to the frequencies contained in the specrogram\n",
    "\n",
    "    Args:\n",
    "        spec: a Specrogram object\n",
    "        tf_dataframe: dataframe with columns 'freq' (frequencies in Hz) and 'intensity' (dB offset)\n",
    "        decibel_limits: default None will use original spectrogram's .decibel_units attribute;\n",
    "            optionally specify a new decibel_limits range for the returned Spectrogram\n",
    "    \"\"\"\n",
    "    if decibel_limits is None:\n",
    "        decibel_limits = spec.decibel_limits\n",
    "        \n",
    "    #extract frequency column and intensity column from transfer function dataframe\n",
    "    transfer_function_freqs = tf_dataframe.frequency.values\n",
    "    transfer_function_offsets = tf_dataframe.calibration.values\n",
    "    \n",
    "    # linearly interpolate the frequencies from the transfer function table\n",
    "    # onto the frequencies of the spectrogram to get offsets for each spectrogram row\n",
    "    spec_offsets = np.interp(spec.frequencies,transfer_function_freqs, transfer_function_offsets)\n",
    "    \n",
    "    # add the offset values to each row of the spectrogram\n",
    "    new_spec_values = (spec.spectrogram.transpose() + np.array(spec_offsets)).transpose()\n",
    "    \n",
    "    #create a new spectrogram object with the new values\n",
    "    return opensoundscape.Spectrogram(new_spec_values,times=spec.times,frequencies=spec.frequencies,decibel_limits=decibel_limits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "25,920*10*20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validate_A_pos_idx=validate_evaluation.index[validate_evaluation['pred_A']>-16]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validate_A_pos = validate_evaluation.iloc[validate_A_pos_idx] # v excited to get back to my computer.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validate_A_pos "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "B_noise=opensoundscape.Audio.from_file(B_noise.iloc[0]['file'],sample_rate=2000,offset=B_noise['start_time'][0],duration=15) # is this the first 30 sec?\n",
    "\n",
    "bits = 16 \n",
    "abs_max = 2 ** (bits - 1)\n",
    "B_noise.samples = np.float64(B_noise.samples)*abs_max\n",
    "\n",
    "#test_spec = Spectrogram.from_audio(test_wav, window_type='hamming',window_samples=1000,overlap_samples=900,fft_size=2000,decibel_limits=(-200,200),scaling='density')\n",
    "test_spec = opensoundscape.Spectrogram.from_audio(B_noise, window_type='hamming',window_samples=1000,overlap_samples=900,fft_size=2000,decibel_limits=(-200,200),scaling='density')\n",
    "new_spec = apply_transfer_function(test_spec,TF,decibel_limits=(40,140))\n",
    "#new_spec.bandpass(10,150).plot()\n",
    "new_spec.bandpass(10,150).to_image() # ok so seems to be working fine????"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "B_noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A_eval_index = validate_evaluation.index[validate_evaluation['A NE Pacific']==1]\n",
    "validate_A_pos_idx=validate_evaluation.index[validate_evaluation['pred_A']>-16]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(validate_A_pos_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(A_eval_index )) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = model.generate_cams(samples=train_clips, classes='D',guided_backprop=True) # its clearly doing somethnig if its taking 20 minutes\n",
    "#samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples[0].cam.plot() # still not going to work for me. sad. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(samples[0].cam) # maybe this will work when i do it on my computer next week on the cpu lol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples[0].cam # Get gradcam working somehow plz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.generate_cams?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "oss_master",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}