Hyperparameter_search_yamnet_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of files in Class_00_Modified_car_engines: 4644\n",
      "Number of files in Class_01_Regular_Vehicles: 9539\n",
      "Number of files in Class_02_Tools_and_Mechanisms: 13307\n",
      "Number of files in Class_03_Environmental_Sounds: 10488\n",
      "Number of files:  37978\n",
      "classes: ['Class_00_Modified_car_engines', 'Class_01_Regular_Vehicles', 'Class_02_Tools_and_Mechanisms', 'Class_03_Environmental_Sounds'], num_classes: 4\n",
      "INFO:tensorflow:Reloading Oracle from existing project my_dir\\intro_to_kt\\oracle.json\n",
      "INFO:tensorflow:Reloading Tuner from my_dir\\intro_to_kt\\tuner0.json\n",
      "INFO:tensorflow:Oracle triggered exit\n",
      "\n",
      "The hyperparameter search is complete. \n",
      "Optimal Dense: 640 \n",
      "Optimal Dropout: 0.30000000000000004\n",
      "Optimal lr for optimizer: 0.0001.\n",
      "\n",
      "Epoch 1/15\n"
     ]
    }
   ],
   "source": [
    "import os, datetime\n",
    "# os.system(\"activate usc39\")\n",
    "DATETIME_NOW = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "SEED = 1\n",
    "DATASET_DIR = \"./datasets/cer_dataset_16k_resampled_split/\"\n",
    "\n",
    "EPOCHS = 20\n",
    "MODEL_NAME = \"YAMNET\"\n",
    "SEED = 1\n",
    "FILE_RATIO = 0.01\n",
    "PATCH_HOP_DISTANCE = 0.25\n",
    "    \n",
    "import os, sys, glob\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow_io as tfio\n",
    "import random\n",
    "# import datetime\n",
    "\n",
    "sys.path.append('src')\n",
    "from models.get_models import get_model, model_builder_yamnet, model_builder_vggish\n",
    "import models.yamnet_tf2.params as params\n",
    "params = params.Params(sample_rate=16000, patch_hop_seconds=PATCH_HOP_DISTANCE) # 0.25\n",
    "\n",
    "# from dataload_utils.data_load import get_dataset, get_filenames_and_classnames_list\n",
    "import dataload_utils.data_load as data_load\n",
    "from dataload_utils.data_aug import mix_up\n",
    "\n",
    "# SEED = 42\n",
    "random.seed(SEED)\n",
    "tf.random.set_seed(SEED)\n",
    "\n",
    "# parent_dir = \"C:\\\\Users\\\\User\\\\Documents\\\\cer_dataset_16k_flattened_resampled\\\\\"\n",
    "dataset_loader = data_load.Dataset_loader(DATASET_DIR, params)\n",
    "filenames_all = dataset_loader.__filenames_all__\n",
    "classes = dataset_loader.__classes__\n",
    "num_classes = dataset_loader.__num_classes__\n",
    "print(\"classes: {}, num_classes: {}\".format(classes, num_classes))\n",
    "\n",
    "# To do real shuffling\n",
    "AUTOTUNE = tf.data.AUTOTUNE\n",
    "batch_size=64\n",
    "random.shuffle(filenames_all)\n",
    "filenames_all=filenames_all[:int(len(filenames_all)*FILE_RATIO)]\n",
    "filenames_train = filenames_all[:int(len(filenames_all)*0.7)]\n",
    "filenames_eval = filenames_all[int(len(filenames_all)*0.7):int(len(filenames_all)*0.9)]\n",
    "filenames_test = filenames_all[int(len(filenames_all)*0.9):]\n",
    "\n",
    "# Training set preparation\n",
    "dataset_aug = dataset_loader.get_dataset(filenames_train, augment=True)\n",
    "train_dataset = dataset_aug.shuffle(batch_size*2).batch(batch_size) # Batch before doing mixup\n",
    "\n",
    "# Mixup -\n",
    "random.shuffle(filenames_train)\n",
    "dataset_no_aug = dataset_loader.get_dataset(filenames_train, augment=False)\n",
    "\n",
    "zipped_ds = tf.data.Dataset.zip((\n",
    "    dataset_aug.shuffle(batch_size*2).batch(batch_size), \n",
    "    dataset_no_aug.shuffle(batch_size*2).batch(batch_size)\n",
    "    ))\n",
    "\n",
    "train_dataset = zipped_ds.map(\n",
    "    map_func = lambda ds_one, ds_two: mix_up(ds_one, ds_two, alpha=0.2), \n",
    "    num_parallel_calls=AUTOTUNE\n",
    "    )\n",
    "\n",
    "eval_dataset = dataset_loader.get_dataset(filenames_eval, augment=False).shuffle(batch_size*2).batch(batch_size)\n",
    "test_dataset = dataset_loader.get_dataset(filenames_test, augment=False, flat_map=False).shuffle(batch_size*2)#.batch(batch_size)\n",
    "\n",
    "train_dataset = train_dataset.cache().prefetch(AUTOTUNE)\n",
    "eval_dataset = eval_dataset.cache().prefetch(AUTOTUNE)\n",
    "test_dataset = test_dataset.cache().prefetch(AUTOTUNE)\n",
    "\n",
    "# length = len(list(dataset_train_eval))\n",
    "# print(\"Total length of dataset: \", length)\n",
    "\n",
    "# Paths\n",
    "training_path = \"./training/{}\".format(DATETIME_NOW)\n",
    "\n",
    "model_training_path = training_path + \"/{}\".format(MODEL_NAME)\n",
    "ckp_path = model_training_path + \"/checkpoints/cp.ckpt\"\n",
    "log_path = model_training_path + \"/logs/fit\"    \n",
    "hd5_path = model_training_path + \"/model.hd5\"\n",
    "cfm_path = model_training_path + \"/confusion_matrix.png\"\n",
    "\n",
    "# Create a callback that saves the model's weights\n",
    "cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=ckp_path,\n",
    "                                                save_weights_only=True,\n",
    "                                                verbose=1)\n",
    "# Create a tensorboard callback                         \n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_path, histogram_freq=1)\n",
    "stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)\n",
    "\n",
    "import keras_tuner as kt\n",
    "tuner = kt.Hyperband(model_builder_yamnet,\n",
    "                objective='val_accuracy',\n",
    "                max_epochs=10,\n",
    "                factor=3,\n",
    "                directory='my_dir',\n",
    "                project_name='intro_to_kt')    \n",
    "\n",
    "tuner.search(train_dataset, validation_data = eval_dataset, epochs=50, callbacks=[stop_early])\n",
    "\n",
    "best_hps=tuner.get_best_hyperparameters(num_trials=1)[0]\n",
    "\n",
    "print(f\"\"\"\n",
    "The hyperparameter search is complete. \n",
    "Optimal Dense: {best_hps.get('units')} \n",
    "Optimal Dropout: {best_hps.get('dropout')}\n",
    "Optimal lr for optimizer: {best_hps.get('learning_rate')}.\n",
    "\"\"\")\n",
    "\n",
    "model = tuner.hypermodel.build(best_hps)\n",
    "# Fit model from scratch\n",
    "\n",
    "if MODEL_NAME==\"YAMNET\" or MODEL_NAME==\"VGGISH\":\n",
    "    # Transfer learn\n",
    "    # make all layers untrainable by freezing weights (except for last layer)\n",
    "    for l, layer in enumerate(model.layers[:-7]):\n",
    "        layer.trainable = False\n",
    "\n",
    "    # First first time\n",
    "    model.fit(train_dataset, validation_data = eval_dataset, epochs=EPOCHS-5, \n",
    "        verbose=1, callbacks=[cp_callback,tensorboard_callback])\n",
    "    \n",
    "    # unfreeze all layers\n",
    "    for l, layer in enumerate(model.layers[:-7]):\n",
    "        layer.trainable = True\n",
    "        \n",
    "    model.compile(\n",
    "        optimizer=tf.keras.optimizers.Adam(learning_rate=1e-6),\n",
    "        loss=tf.keras.losses.CategoricalCrossentropy(), #from_logits=True\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    model.fit(train_dataset, validation_data = eval_dataset, initial_epoch = EPOCHS-5, epochs=EPOCHS, \n",
    "        verbose=1, callbacks=[cp_callback,tensorboard_callback])\n",
    "    \n",
    "else:\n",
    "    # Fit model from scratch\n",
    "    model.fit(train_dataset, validation_data = eval_dataset, epochs=EPOCHS, \n",
    "        verbose=1, callbacks=[cp_callback,tensorboard_callback])\n",
    "\n",
    "# Evaluate performance of model with test fold (that it wasn't trained on)\n",
    "model.load_weights(ckp_path)\n",
    "loss, acc = model.evaluate(test_dataset, verbose=2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "import matplotlib.pyplot as plt\n",
    "# Get y_preds = predictions made by model\n",
    "y_preds,y_trues = [],[]\n",
    "for x_test, y_true in list(test_dataset):\n",
    "    y_pred = np.argmax(model.predict(x_test), axis=1)\n",
    "    y_true = np.argmax(y_true, axis=1)\n",
    "    y_preds.extend(y_pred)\n",
    "    y_trues.extend(y_true)\n",
    "y_trues = np.array(y_trues)\n",
    "\n",
    "y_preds = np.array(y_preds)\n",
    "\n",
    "accuracy = accuracy_score(y_trues, y_preds)\n",
    "print(\"Testing accuracy: \", accuracy)\n",
    "\n",
    "\n",
    "cm, ax = plt.subplots(figsize=(10,10))\n",
    "try:\n",
    "    cm = ConfusionMatrixDisplay.from_predictions(\n",
    "        y_trues, y_preds, normalize='true', \n",
    "        display_labels=classes, xticks_rotation=90,\n",
    "        ax=ax\n",
    "    )\n",
    "except:\n",
    "    cm = ConfusionMatrixDisplay.from_predictions(\n",
    "        y_trues, y_preds, normalize='true', \n",
    "        xticks_rotation=90,\n",
    "        ax=ax\n",
    "    )\n",
    "ax.set_title(\"{}, Acc: {:02f}\".format(model_training_path.split(\"/\")[-1], accuracy))\n",
    "cm.figure_.savefig(cfm_path,dpi=300)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "aef9c008b311f0b7f3d27d4f3907c3c9c136ad861e53efda71f92a04d644c5c8"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('usc39')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}