05_3layer-LSTM-norm_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0. Import the packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os,sys,shutil,mmap\n",
    "import pickle as pickle\n",
    "import numpy as np                                       # fast vectors and matrices\n",
    "import matplotlib.pyplot as plt                          # plotting\n",
    "from scipy.fftpack import fft\n",
    "\n",
    "import time\n",
    "from sklearn.metrics import average_precision_score\n",
    "import tensorflow as tf\n",
    "import mir_eval\n",
    "\n",
    "sys.path.insert(0,'lib/')\n",
    "import config\n",
    "import model_functions_norm2 as model_functions_norm # contains self-written functions\n",
    "\n",
    "from versions_norm import version_dicc\n",
    "\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Pick the version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "graph                 :  <function LSTM_PT_graph_MomOpt_noReg at 0x7fd8af4717b8>\n",
      "pretrained            :  190319_11HZ\n",
      "window                :  4096\n",
      "timesteps             :  9\n",
      "stride                :  256\n",
      "kk                    :  256\n",
      "dd                    :  1024\n",
      "d2_x                  :  1\n",
      "d2_y                  :  128\n",
      "k2                    :  128\n",
      "stride_y              :  2\n",
      "d3_x                  :  13\n",
      "d3_y                  :  1\n",
      "k3                    :  1024\n",
      "num_units             :  1024\n",
      "starter_learning_rate :  0.001\n",
      "decay_steps           :  10000\n",
      "lr_decay              :  0.95\n",
      "mom                   :  0.95\n",
      "max_steps             :  99999\n",
      "comment               :  use pretrained weights, try more timesteps and bigger LSTM-Unit\n",
      "\n",
      "Use pretrained weights\n"
     ]
    }
   ],
   "source": [
    "vers = \"190322_LSTM_pt\"\n",
    "\n",
    "for key, value in version_dicc[vers].items():\n",
    "    print(f'{key:21}', \": \", value)\n",
    "\n",
    "if \"pt\" in vers:\n",
    "    PRETRAIN = True\n",
    "    pretrain = version_dicc[vers][\"pretrained\"]\n",
    "    print(\"\\nUse pretrained weights\")\n",
    "else:\n",
    "    PRETRAIN = False\n",
    "#model_functions.test_import(\"Import successful.\")\n",
    "\n",
    "debugging = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Import the data\n",
    "Import the recording and labels of the 331 recordings from the data base.\n",
    "\n",
    "#### 2.1 Import the labels of the data set.\n",
    "The 331 labels are saved in a dictionary whose\n",
    "- keys are integers (the ids of the recordings)\n",
    "- values are intervaltrees (for the different intervals of the recording it gives the played notes between 1 and 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(config.labels_path_11, 'rb') as f:\n",
    "    labels = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2 Import recordings\n",
    "\n",
    "Create a dictionary containing the data (recordings). Use Memory-mapped file objects.\n",
    "For each recording data[rec_id] is a tuple:\n",
    "1. data[rec_id][0] is the memory-mapped file object to the recording. It can be accesed by \n",
    "~~~~~~\n",
    "np.frombuffer(data[rec_id][0], dtype=np.float32).copy()\n",
    "~~~~~~\n",
    "2. data[rec_id][1] is its length (float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = dict()\n",
    "for record in os.listdir(config.records_path_11_new):\n",
    "    fd = os.open(config.records_path_11_new + record, os.O_RDONLY)\n",
    "    buff = mmap.mmap(fd, 0, mmap.MAP_SHARED, mmap.PROT_READ)\n",
    "    data[int(record[:-4])] = (buff, len(buff)/model_functions_norm.sz_float)\n",
    "    os.close(fd)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.3 Split up in the test and train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 11 samples in the test set\n",
      "There are 320 samples in the train set\n"
     ]
    }
   ],
   "source": [
    "extended_test_set = True\n",
    "use_mirex = True\n",
    "\n",
    "# use the small or big test set\n",
    "if extended_test_set:\n",
    "    test_ids = config.test_ids_ext\n",
    "else:\n",
    "    test_ids = config.test_ids\n",
    "\n",
    "# include the mirex dev set in test results\n",
    "if use_mirex:\n",
    "    test_ids = test_ids + config.mirex_id\n",
    "\n",
    "train_ids = [rec_id for rec_id in labels.keys() if rec_id not in test_ids]\n",
    "\n",
    "if debugging == True:\n",
    "    train_ids = [rec_id for rec_id in train_ids if rec_id not in model_functions_norm.delete_ids]\n",
    "\n",
    "print('There are {} samples in the test set'.format(len(test_ids)))\n",
    "print('There are {} samples in the train set'.format(len(train_ids)))\n",
    "    \n",
    "#model_functions_norm.get_mean_std(train_ids, data)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.4 Calculate mean and Standard Deviation\n",
    "\n",
    "The mean is approximatly 0 and standard deviation 1. Therefore, no normalization is applied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#_,_ = model_functions_norm.mean_std(data, train_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Preprocessing\n",
    "\n",
    "The input data (recordings) are normalized and the possible note range is defines to 128 notes. The lowest note is ..., the highest ... ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Normalize the recordings\n",
    "normalize = True\n",
    "# Possible notes are from range [base_note, base_note + m)\n",
    "base_note = 0\n",
    "mm = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1 Data augmentation\n",
    "\n",
    "Always make two small changes to input data by randomly stretching or shrinking our input audio that are not noticable by human hearing, but augment the data set:\n",
    "1. Pitch-shift in the frequency domain: Randomly  shifting  each data points in a minibatch by an integral number of semitones. This reinforces the architectural structure of the translation-invariant network.\n",
    "2. Jittering: continuous shift to each data point. This makes the models more robust to tuning variation between recordings --> regularization\n",
    "\n",
    "Define a function for extracting a sequence and its label as well as applying the data augmentation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pitching: stretching or shrinking our input audio with linear interpolation\n",
    "pitch_transforms=5 # =/- 5 semitones\n",
    "jitter=.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pitch-shifting corresponds to a multiplication by $2^{1/12}$ (an octave is equal to 12 semitones and corresponds to a factor 2 in the frequency domain) for each semirone.\n",
    "\n",
    "A scaling factor in the frequency domain implies also a scaling factor in the time domain. (speeding up the recording $\\to$ higher frequency $\\to$ sounds higher.\n",
    "\n",
    "All these methods are applied in the function \n",
    "~~~~~~\n",
    "model_functions.get_data(rec_id, timesteps, s, window, pitch_shift, scaling_factor, normalize=True)\n",
    "~~~~~~\n",
    "\n",
    "#### 3.2 Batch-selection\n",
    "Create the training batch applying the above explained methods for Data augmentation utilizing the function\n",
    "~~~~~~\n",
    "model_functions.get_training_batch(model_stats, batch_size, timesteps, window, pitch_transforms=0, jitter=0, normalize=True)\n",
    "~~~~~~\n",
    "\n",
    "#### 3.3 Creation of Training and Test set for protocolling\n",
    "\n",
    "Routines that create sample for the protcoling:\n",
    "1. Routine to obtain the training samples given the recording IDs of the training samples.\n",
    "   -> If sample with same settings already got created and was saved to file it is loaded from numpy.\n",
    "   -> Else: create the sample by taking at most 100 samples for each recording. Save it to file.\n",
    "   \n",
    "  ~~~~~~\n",
    "  model_functions.get_training_sample(rec_ids, timesteps, d=16384, normalize=True)\n",
    "  ~~~~~~\n",
    "   \n",
    "2. Routine to obtain test samples given the recording IDs of the test samples.\n",
    "   -> If sample with same settings already got created and was saved to file it is loaded from numpy.\n",
    "   -> Else: create the sample by taking at most 1000 samples for each recording. Save it to file.\n",
    "   ~~~~~~\n",
    "   model_functions.get_test_sample(rec_ids, window, count, timesteps, fixed_stride=-1, pitch_shift=0, normalize=True)\n",
    "   ~~~~~~\n",
    "   \n",
    "### 4. Prepare Protocolling\n",
    "Save the results, losses, precicions and weights at each protocol point in a diccionary, that is created and updated using the class \n",
    "~~~~~~\n",
    "model_results\n",
    "~~~~~~\n",
    "\n",
    "### 5. Error measures\n",
    "Define the error measures for comparing the correct labeling Y with the extimated labeling Y_hat.\n",
    "Both labelings are of size (mm) for each segment. The errors are calculated at each protocol point for the training and the test set.\n",
    "1. Mean Squared Error: When applying the model to a sequence or sequences the mean squared error is calculated directly by applying the tensorflow function.\n",
    "~~~~~~~~~~~~~~~~~~~~~\n",
    "tf.reduce_mean(tf.nn.l2_loss( Y_hat - Y ))\n",
    "~~~~~~~~~~~~~~~~~~~~~\n",
    "2. Average Precision: The average precision is calculated with the metric function from the sklear package.\n",
    "~~~~~~~~~~~~~~~~~~~~~\n",
    "sklearn.metrics.average_precision_score\n",
    "~~~~~~~~~~~~~~~~~~~~~\n",
    "\n",
    "### 6. Define the graph\n",
    "\n",
    "Define batch size (batch_size segments at each training step). Also define a small factor wscale to scale the weights. Regularization can be avoided."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size=150\n",
    "if debugging == True:\n",
    "    batch_size = 2\n",
    "out = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6.1 Define layers\n",
    "\n",
    "#### First layer: \n",
    "\n",
    "Lenth of the input (segment) is window=16384. Compute a strided convolution with a d=4096-sample recptive field and a stride=512 sample stride. Since every segment has 16384 samples there are 25 regions for each frame.\n",
    "\n",
    "After the convolution for each region a filterbank computation is calculated. \n",
    "As a filter a log-spaced filterbank is chosen (k=512 sine and cosine filters with logarithmically spaced frequencies from 50 hz to 6000 hz) which is going to be applied to all the regions of the convolution of size d=4096.\n",
    "The filterbank is created by \n",
    "~~~~~~\n",
    "model_functions.create_filters(d,k)\n",
    "~~~~~~\n",
    "\n",
    "#### Second layer:\n",
    "A convolution is applied along the log-frequency axis opening up a third channel dimension.\n",
    "\n",
    "#### Third layer:\n",
    "Convolution along the log-frequency axis. Filters of height 1 are used that fully connect along the time and channel axes of Layer 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##### variables for input layer\n",
    "\n",
    "# input size of segment, predict notes played at midpoint of segment\n",
    "window = version_dicc[vers][\"window\"]\n",
    "\n",
    "# number of timesteps to be considered for memory cell\n",
    "timesteps = version_dicc[vers][\"timesteps\"]\n",
    "\n",
    "# size of regions that outputs the convolution from layer one (receptive field)\n",
    "dd = version_dicc[vers][\"dd\"]\n",
    "\n",
    "#stride for first layer convolution --> 13 regions\n",
    "stride = version_dicc[vers][\"stride\"]\n",
    "\n",
    "# variables for filterbank (= first layer)\n",
    "# number of nodes in first layer (by filterbank get from dd nodes to kk nodes)\n",
    "kk = version_dicc[vers][\"kk\"]\n",
    "wsin,wcos = model_functions_norm.create_filters(dd,kk)\n",
    "\n",
    "##### variables for second layer\n",
    "d2_x = version_dicc[vers][\"d2_x\"]           # lvl2 input dims_x\n",
    "d2_y = version_dicc[vers][\"d2_y\"]           # lvl2 input dims_y\n",
    "k2 = version_dicc[vers][\"k2\"]               # num lvl2 filters\n",
    "stride_y = version_dicc[vers][\"stride_y\"]   # lvl2 stride\n",
    "\n",
    "##### variables for third layer\n",
    "d3_x = version_dicc[vers][\"d3_x\"]           # lvl3 input dims_x\n",
    "d3_y = version_dicc[vers][\"d3_y\"]           # lvl3 input dims_y (fully connected)\n",
    "k3 = version_dicc[vers][\"k3\"]               # num lvl3 filters\n",
    "\n",
    "#### variables for LSTM-cell\n",
    "num_units = version_dicc[vers][\"num_units\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the sizes of nodes at each layer and the first layer filterbank."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First layer regions: (13,256)\n",
      "Second layer regions: (13,65)\n",
      "Third layer regions: (1,65)\n"
     ]
    }
   ],
   "source": [
    "num_regions, num_regions2_x, num_regions2_y, num_regions3_x, num_regions3_y = \\\n",
    "model_functions_norm.calc_region_sizes(window, dd, stride, kk, d2_x, d2_y, stride_y, d3_x, d3_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6.2 Define tf.Graph()\n",
    "Set it all together and define the graph.\n",
    "\n",
    "Define:\n",
    "- the starter leraning rate\n",
    "- learning rate decay\n",
    "- number of steps after which learning rate decay is applied\n",
    "- the L2-regularization weight\n",
    "- the Momentum for the tf.train.MomentumOptimizer\n",
    "\n",
    "For evaluation not the trained weights, but its moving averages with decay 0.9998 are utilized. The shadow variables are maintained by tf.train.ExponentialMovingAverage. In the block for evaluating the direct model their values are retrieved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---- Weights ----\n",
      "w <tf.Variable 'parameters/w:0' shape=(1, 128, 1, 128) dtype=float32_ref>\n",
      "w2 <tf.Variable 'parameters/w2:0' shape=(13, 1, 128, 1024) dtype=float32_ref>\n",
      "beta <tf.Variable 'parameters/beta:0' shape=(66560, 128) dtype=float32_ref>\n",
      "w_proj <tf.Variable 'parameters_train/w_proj:0' shape=(2048, 128) dtype=float32_ref>\n",
      "\n",
      "---- Layers ----\n",
      "zx Tensor(\"model/add:0\", shape=(?, 1, 13, 256), dtype=float32)\n",
      "z2 Tensor(\"model/Relu:0\", shape=(?, 128, 13, 65), dtype=float32)\n",
      "z3 Tensor(\"model/Relu_1:0\", shape=(?, 1024, 1, 65), dtype=float32)\n",
      "y_feed_forward Tensor(\"model/MatMul:0\", shape=(1350, 128), dtype=float32)\n",
      "\n",
      "---- LSTM-Cell ----\n",
      "Number of units: 1024\n",
      "Shape of input: (150, 9, 128)\n",
      "Shape of output: 2 x (150, 1024)\n",
      "y_output: Tensor(\"model/MatMul_1:0\", shape=(150, 128), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# initialize diccionary to save results\n",
    "model_stats = model_functions_norm.model_results(test_ids, train_ids, labels, data, base_note, mm)\n",
    "\n",
    "tf.set_random_seed(999)\n",
    "\n",
    "starter_learning_rate = version_dicc[vers][\"starter_learning_rate\"]  # starter leraning rate\n",
    "decay_steps = version_dicc[vers][\"decay_steps\"]                      # steps until learning rate decay\n",
    "lr_decay = version_dicc[vers][\"lr_decay\"]                            # learning rate decay\n",
    "mom = version_dicc[vers][\"mom\"]                                      # Momentum for Optimizer\n",
    "beta_reg = 0                                                         # L2-regularization weight\n",
    "\n",
    "\n",
    "if PRETRAIN:\n",
    "    graph, training_op, loss, global_step,lr,reg,y_pd,direct_loss,xb,yb,xd,yd,saver,w,wavg,w2,w2avg,beta,betaavg = \\\n",
    "                                    version_dicc[vers][\"graph\"] (\n",
    "                                                model_stats, batch_size, window, timesteps, \n",
    "                                                stride, out, wsin, wcos, \n",
    "                                                d2_x, d2_y, k2, stride_y, \n",
    "                                                d3_x, d3_y, k3, num_regions3_x, num_regions3_y, num_units,\n",
    "                                                starter_learning_rate, decay_steps, lr_decay)\n",
    "else:   \n",
    "    graph, training_op, loss, global_step, lr, reg, y_pd, direct_loss, xb, yb, xd, yd, saver = \\\n",
    "                                    version_dicc[vers][\"graph\"] (\n",
    "                                                model_stats, batch_size, window, timesteps, \n",
    "                                                stride, out, wsin, wcos, \n",
    "                                                d2_x, d2_y, k2, stride_y, \n",
    "                                                d3_x, d3_y, k3, num_regions3_x, num_regions3_y, num_units,\n",
    "                                                starter_learning_rate, decay_steps, lr_decay)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6.3 Evaluation of the graph\n",
    "\n",
    "For the aplication of the neuronal network a moving average of the weights is used. Therefore, a different function for evaluation is used.\n",
    "\n",
    "Inside the function \n",
    "~~~~~~\n",
    "model_functions.predict_direct_model(sess, X, Y) \n",
    "~~~~~~\n",
    "the mean squared error is calculated directly.\n",
    "\n",
    "### 7. Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_steps = version_dicc[vers][\"max_steps\"]+1\n",
    "control_step = 1000\n",
    "\n",
    "if debugging == True:\n",
    "    num_steps = 5\n",
    "    control_step = 1\n",
    "\n",
    "gpu_memory_growth = False\n",
    "\n",
    "# arrays for input data (batch)\n",
    "xmb = np.empty([timesteps*batch_size,1,window,1],dtype=np.float32)\n",
    "ymb = np.empty([timesteps*batch_size,1,mm],dtype=np.float32)\n",
    "\n",
    "\n",
    "if PRETRAIN:\n",
    "    pretrained = tf.train.Saver({'w':w,'wavg':wavg,'w2':w2,'w2avg':w2avg,'beta':beta,'betaavg':betaavg})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "INFO:tensorflow:Restoring parameters from models/model_190319_11HZ/model-149999\n",
      "Model restored.\n",
      "iter\t test_squ_loss\t test_avg_prec\t train_squ_loss\t train_avg_prec\t regularization\t time (in s)\n",
      "1 \t 13.18313729 \t 0.05280126 \t 97.89511505 \t 5.912e-05 \t 120.67827 \t 63.0394268\n",
      "1001 \t 10.69451646 \t 0.48306763 \t 0.16975219 \t 0.53858495 \t 110.63314 \t 273.34225583\n",
      "2001 \t 10.34700216 \t 0.49372473 \t 0.11702802 \t 0.56319306 \t 108.410904 \t 232.17761898\n",
      "3001 \t 10.05900031 \t 0.49507636 \t 0.10410093 \t 0.57097936 \t 112.15851 \t 233.26622963\n",
      "4001 \t 9.87731335 \t 0.50289139 \t 0.09904309 \t 0.57550912 \t 118.21121 \t 233.77083516\n",
      "5001 \t 9.645758 \t 0.50606043 \t 0.0936597 \t 0.59166182 \t 128.5632 \t 232.23883772\n",
      "6001 \t 9.41237712 \t 0.5112112 \t 0.08982135 \t 0.5922323 \t 137.80467 \t 233.74261212\n",
      "7001 \t 9.09859399 \t 0.5184459 \t 0.08947814 \t 0.58516509 \t 144.84178 \t 240.550138\n",
      "8001 \t 8.84999459 \t 0.52224673 \t 0.08332106 \t 0.60346727 \t 149.12273 \t 233.76279521\n",
      "9001 \t 8.67375325 \t 0.52769099 \t 0.08055906 \t 0.6100176 \t 152.77515 \t 233.86532736\n",
      "10001 \t 8.45957614 \t 0.53931455 \t 0.07941461 \t 0.60591445 \t 154.31789 \t 238.32651544\n",
      "11001 \t 8.31740098 \t 0.54282245 \t 0.07788738 \t 0.62022531 \t 154.6583 \t 239.69523525\n",
      "12001 \t 8.25998942 \t 0.54637016 \t 0.08839949 \t 0.62566216 \t 155.95174 \t 236.88243556\n",
      "13001 \t 8.19399694 \t 0.55014063 \t 0.08212623 \t 0.62561384 \t 157.24094 \t 234.31798959\n",
      "14001 \t 8.07988379 \t 0.55723304 \t 0.08251384 \t 0.6184698 \t 159.03183 \t 232.99513006\n",
      "15001 \t 8.09701786 \t 0.55535692 \t 0.08119048 \t 0.62508808 \t 159.8551 \t 239.1661067\n",
      "16001 \t 8.07725033 \t 0.55619355 \t 0.07774589 \t 0.63618267 \t 161.47688 \t 242.44982266\n",
      "17001 \t 7.98581133 \t 0.56087408 \t 0.0807693 \t 0.63842509 \t 163.12045 \t 242.19547963\n",
      "18001 \t 8.007449 \t 0.5587602 \t 0.08399467 \t 0.6250685 \t 164.05688 \t 244.84783173\n",
      "19001 \t 8.00246391 \t 0.56051278 \t 0.08579204 \t 0.63029034 \t 165.29239 \t 242.30821967\n",
      "20001 \t 7.99108254 \t 0.55892994 \t 0.09509787 \t 0.63605271 \t 166.51721 \t 242.20112443\n",
      "21001 \t 7.96738315 \t 0.56047316 \t 0.09763053 \t 0.63153494 \t 166.98099 \t 234.33127642\n",
      "22001 \t 7.90556261 \t 0.56688366 \t 0.09997757 \t 0.62801361 \t 167.53842 \t 232.41453171\n",
      "23001 \t 7.91355755 \t 0.56583227 \t 0.09941461 \t 0.63309247 \t 167.73697 \t 232.70122266\n",
      "24001 \t 7.92649724 \t 0.5662793 \t 0.10167433 \t 0.64335398 \t 170.23766 \t 234.85043573\n",
      "25001 \t 7.84037642 \t 0.57253846 \t 0.11114898 \t 0.64418738 \t 171.5474 \t 232.48832631\n",
      "26001 \t 7.84661278 \t 0.57266263 \t 0.11570845 \t 0.65186122 \t 172.36931 \t 231.67217708\n",
      "27001 \t 7.88740863 \t 0.5681158 \t 0.11191093 \t 0.6428041 \t 173.9468 \t 229.3048563\n",
      "28001 \t 7.84488071 \t 0.57375157 \t 0.12348004 \t 0.65535992 \t 175.38959 \t 229.8255949\n",
      "29001 \t 7.83705552 \t 0.57377286 \t 0.13738605 \t 0.65154801 \t 176.97261 \t 230.27666473\n",
      "30001 \t 7.85831144 \t 0.57298534 \t 0.15303881 \t 0.65218527 \t 178.82272 \t 227.78507471\n",
      "31001 \t 7.90233023 \t 0.57107082 \t 0.16977527 \t 0.65451734 \t 179.25134 \t 228.7396791\n",
      "32001 \t 7.83195541 \t 0.57559942 \t 0.19393432 \t 0.66032739 \t 180.36046 \t 228.70019341\n",
      "33001 \t 7.82837905 \t 0.57660538 \t 0.20343363 \t 0.64913545 \t 181.73279 \t 229.69924974\n",
      "34001 \t 7.81162906 \t 0.57749801 \t 0.16808874 \t 0.64956709 \t 183.15811 \t 228.27091765\n",
      "35001 \t 7.82016329 \t 0.57575936 \t 0.27105351 \t 0.66658641 \t 184.75084 \t 229.66496992\n",
      "36001 \t 7.78569364 \t 0.57985721 \t 0.20241603 \t 0.65531549 \t 186.02899 \t 232.61574125\n",
      "37001 \t 7.77269484 \t 0.58072528 \t 0.17651261 \t 0.64736947 \t 187.4521 \t 228.95271158\n",
      "38001 \t 7.80469189 \t 0.58081318 \t 0.1611571 \t 0.65123582 \t 189.29515 \t 231.50003386\n",
      "39001 \t 7.82270554 \t 0.57788719 \t 0.13194217 \t 0.66059511 \t 190.50937 \t 232.68415761\n",
      "40001 \t 7.83647825 \t 0.57994929 \t 0.15717229 \t 0.67120834 \t 191.98915 \t 232.87658548\n",
      "41001 \t 7.80830805 \t 0.57874373 \t 0.24195197 \t 0.65662601 \t 193.00075 \t 230.30868602\n",
      "42001 \t 7.78615067 \t 0.58073007 \t 0.15082369 \t 0.65421395 \t 194.41818 \t 230.88319302\n",
      "43001 \t 7.75878858 \t 0.58372478 \t 0.19206598 \t 0.65530527 \t 195.7088 \t 230.30169892\n",
      "44001 \t 7.76014365 \t 0.58450139 \t 0.19629255 \t 0.65958051 \t 197.32784 \t 231.15877271\n",
      "45001 \t 7.74154042 \t 0.58544832 \t 0.29692926 \t 0.65559054 \t 198.18925 \t 231.45579433\n",
      "46001 \t 7.71880457 \t 0.5872512 \t 0.25175498 \t 0.65441953 \t 199.38037 \t 231.88605022\n",
      "47001 \t 7.7310418 \t 0.58653817 \t 0.21929682 \t 0.65503747 \t 201.63141 \t 234.99150872\n",
      "48001 \t 7.70270282 \t 0.58810205 \t 0.15481636 \t 0.66457879 \t 202.65196 \t 231.62206364\n",
      "49001 \t 7.75135833 \t 0.58506182 \t 0.13821364 \t 0.65171075 \t 203.63336 \t 231.7831862\n",
      "50001 \t 7.71344637 \t 0.58786376 \t 0.18867459 \t 0.65682809 \t 205.18582 \t 232.83149242\n",
      "51001 \t 7.74035103 \t 0.58726656 \t 0.17423394 \t 0.66052326 \t 205.765 \t 231.90046906\n",
      "52001 \t 7.71805672 \t 0.58809253 \t 0.17827838 \t 0.66184379 \t 207.80885 \t 232.5903585\n",
      "53001 \t 7.68370377 \t 0.58984583 \t 0.14202659 \t 0.65631768 \t 208.50504 \t 234.8929832\n",
      "54001 \t 7.72032443 \t 0.58692772 \t 0.15580878 \t 0.66957705 \t 210.11728 \t 232.19026494\n",
      "55001 \t 7.74529252 \t 0.58744177 \t 0.14162746 \t 0.66399081 \t 210.29245 \t 233.66887188\n",
      "56001 \t 7.68386291 \t 0.590009 \t 0.16004941 \t 0.67051294 \t 212.28445 \t 234.13538527\n",
      "57001 \t 7.72502683 \t 0.58770114 \t 0.12978855 \t 0.66837786 \t 213.11433 \t 235.32617974\n",
      "58001 \t 7.65940422 \t 0.59137533 \t 0.12893965 \t 0.66377698 \t 214.48244 \t 233.10110974\n",
      "59001 \t 7.64325841 \t 0.5951584 \t 0.13147683 \t 0.66412948 \t 215.77695 \t 233.66004562\n",
      "60001 \t 7.68278137 \t 0.59252545 \t 0.12841028 \t 0.66131653 \t 217.87811 \t 233.10085797\n",
      "61001 \t 7.7085833 \t 0.59109744 \t 0.14122757 \t 0.67069437 \t 217.89069 \t 234.54810762\n",
      "62001 \t 7.64154167 \t 0.59405298 \t 0.14563452 \t 0.66532685 \t 219.48131 \t 234.06425214\n",
      "63001 \t 7.65640926 \t 0.59368125 \t 0.2232155 \t 0.66565 \t 220.9865 \t 233.70990038\n",
      "64001 \t 7.6688694 \t 0.59113583 \t 0.14531041 \t 0.66336653 \t 222.03867 \t 233.5592041\n",
      "65001 \t 7.69078147 \t 0.58934266 \t 0.16228897 \t 0.66273687 \t 222.58173 \t 233.77927661\n",
      "66001 \t 7.66545663 \t 0.59224568 \t 0.19803903 \t 0.66518878 \t 224.57506 \t 234.45721889\n",
      "67001 \t 7.68446061 \t 0.5938695 \t 0.12509231 \t 0.65945853 \t 226.07768 \t 234.12057638\n",
      "68001 \t 7.61469184 \t 0.59669792 \t 0.12737626 \t 0.66104519 \t 227.31647 \t 234.07785845\n",
      "69001 \t 7.61624104 \t 0.59598134 \t 0.14605276 \t 0.6706593 \t 229.02274 \t 233.96353054\n",
      "70001 \t 7.63548668 \t 0.5941792 \t 0.13659609 \t 0.6680785 \t 230.69215 \t 233.69436026\n",
      "71001 \t 7.6220819 \t 0.5956977 \t 0.1044057 \t 0.66532416 \t 230.66058 \t 234.50999594\n",
      "72001 \t 7.64088595 \t 0.59484764 \t 0.11334252 \t 0.66351069 \t 232.47209 \t 233.74002337\n",
      "73001 \t 7.64184122 \t 0.59478311 \t 0.12102881 \t 0.65587663 \t 232.80136 \t 234.63191676\n",
      "74001 \t 7.63779814 \t 0.59414229 \t 0.11135557 \t 0.65915367 \t 234.50894 \t 234.05545664\n",
      "75001 \t 7.62019379 \t 0.59659875 \t 0.11190495 \t 0.66578453 \t 235.47278 \t 234.0595119\n",
      "76001 \t 7.62356007 \t 0.59665653 \t 0.18322824 \t 0.66435538 \t 236.55182 \t 233.71394777\n",
      "77001 \t 7.59024944 \t 0.6001377 \t 0.12997499 \t 0.66181232 \t 237.74023 \t 234.25120568\n",
      "78001 \t 7.60685494 \t 0.59756321 \t 0.20623249 \t 0.66680925 \t 239.45236 \t 234.36568403\n",
      "79001 \t 7.63238654 \t 0.59751228 \t 0.12522944 \t 0.6738787 \t 240.16228 \t 233.21512127\n",
      "80001 \t 7.60925145 \t 0.59757165 \t 0.11923286 \t 0.67358441 \t 241.72327 \t 233.5013957\n",
      "81001 \t 7.60102648 \t 0.59935982 \t 0.16455987 \t 0.66964343 \t 242.82925 \t 233.70055914\n",
      "82001 \t 7.60820027 \t 0.59871095 \t 0.1133826 \t 0.67025806 \t 243.55768 \t 233.40656471\n",
      "83001 \t 7.58038836 \t 0.60071594 \t 0.10844676 \t 0.68790262 \t 244.68896 \t 233.64388156\n",
      "84001 \t 7.62164282 \t 0.5962228 \t 0.12440072 \t 0.66845327 \t 246.19093 \t 233.36507487\n",
      "85001 \t 7.56463887 \t 0.60021663 \t 0.12516397 \t 0.67091832 \t 246.60239 \t 233.52542424\n",
      "86001 \t 7.58051058 \t 0.59998827 \t 0.19369108 \t 0.66594514 \t 247.28099 \t 232.88563657\n",
      "87001 \t 7.58322115 \t 0.60149643 \t 0.17606763 \t 0.66575631 \t 248.8861 \t 234.07508659\n",
      "88001 \t 7.57927362 \t 0.59958625 \t 0.16877077 \t 0.6713033 \t 249.76164 \t 234.13003922\n",
      "89001 \t 7.59208014 \t 0.59956749 \t 0.14287319 \t 0.67057805 \t 250.53712 \t 258.74001503\n",
      "90001 \t 7.57047815 \t 0.60107581 \t 0.11372264 \t 0.67724925 \t 251.1359 \t 233.65820599\n",
      "91001 \t 7.58179662 \t 0.60110267 \t 0.14004396 \t 0.66354029 \t 251.76505 \t 233.88952184\n",
      "92001 \t 7.56601226 \t 0.60225555 \t 0.14166524 \t 0.67139003 \t 252.53775 \t 233.39063931\n",
      "93001 \t 7.58261458 \t 0.60296191 \t 0.13488912 \t 0.66946544 \t 253.74673 \t 232.90435982\n",
      "94001 \t 7.62843364 \t 0.59664914 \t 0.14219423 \t 0.66568014 \t 254.60712 \t 233.29196763\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95001 \t 7.58913076 \t 0.60066228 \t 0.12810239 \t 0.67344961 \t 255.7249 \t 233.08738899\n",
      "96001 \t 7.51473546 \t 0.60562079 \t 0.11843301 \t 0.67606466 \t 256.70346 \t 233.57630944\n",
      "97001 \t 7.56614211 \t 0.60071388 \t 0.1336378 \t 0.67289428 \t 256.90967 \t 233.3524754\n",
      "98001 \t 7.56241515 \t 0.60401771 \t 0.1658953 \t 0.66938631 \t 258.03296 \t 233.2935226\n",
      "99001 \t 7.52503132 \t 0.60589801 \t 0.12144655 \t 0.67836046 \t 258.89725 \t 234.59221125\n",
      "\n",
      "TRAINING DONE\n"
     ]
    }
   ],
   "source": [
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(\"Initialized\")\n",
    "    ptime = time.time()\n",
    "    tfconfig = tf.ConfigProto()\n",
    "    tfconfig.gpu_options.allow_growth=gpu_memory_growth\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess = tf.Session(config=tfconfig)\n",
    "    \n",
    "    if PRETRAIN:\n",
    "        pretrained.restore(sess, 'models/model_'+pretrain+'/model-'+str(version_dicc[pretrain][\"max_steps\"]))\n",
    "        print(\"Model restored.\")\n",
    "           \n",
    "    sess.run(init_op)\n",
    "    \n",
    "    print ('iter\\t test_squ_loss\\t test_avg_prec\\t',\n",
    "           'train_squ_loss\\t train_avg_prec\\t',\n",
    "           'regularization\\t time (in s)')\n",
    "    for step in range(num_steps):\n",
    "       \n",
    "        xmb, ymb = model_functions_norm.get_training_batch(model_stats, batch_size, timesteps, window, \n",
    "                                      pitch_transforms, jitter)\n",
    "        \n",
    "        # every 1000-nd step save the results           \n",
    "        if step%control_step==0:\n",
    "            # training step \n",
    "            # to check output the loss (save the loss but you cannot compare it with the mse_train)\n",
    "            _, np_loss, np_global_step, np_lr, reg_np = \\\n",
    "            sess.run([training_op, loss, global_step, lr, reg], \n",
    "                                     feed_dict={xb: xmb, yb: ymb})\n",
    "            \n",
    "            #print(\"training step done\")\n",
    "            # A) run model on test set\n",
    "                        \n",
    "            # 1. get the test set\n",
    "            X,Y = model_functions_norm.get_test_sample(model_stats, timesteps, window, 1000, debugging=debugging)\n",
    "\n",
    "            Y = Y[:,base_note:base_note+mm] # in general without effect\n",
    "            # shape Xtest: (1000*len(test_ids), timesteps, 1, 16384, 1)   -> (11000, 5, 1, 16384, 1)\n",
    "            # shape Ytest: (1000*len(test_ids), 128)\n",
    "            # extended test-set: 11 samples\n",
    "\n",
    "            \n",
    "            # 2. prediction of output and calculation of\n",
    "            # - MSE\n",
    "            # - average precision\n",
    "            Y_p, mse_test = model_functions_norm.predict_direct_model(sess, y_pd, direct_loss, xd, yd, X, Y)\n",
    "            avp_test = average_precision_score(Y.flatten(), Y_p.flatten())\n",
    "            del X,Y,Y_p\n",
    "            \n",
    "            # B) run model on training set\n",
    "            \n",
    "            # 1. get training sample\n",
    "            X,Y = model_functions_norm.get_training_sample(model_stats, timesteps, window, debugging=debugging) #10-20 seconds\n",
    "            Y = Y[:,base_note:base_note+mm] # in general without effect\n",
    "            # shape Xtrain: (100*len(train_ids), timesteps, 1, 16384, 1)   -> (32000, 5, 1, 16384, 1)\n",
    "            # shape Ytrain: (100*len(train_ids), 128)\n",
    "            # train-set: 320 samples\n",
    "            \n",
    "            # 2. prediction of output and calculation of\n",
    "            # - MSE\n",
    "            # - average precision\n",
    "            Y_p, mse_train = model_functions_norm.predict_direct_model(sess, y_pd, direct_loss, xd, yd, X, Y) #110-120 seconds\n",
    "            avp_train = average_precision_score(Y.flatten(), Y_p.flatten()) \n",
    "            \n",
    "            del X,Y,Y_p\n",
    "            \n",
    "            # c) Print the temporary results\n",
    "            print (np_global_step,'\\t', round(mse_test,8),\n",
    "                    '\\t', round(avp_test,8),\n",
    "                    '\\t', round(mse_train,8),\n",
    "                    '\\t', round(avp_train,8),\n",
    "                    '\\t', round(reg_np,8),\n",
    "                    '\\t', round(time.time() - ptime,8))\n",
    "            \n",
    "            #D) Save the temporary results\n",
    "            model_stats.stats['iter'][2].append(np_global_step)\n",
    "            # protocol learning rate\n",
    "            model_stats.stats['lr'][2].append(np_lr)\n",
    "            # protocol runtime\n",
    "            model_stats.stats['time'][2].append(time.time() - ptime)\n",
    "            # protocol weights and its norms\n",
    "            weight_norms = dict()\n",
    "            for name, weight in model_stats.weights.items():\n",
    "                weight_norms[name] = np.mean(np.linalg.norm(weight.eval(session=sess),axis=0))\n",
    "            for name, norm in weight_norms.items():\n",
    "                model_stats.stats['n'+name][2].append(norm)\n",
    "            # protocol training errors     \n",
    "            model_stats.stats['mse_train_1'][2].append(np_loss)\n",
    "            model_stats.stats['mse_train'][2].append(mse_train)\n",
    "            model_stats.stats['avp_train'][2].append(avp_train)\n",
    "            # protocol test errors\n",
    "            model_stats.stats['mse_test'][2].append(mse_test)\n",
    "            model_stats.stats['avp_test'][2].append(avp_test)\n",
    "            ptime = time.time()            \n",
    "                   \n",
    "        else:\n",
    "            #training step \n",
    "            _ = sess.run([training_op], feed_dict={xb: xmb, yb: ymb})\n",
    "            \n",
    "        if step/num_steps==0.5 or (step+1)/num_steps==1:\n",
    "            saver.save(sess, 'models/model_'+vers+'/model', global_step=step)\n",
    "                                \n",
    "    print ('\\nTRAINING DONE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save diccionary to have a look at outputs at other times\n",
    "model_functions_norm.save_obj(model_stats.stats, 'results/'+vers+'_stats')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Post Processing\n",
    "#### 8.1 Plots\n",
    "\n",
    "Plot leraning rate, possibly runtime, errors, precision and norms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results_stats = model_functions_norm.load_obj('results/'+vers+'_stats')\n",
    "\n",
    "# size of plots\n",
    "plot_characteristics = {'width': 20, 'height': 13, 'fontsize': 24, 'control_step': 1000, 'savefig': False}\n",
    "\n",
    "plt.rcParams['xtick.labelsize']=plot_characteristics['fontsize']\n",
    "plt.rcParams['ytick.labelsize']=plot_characteristics['fontsize']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################################################################\n",
    "# learning rate\n",
    "model_functions_norm.plot_lr(results_stats, plot_characteristics, vers)\n",
    "\n",
    "################################################################\n",
    "# error and precision\n",
    "model_functions_norm.plot_prec(results_stats, plot_characteristics, vers)\n",
    "\n",
    "################################################################\n",
    "# norms of the weights\n",
    "if \"LSTM\" not in vers:\n",
    "    model_functions_norm.plot_norms(results_stats, plot_characteristics, vers)\n",
    "else:\n",
    "    model_functions_norm.plot_norms_lstm(results_stats, plot_characteristics, vers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### 8.2 Mirex Statistics\n",
    "\n",
    "Statistics that are necesarry for evaluation. The autuor used a threshold of 0.4 for calculation of error etc...\n",
    "\n",
    "First, restore the model and calculate the output to the test sample:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from models/model_190322_LSTM_pt/model-99999\n",
      "Model restored.\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "num_steps = version_dicc[vers][\"max_steps\"]\n",
    "timesteps = version_dicc[vers][\"timesteps\"]\n",
    "\n",
    "\n",
    "\n",
    "# Later, launch the model, use the saver to restore variables from disk, and\n",
    "# do some work with the model.\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    # Restore variables from disk.\n",
    "    saver.restore(sess, 'models/model_'+vers+'/model-'+str(num_steps))\n",
    "    print(\"Model restored.\")\n",
    "\n",
    "    # 1. get the test set\n",
    "    X, Y = model_functions_norm.get_test_sample(model_stats, timesteps, window, 1000)\n",
    "    Y = Y[:,base_note:base_note+mm] # in general without effect\n",
    "    # shape Xtest: (1000*len(test_ids), 1, window, 1)\n",
    "    # shape Ytest: (1000*len(test_ids), 128)\n",
    "    # extended test-set: 11 samples\n",
    "\n",
    "    # 2. prediction of output\n",
    "    Y_p, mse_test = model_functions_norm.predict_direct_model(sess, y_pd, direct_loss, xd, yd, X, Y)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With the output of the test sample predicted calculate the mirex statistics:\n",
    "\n",
    "Average Precision (not from mireval), Precision, Recall, Accuracy, Total Error, Substitution, Miss, False Alarm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thr\tAvgP\tP\tR\tAcc\tETot\tESub\tEmiss\tEfa\n",
      "0.30\t60.521\t54.626\t63.994\t0.418\t0.684\t0.207\t0.153\t0.324\n",
      "0.35\t60.521\t58.306\t58.326\t0.412\t0.642\t0.192\t0.225\t0.225\n",
      "0.40\t60.521\t62.079\t53.098\t0.401\t0.626\t0.167\t0.302\t0.157\n",
      "0.45\t60.521\t65.461\t47.291\t0.378\t0.637\t0.140\t0.387\t0.110\n",
      "0.50\t60.521\t68.884\t41.574\t0.350\t0.660\t0.112\t0.472\t0.075\n"
     ]
    }
   ],
   "source": [
    "print('Thr\\tAvgP\\tP\\tR\\tAcc\\tETot\\tESub\\tEmiss\\tEfa')\n",
    "for th in [.3, .35,.4, .45, .5]:\n",
    "    #print(\"When using threshold\",th)\n",
    "    avp,P,R,Acc,Etot = model_functions_norm.mirex_statistics(Y, Y_p, th)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}