performance-vs-dayinmonth.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlating performance with the calendar\n",
    "\n",
    "This notebook attempts to find correlations between performance and different elements of the Gregorian calendar.  Specifically, it attempts to find days on which scheduled maintenance may be causing recurring performance problems for users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import datetime\n",
    "import warnings\n",
    "import matplotlib\n",
    "matplotlib.rcParams.update({'font.size': 16})\n",
    "import pandas\n",
    "import numpy\n",
    "import scipy.stats\n",
    "import abcutils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Below what p-value should we consider correlation significant?\n",
    "SIGNIFICANT_P = 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = abcutils.sc18paper.load_raw_datasets(\n",
    "    input_datasets={\n",
    "#       'edison': 'summaries/edison-summaries_2017-02-14-2018-08-09.csv',\n",
    "        'cori': 'summaries/cori-summaries_2017-02-14-2019-01-31.csv'\n",
    "    },\n",
    "    cache_file='cori-summaries_2017-02-14-2019-01-31.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze_df = df[['_datetime_start', 'darshan_normalized_perf_by_max']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyze_df['day_of_month'] = [x.day for x in df['_datetime_start']]\n",
    "analyze_df['day_of_week'] = [x.isoweekday() for x in df['_datetime_start']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average as a function of day of month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(8, 6))\n",
    "avg_perf = [None] * 32\n",
    "errorbars = [None] * 32\n",
    "for x in range(1, 32):\n",
    "    avg_perf[x] = analyze_df[analyze_df['day_of_month'] == x]['darshan_normalized_perf_by_max'].mean()\n",
    "    errorbars[x] = analyze_df[analyze_df['day_of_month'] == x]['darshan_normalized_perf_by_max'].std()\n",
    "ax.plot(range(1, 32), \n",
    "             avg_perf[1:],\n",
    "             marker='.')\n",
    "#ax.errorbar(range(1, 32), errorbars[1:], label=\"Std Dev\")\n",
    "ax.set_xlabel(\"Day of month\")\n",
    "ax.set_ylabel(\"Fraction of peak I/O performance\")\n",
    "ax.grid()\n",
    "ax.set_title(\"Average daily I/O benchmark performance\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding error bars based on the standard deviation of performance on each day of the month:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(8, 6))\n",
    "ax.errorbar(range(1, 32), \n",
    "             avg_perf[1:],\n",
    "             marker='.',\n",
    "#            label=\"Mean\",\n",
    "             yerr=errorbars[1:])\n",
    "#ax.errorbar(range(1, 32), errorbars[1:], label=\"Std Dev\")\n",
    "ax.set_xlabel(\"Day of month\")\n",
    "ax.set_ylabel(\"Fraction of peak I/O performance\")\n",
    "ax.grid()\n",
    "ax.set_title(\"Average daily I/O benchmark performance\")\n",
    "ax.set_ylim(0, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average as a function of day of week"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(8, 6))\n",
    "avg_perf = [None] * 32\n",
    "errorbars = [None] * 32\n",
    "for x in range(1, 32):\n",
    "    avg_perf[x] = analyze_df[analyze_df['day_of_week'] == x]['darshan_normalized_perf_by_max'].mean()\n",
    "    errorbars[x] = analyze_df[analyze_df['day_of_week'] == x]['darshan_normalized_perf_by_max'].std()\n",
    "ax.plot(range(1, 32), \n",
    "             avg_perf[1:],\n",
    "             marker='.')\n",
    "#ax.errorbar(range(1, 32), errorbars[1:], label=\"Std Dev\")\n",
    "ax.set_xlabel(\"Day of month\")\n",
    "ax.set_xticks(range(1,8))\n",
    "ax.set_xticklabels([\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\"])\n",
    "ax.set_ylabel(\"Fraction of peak I/O performance\")\n",
    "ax.grid()\n",
    "ax.set_title(\"Average daily I/O benchmark performance\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(8, 6))\n",
    "ax.errorbar(range(1, 32), \n",
    "             avg_perf[1:],\n",
    "             marker='.',\n",
    "#            label=\"Mean\",\n",
    "             yerr=errorbars[1:])\n",
    "#ax.errorbar(range(1, 32), errorbars[1:], label=\"Std Dev\")\n",
    "ax.set_xlabel(\"Day of month\")\n",
    "ax.set_xticks(range(1,8))\n",
    "ax.set_xticklabels([\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\"])\n",
    "ax.set_ylabel(\"Fraction of peak I/O performance\")\n",
    "ax.grid()\n",
    "ax.set_title(\"Average daily I/O benchmark performance\")\n",
    "ax.set_ylim(0, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Correlation over week-long windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(8, 6))\n",
    "\n",
    "window_days = 7\n",
    "month_days_max = 31\n",
    "\n",
    "cvals = []\n",
    "weeks = []\n",
    "\n",
    "for week in range(month_days_max // window_days + 1):\n",
    "    domstart = week * window_days + 1\n",
    "    domend = (week + 1) * window_days\n",
    "\n",
    "    filt = analyze_df['day_of_month'] >= domstart\n",
    "    filt &= analyze_df['day_of_month'] <= domend\n",
    "    cval, pval = scipy.stats.pearsonr(analyze_df[filt]['day_of_month'], analyze_df[filt]['darshan_normalized_perf_by_max'])\n",
    "    print(\"Day %2d to %2d: correlation = %10.2e, p-value = %10.2e\" % (domstart, domend, cval, pval))\n",
    "    cvals.append(cval)\n",
    "    weeks.append(week)\n",
    "\n",
    "\n",
    "ax.plot(weeks, cvals,'-', marker='o')\n",
    "ax.set_xlabel(\"Time in month\")\n",
    "ax.set_xticklabels([])\n",
    "ax.set_ylabel(\"Correlation coefficient\")\n",
    "ax.grid()\n",
    "ax.set_title(\"Correlation between good performance and day in month\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(8, 6))\n",
    "\n",
    "window_days = 7\n",
    "month_days_max = 31\n",
    "\n",
    "cvals = []\n",
    "\n",
    "for domstart in range(1, month_days_max - window_days):\n",
    "    domend = domstart + window_days\n",
    "\n",
    "    filt = analyze_df['day_of_month'] >= domstart\n",
    "    filt &= analyze_df['day_of_month'] <= domend\n",
    "    cval, pval = scipy.stats.pearsonr(analyze_df[filt]['day_of_month'], analyze_df[filt]['darshan_normalized_perf_by_max'])\n",
    "    print(\"Day %2d to %2d: correlation = %10.2e, p-value = %10.2e\" % (domstart, domend, cval, pval))\n",
    "    cvals.append((cval, pval))\n",
    "\n",
    "ax.scatter(range(1, month_days_max - window_days),\n",
    "        [x[0] for x in cvals],\n",
    "        s=-50.0 * numpy.log10([x[1] for x in cvals]),\n",
    "#       '-',\n",
    "        marker='o')\n",
    "ax.set_xlabel(\"Day of month\")\n",
    "#x.set_xticklabels([])\n",
    "ax.set_ylabel(\"Correlation coefficient\")\n",
    "ax.grid()\n",
    "ax.set_axisbelow(True)\n",
    "ax.set_title(\"Correlation between good performance and day in month\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}