cug18_correlation_table.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import datetime\n",
    "import pandas\n",
    "import numpy\n",
    "import scipy.stats\n",
    "import abcutils\n",
    "import matplotlib\n",
    "matplotlib.rcParams.update({'font.size': 16})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Global Analysis Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_PLATFORMS = [\n",
    "    'scratch1@edison',\n",
    "#   'scratch2@edison',\n",
    "    'scratch3@edison',\n",
    "    'cscratch@cori-knl',\n",
    "    'mira-fs1@mira'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Synthesize Data from CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "filtered_df = abcutils.sc18paper.load_dataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Global Correlation Table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Show the most compelling correlations across all data.  This will be messy because it includes all file systems and test conditions, so there are many uncontrolled variables represented."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pandas.options.display.max_rows = 40\n",
    "\n",
    "correlation = abcutils.correlation.calc_correlation_vector(filtered_df, correlate_with='darshan_normalized_perf_by_max')\n",
    "\n",
    "filtered_correlations = abcutils.apply_filters(correlation, [correlation['p-value'] < 1.0e-5], verbose=True)\n",
    "filtered_correlations.sort_values('coefficient')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "ax = abcutils.plot.correlation_vector_table(filtered_correlations, row_name_map=abcutils.CONFIG['metric_labels'])\n",
    "ax.get_figure().set_size_inches(4, 0.4 * len(filtered_correlations))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now draw the entire correlation table split out by _test platform_--a combination of the file system being tested and the node configuration being used to test it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "correlations = None\n",
    "grouped_df = filtered_df.groupby('_test_platform')\n",
    "for fs in TEST_PLATFORMS:\n",
    "    # generate a single file system's correlation vector\n",
    "    correlation = abcutils.correlation.calc_correlation_vector(\n",
    "        grouped_df.get_group(fs),\n",
    "        correlate_with='darshan_normalized_perf_by_max')\n",
    "            \n",
    "    # rename the columns in this vector to include the file system name\n",
    "    new_cols = {}\n",
    "    for index, col_name in enumerate(correlation.columns):\n",
    "        new_cols[col_name] = \"%s %s\" % (fs, col_name)\n",
    "    correlation.rename(columns=new_cols, inplace=True)\n",
    "    \n",
    "    # join the vector to the previous vectors' dataframe\n",
    "    if correlations is None:\n",
    "        correlations = correlation\n",
    "    else:\n",
    "        correlations = pandas.concat([correlations, correlation], axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Construct filter to show any metric that registered a low p-value for _any_ file system\n",
    "filters = None\n",
    "for fs in TEST_PLATFORMS:\n",
    "    subfilter = correlations['%s p-value' % fs] < 1.0e-5\n",
    "    if filters is None:\n",
    "        filters = subfilter\n",
    "    else:\n",
    "        filters |= subfilter\n",
    "\n",
    "ax = abcutils.plot.correlation_vector_table(\n",
    "    correlations[filters],\n",
    "    row_name_map=abcutils.CONFIG['metric_labels'])\n",
    "\n",
    "# Set the table width larger if displaying lots of metrics\n",
    "ax.get_figure().set_size_inches(20, 0.4 * len(correlations[filters]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cell_styler(cell_obj, coeff, pval):\n",
    "    if pval < 1.0e-5:\n",
    "        cell_obj.get_text().set_fontweight('bold')\n",
    "    else:\n",
    "        cell_obj.get_text().set_color('#00000099')\n",
    "    set_color = (matplotlib.cm.get_cmap('YlGnBu'))(abs(coeff) / 1.0)\n",
    "\n",
    "    cell_obj.set_color(set_color)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "col_order = [\n",
    "    'ior_fpp_write',\n",
    "    'hacc_io_write_fpp_write',\n",
    "    'ior_fpp_read',\n",
    "    'hacc_io_read_fpp_read',\n",
    "#   'ior_shared_write',\n",
    "#   'vpicio_uni_shared_write',\n",
    "#   'ior_shared_read',\n",
    "#   'dbscan_read_shared_read'\n",
    "]\n",
    "\n",
    "good_counters = [\n",
    "    'coverage_factor_bw',\n",
    "    'coverage_factor_opens',\n",
    "#   'coverage_factor_stats',\n",
    "    'fs_ave_mds_cpu',\n",
    "    'fs_ave_oss_cpu',\n",
    "    'fs_max_mds_cpu',\n",
    "    'fs_max_oss_cpu',\n",
    "    'fshealth_ost_most_full_pct',\n",
    "    'fshealth_ost_overloaded_oss_count',\n",
    "    'topology_job_avg_radius',\n",
    "]\n",
    "\n",
    "correlations = None\n",
    "\n",
    "apply_filters = filtered_df['_test_platform'] == 'cscratch@cori-knl'\n",
    "#apply_filters &= filtered_df['coverage_factor_stats'] > 0.0\n",
    "#apply_filters &= numpy.isfinite(filtered_df['coverage_factor_stats'])\n",
    "#apply_filters &= filtered_df['coverage_factor_bw'] != 0.0\n",
    "#apply_filters &= filtered_df['coverage_factor_opens'] != 0.0\n",
    "#apply_filters &= filtered_df['_datetime_start'] >= datetime.datetime(2017, 8, 1)\n",
    "#apply_filters &= filtered_df['_datetime_start'] < datetime.datetime(2018, 1, 1)\n",
    "\n",
    "input_df = filtered_df[apply_filters][good_counters + ['_benchmark_id', 'darshan_normalized_perf_by_max']]\n",
    "\n",
    "grouped_df = input_df.groupby('_benchmark_id')\n",
    "\n",
    "for fs in col_order:\n",
    "    # generate a single file system's correlation vector\n",
    "    correlation = abcutils.correlation.calc_correlation_vector(\n",
    "        grouped_df.get_group(fs),\n",
    "        correlate_with='darshan_normalized_perf_by_max')\n",
    "            \n",
    "    # rename the columns in this vector to include the file system name\n",
    "    new_cols = {}\n",
    "    for index, col_name in enumerate(correlation.columns):\n",
    "        new_cols[col_name] = \"%s %s\" % (fs, col_name)\n",
    "    correlation.rename(columns=new_cols, inplace=True)\n",
    "    \n",
    "    # join the vector to the previous vectors' dataframe\n",
    "    if correlations is None:\n",
    "        correlations = correlation\n",
    "    else:\n",
    "        correlations = pandas.concat([correlations, correlation], axis='columns')\n",
    "        \n",
    "# Construct filter to show any metric that registered a low p-value for _any_ file system\n",
    "\n",
    "filters = [True] * len(correlations)\n",
    "#for fs in input_df['_benchmark_id'].unique():\n",
    "#   subfilter = correlations['%s p-value' % fs] < 1.0e-5\n",
    "#   if filters is None:\n",
    "#       filters = subfilter\n",
    "#   else:\n",
    "#       filters |= subfilter\n",
    "\n",
    "ax = abcutils.plot.correlation_vector_table(\n",
    "    correlations.loc[good_counters],\n",
    "    fontsize=18,\n",
    "    row_name_map={\n",
    "        'coverage_factor_bw': \"Coverage Factor (Bandwidth)\",\n",
    "        \"coverage_factor_opens\": \"Coverage Factor (opens)\",\n",
    "        \"fs_ave_mds_cpu\": \"Average MDS CPU Load\",\n",
    "        \"fs_ave_oss_cpu\": \"Average OSS CPU Load\",\n",
    "        \"fs_max_mds_cpu\": \"Peak MDS CPU Load\",\n",
    "        \"fs_max_oss_cpu\": \"Peak OSS CPU Load\",\n",
    "        \"fshealth_ost_most_full_pct\": \"OST Fullness\",\n",
    "        \"fshealth_ost_overloaded_oss_count\": \"Number of failed-over OSTs\",\n",
    "        \"topology_job_avg_radius\": \"Average Job Radius\",\n",
    "    },\n",
    "    col_name_map={\n",
    "        'ior_fpp_write coefficient': 'IOR Write',\n",
    "        'hacc_io_write_fpp_write coefficient': \"HACC Write\",\n",
    "        'ior_fpp_read coefficient': \"IOR Read\",\n",
    "        \"hacc_io_read_fpp_read coefficient\": \"HACC Read\",\n",
    "    },\n",
    "    cell_styler=cell_styler)\n",
    "\n",
    "# Set the table width larger if displaying lots of metrics\n",
    "ax.get_figure().set_size_inches(10, 5)\n",
    "ax.get_figure().savefig('correlation_table_fpp_writes.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}