set-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "542c9d79",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pylab as plt\n",
    "authors = pd.read_csv(r'C:\\Users\\ketli\\Documents\\Summer 2021\\ORNL\\Repo\\reposcanner-data\\authors.csv')\n",
    "repolist = pd.read_csv(r'C:\\Users\\ketli\\Documents\\Summer 2021\\ORNL\\Repo\\reposcanner-data\\repolist.csv')\n",
    "rid_sum = authors.groupby(by='RID')['commits'].sum()\n",
    "\n",
    "# total number of lines 'LIN'\n",
    "authors['LIN'] = authors['adds'] + authors['dels']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e82fe074",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set_dev1k = The set of developers who contributed at least 1,000 lines of code in total.\n",
    "def core_dev_1k():\n",
    "    #Grouped by FID as index, .sum() all values for FID\n",
    "    fid_info = authors.groupby('FID', sort=True, as_index=True).sum()\n",
    "    set_dev1k = set(fid_info[fid_info['LIN'] >= 1000].index)\n",
    "    return set_dev1k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c7053415",
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_contributors(rid):\n",
    "    repo = authors.loc[authors['RID'] == rid]\n",
    "    sort_lines = repo.sort_values(by='LIN', ascending=False)\n",
    "    core = core_dev_repo(sort_lines)\n",
    "    return core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5cdbbf4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def core_dev_repo(sort_lines, pct = 0.8):\n",
    "    total = 0\n",
    "    ele = 0\n",
    "    cutoff = sort_lines['LIN'].sum() * pct\n",
    "   \n",
    "    # Iterate each element in list\n",
    "    # and add them in variable total\n",
    "    while(ele < len(sort_lines)):\n",
    "        total = total + sort_lines.iloc[ele]['LIN']\n",
    "        if total >= cutoff:\n",
    "            break\n",
    "        ele += 1\n",
    "    return sort_lines.iloc[:ele + 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c49f2181",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set_coredev = The set of developers who were \"core\" contributors to at least 1 repository.\n",
    "def core_dev():\n",
    "    all_core = []\n",
    "    set_coredev = set()\n",
    "\n",
    "    for rid in rid_sum.index:\n",
    "        core = top_contributors(rid)   \n",
    "        fid = core['FID'].values.tolist()\n",
    "        all_core.append(fid)\n",
    "    \n",
    "    for i in all_core:\n",
    "        for j in i:\n",
    "            set_coredev.add(j)\n",
    "    return set_coredev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "15ca5311",
   "metadata": {},
   "outputs": [],
   "source": [
    "repo_project = pd.merge(authors, repolist[['RID','PID']], on='RID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3e63f4c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set_crosscontrib = The set of developers who made commits to repositories belonging to more than one ECP project.\n",
    "def crosscontrib():\n",
    "    pid_group = repo_project.groupby('PID')['FID'].apply(list)\n",
    "    dictionary = {}\n",
    "    for i in pid_group:\n",
    "        unique_fid = list(dict.fromkeys(i))\n",
    "        for j in unique_fid:\n",
    "            dictionary[j] = dictionary.setdefault(j, 0) + 1        \n",
    "    set_crosscontrib = set(key for key, value in dictionary.items() if value >= 2)\n",
    "    return set_crosscontrib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "feb31bbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross contributors:  {257, 136, 9, 907, 5136, 8337, 784, 919, 3352, 1561, 28, 8350, 928, 933, 42, 811, 174, 7345, 1210, 957, 318, 64, 65, 834, 9286, 2374, 10056, 9163, 9547, 461, 4815, 592, 1105, 82, 4435, 340, 981, 342, 719, 601, 3934, 3809, 867, 2147, 3428, 487, 104, 3690, 2026, 1005, 623, 113, 499, 1012, 374, 7415, 9464, 6012, 10110}\n"
     ]
    }
   ],
   "source": [
    "print('Cross contributors: ', crosscontrib())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6faa79f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Core devs. {513, 6146, 2054, 9, 10763, 15, 9237, 22, 25, 1561, 28, 1054, 8227, 35, 1063, 9255, 8231, 42, 1070, 8753, 1074, 9781, 54, 2103, 569, 571, 5183, 65, 9286, 584, 586, 76, 5710, 593, 82, 7252, 11352, 601, 605, 609, 101, 104, 105, 8298, 8808, 621, 623, 111, 8824, 5759, 133, 136, 138, 8844, 653, 141, 6800, 9878, 160, 675, 168, 169, 685, 7345, 5298, 3761, 183, 1209, 200, 4815, 4816, 213, 9942, 9943, 5846, 6871, 218, 3809, 228, 755, 2820, 4871, 784, 6934, 283, 284, 795, 796, 799, 811, 815, 818, 6962, 12597, 4407, 830, 318, 320, 832, 322, 6469, 328, 10056, 9547, 5964, 845, 6992, 849, 338, 10579, 340, 342, 856, 3418, 859, 6495, 355, 8556, 374, 6012, 6014, 899, 903, 908, 8077, 400, 6551, 4507, 8607, 928, 929, 933, 434, 8628, 6068, 438, 951, 956, 957, 10172, 1983, 11201, 6082, 1990, 969, 461, 973, 467, 981, 2518, 986, 9182, 478, 2022, 487, 1005, 8686, 7663, 5614, 9202, 499, 1012, 1014, 1015, 1017, 1021, 1022}\n"
     ]
    }
   ],
   "source": [
    "print('Core devs.', core_dev())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "31e9386e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Devs 1k lines:  {513, 2, 1025, 2562, 4612, 2054, 6146, 9, 10763, 8206, 15, 10257, 3092, 22, 1559, 25, 1561, 28, 29, 1054, 9759, 5664, 6178, 35, 8227, 9252, 1063, 552, 3113, 42, 8231, 9255, 1070, 8753, 1074, 9781, 54, 10805, 10807, 57, 569, 571, 62, 5183, 576, 65, 5184, 9286, 3655, 584, 586, 1099, 76, 3148, 8269, 1103, 592, 593, 82, 10830, 7252, 10832, 10833, 10834, 11352, 601, 605, 4701, 1120, 609, 2147, 101, 104, 105, 6760, 8298, 8808, 621, 111, 623, 1651, 6772, 4215, 8824, 124, 1149, 3710, 5245, 5759, 133, 5767, 136, 9353, 138, 8844, 141, 653, 9870, 6800, 8337, 9878, 6811, 8350, 160, 168, 169, 685, 174, 3761, 5298, 7345, 9397, 5302, 183, 1209, 200, 719, 4815, 4816, 10450, 213, 5846, 6871, 9942, 9943, 218, 6366, 3809, 228, 1261, 2800, 1777, 755, 1779, 5365, 9464, 8450, 2820, 4871, 5389, 784, 6934, 3352, 3354, 283, 284, 795, 796, 6940, 1316, 7461, 12580, 12581, 12583, 12584, 811, 818, 6962, 8499, 12597, 4407, 5431, 5947, 318, 830, 320, 832, 322, 8514, 6469, 3910, 328, 4424, 10056, 9547, 5964, 845, 6992, 849, 4435, 340, 341, 342, 10579, 856, 857, 3418, 859, 6495, 867, 3428, 2920, 3435, 8556, 12794, 4468, 7541, 374, 6012, 10110, 899, 908, 8077, 1422, 10639, 400, 12691, 6037, 919, 6551, 1433, 10138, 4507, 7582, 415, 928, 417, 929, 933, 936, 427, 428, 429, 5551, 3504, 6575, 434, 2994, 6068, 8628, 438, 950, 951, 956, 957, 10172, 1983, 11201, 6082, 452, 453, 1990, 10180, 969, 9163, 461, 973, 467, 981, 984, 986, 3546, 478, 9182, 2022, 487, 2026, 492, 1004, 1005, 5614, 7663, 8686, 9202, 499, 1012, 2037, 1014, 1015, 9209, 10746, 1021, 1022}\n"
     ]
    }
   ],
   "source": [
    "print('Devs 1k lines: ',  core_dev_1k())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}