Collect.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tweepy\n",
    "import re\n",
    "import json\n",
    "\n",
    "import sqlite3 as lite\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import datetime, time, os, sys\n",
    "import argparse, configparser\n",
    "Config = configparser.ConfigParser()\n",
    "Config.read('config.cnf')\n",
    "\n",
    "consumer_key = Config.get('twittersfupubresearch', 'consumer_key')\n",
    "consumer_secret = Config.get('twittersfupubresearch', 'consumer_secret')\n",
    "access_token = Config.get('twittersfupubresearch', 'access_token')\n",
    "access_token_secret = Config.get('twittersfupubresearch', 'access_token_secret')\n",
    "\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "# set up access to the Twitter API\n",
    "api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_excel('data/Pundits_Altmetric_Coded_Recoded_Current.xlsx')\n",
    "screen_names = list(df.Author_ID_On_Source.dropna().unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pundits = {}\n",
    "timelines = {}\n",
    "for pundit in screen_names:\n",
    "    pundits[pundit] = api.get_user(screen_name=pundit)\n",
    "    timelines[pundit] = api.user_timeline(screen_name=pundit, count=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stphnmaher\t7.3\n",
      "DougSaunders\t26.4\n",
      "cselley\t84.1\n",
      "stephenfgordon\t12.7\n",
      "terencecorcoran\t18.8\n",
      "Scott_Gilmore\t11.4\n",
      "InklessPW\t42.5\n",
      "Garossino\t38.2\n",
      "acoyne\t116.3\n",
      "EmmMacfarlane\t34.9\n",
      "jengerson\t4.9\n",
      "TerryGlavin\t19.2\n",
      "TabathaSouthey\t7.5\n",
      "JenniferRobson8\t3.0\n",
      "HeatherMallick\t33.0\n",
      "IvisonJ\t3.0\n",
      "BarbaraRKay\t3.4\n",
      "leahmclaren\t0.8\n",
      "anne_kingston\t1.3\n",
      "tabathasouthey\t7.5\n",
      "garymasonglobe\t5.3\n",
      "JohnIbbitson\t5.8\n",
      "SusanDelacourt\t3.4\n",
      "ChantalHbert\t1.4\n"
     ]
    }
   ],
   "source": [
    "for pundit in screen_names:\n",
    "    try: \n",
    "        status = timelines[pundit][-1]\n",
    "        l = len(timelines[pundit])\n",
    "#         print(pundit, \"%.1f\" % float(l/(datetime.datetime.today() - status.created_at).days))\n",
    "        diff = datetime.datetime.now() - timelines[pundit][-1].created_at\n",
    "        print(\"%s\\t%.1f\" % (pundit, float(l/(diff.days + diff.seconds/60/60/24))))\n",
    "    except ZeroDivisionError: \n",
    "        print(pundit, \"100+\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tweet = api.get_status(39383838383838383838383)\n",
    "\n",
    "except tweepy.TweepError as error: \n",
    "    err = error\n",
    "    user = api.get_user('juancommander')\n",
    "    print(error.reason[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "err.args[0][0]['message'] += '; juan'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"[{'code': 8, 'message': 'No data available for specified ID.'}]\""
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jsonr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "evolBioCon = sqlite3.connect(\"data/BMCevolBioSample.db\")\n",
    "bioCon = sqlite3.connect(\"data/BMCbioSample.db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def load_user_errors(con):\n",
    "    return pd.read_sql(\"SELECT DISTINCT old_screen_name, error FROM sample WHERE error IS NOT NULL \", con, index_col='old_screen_name')\n",
    "\n",
    "df = load_user_errors(evolBioCon).append(load_user_errors(bioCon))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "user = api.get_user('juancommander')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_sql(\"SELECT doi, tweet_id, old_screen_name, tweet FROM sample WHERE tweet IS NOT NULL \", litecon, index_col='tweet_id')\n",
    "df = df[~df.tweet.isnull()]\n",
    "df['tweet'] = df.tweet.apply(lambda x: json.loads(x) if x is not None else None)\n",
    "\n",
    "df['created_at'] = df.tweet.apply(lambda x: time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(x['created_at'],'%a %b %d %H:%M:%S +0000 %Y')))\n",
    "df['created_at'] = pd.to_datetime(df.created_at)\n",
    "df['created_at_dayofweek'] = df.tweet.apply(lambda x: x['created_at'][0:3])\n",
    "df['user'] = df.tweet.apply(lambda x: x['user'])\n",
    "df['screen_name'] = df.tweet.apply(lambda x: x['user']['screen_name'])\n",
    "#     df['user_id'] = df.tweet.apply(lambda x: int(x['user']['id_str']))\n",
    "#     df['user_utc_offset'] = df.tweet.apply(lambda x: x['user']['utc_offset'])\n",
    "#     df['user_name'] = df.tweet.apply(lambda x: x['user']['name'])    \n",
    "#     df['user_followers_count'] = df.tweet.apply(lambda x: x['user']['followers_count'])\n",
    "#     df['user_friends_count'] = df.tweet.apply(lambda x: x['user']['friends_count'])\n",
    "#     df['user_description'] = df.tweet.apply(lambda x: re.sub( '\\s+', ' ', x['user']['description']).strip())\n",
    "#     df['user_statuses_count'] = df.tweet.apply(lambda x: x['user']['statuses_count'])\n",
    "df['is_retweet'] = df.tweet.apply(lambda x: 'retweeted_status' in x)\n",
    "df['is_retweet'] = df['is_retweet'].fillna(False)\n",
    "df['retweet_of_status_id_str'] = df.tweet.apply(lambda x: x['retweeted_status']['id_str'] if 'retweeted_status' in x else None)\n",
    "df['retweet_of_screen_name'] = df.tweet.apply(lambda x: x['retweeted_status']['user']['screen_name'] if 'retweeted_status' in x else None)\n",
    "df['is_reply'] = df.tweet.apply(lambda x: x['in_reply_to_status_id'] != None)\n",
    "df['in_reply_to_status_id_str'] = df.tweet.apply(lambda x: x['in_reply_to_status_id_str'])\n",
    "df['in_reply_to_screen_name'] = df.tweet.apply(lambda x: x['in_reply_to_screen_name'])\n",
    "df['text'] = df.tweet.apply(lambda x: re.sub( '\\s+', ' ', x['text']).strip()) # remove commas for CSV simplicity\n",
    "del df['tweet']\n",
    "tweetdetails = df.sort_index()\n",
    "del df\n",
    "\n",
    "df = pd.read_sql(\"SELECT doi, tweet_id, old_screen_name FROM sample WHERE error LIKE '%screen_name%'\", litecon, index_col='old_screen_name')\n",
    "users_df = pd.read_sql(\"SELECT screen_name, user_object FROM users\", litecon, index_col='screen_name')\n",
    "users_df['user'] = users_df.user_object.map(json.loads)\n",
    "del users_df['user_object']\n",
    "\n",
    "df = df.join(users_df, how=\"inner\")\n",
    "df.index.name = 'screen_name'  \n",
    "df = df.reset_index().set_index('tweet_id')\n",
    "\n",
    "tweetdetails = tweetdetails.append(df).sort_index()\n",
    "del df\n",
    "\n",
    "for field in ['id', 'name', 'followers_count', 'friends_count','statuses_count', 'description']:\n",
    "    tweetdetails['user_%s' % field] = tweetdetails.user.map(lambda x: x[field])\n",
    "del tweetdetails['user']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tweetdetails.to_excel('data/lis_tweetdetails.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_user(screen_name):\n",
    "    try:\n",
    "        user = api.get_user(screen_name=screen_name)\n",
    "        return None\n",
    "    except tweepy.TweepError, error: \n",
    "        return error.message[0]['message']\n",
    "df['updated_error'] = df.index.map(get_user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96\n",
      "56\n"
     ]
    }
   ],
   "source": [
    "print len(df)\n",
    "print len(df[df.updated_error.isnull()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "litecon = lite.connect('new_yorker_2.0.db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "user_df = pd.read_sql(\"SELECT user_id, screen_name, user_object, timeline, timeline_error, timeline_modified, user_modified FROM users\", litecon, index_col='user_id')\n",
    "user_df.index = user_df.index.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "s = user_df.sample(1)\n",
    "t = json.loads(s.iloc[0]['timeline'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with litecon:\n",
    "    litecur = litecon.cursor()\n",
    "    litecur.execute('SELECT tweet_id, tweet FROM sample WHERE user_id IS NULL AND tweet IS NOT NULL')\n",
    "    sampled = litecur.fetchall()\n",
    "    \n",
    "    for s in sampled:\n",
    "        tweet_id = s[0]\n",
    "        tweet = json.loads(s[1])\n",
    "        \n",
    "        litecur.execute('UPDATE sample SET user_id = ? WHERE tweet_id = ?', (tweet['user']['id_str'], tweet_id))\n",
    "                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{u'display_url': u'wapo.st/2ahV4EV',\n",
       "  u'expanded_url': u'http://wapo.st/2ahV4EV',\n",
       "  u'indices': [116, 139],\n",
       "  u'url': u'https://t.co/NvGeUWWZti'}]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = api.search('washingtonpost.com AND science filter:links', rpp=5)  # \n",
    "results[0]._json['entities']['urls']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://wapo.st/2ahV4EV\n",
      "\n",
      "https://www.washingtonpost.com/national/health-science/hillary-clintons-blood-clot-most-likely-in-a-leg-experts-say/2012/12/31/d2c853ea-5376-11e2-bf3e-76c0a789346f_story.html\n",
      "\n",
      "https://www.washingtonpost.com/national/health-science/infants-death-is-first-zika-related-fatality-in-texas/2016/08/09/1c8e7974-5e4e-11e6-84c1-6d27287896b5_story.html?postshare=4861470765287003&tid=ss_mail\n",
      "\n",
      "https://www.washingtonpost.com/news/post-politics/wp/2016/08/08/armed-with-junk-science-and-old-photos-critics-question-hillaryshealth/\n",
      "\n",
      "http://wapo.st/2aPGt2f\n",
      "\n",
      "http://wapo.st/2aR1gTs\n",
      "\n",
      "http://fb.me/8Shqi15ev\n",
      "\n",
      "http://fb.me/5jRZYO9Xv\n",
      "\n",
      "http://fb.me/8Shqi15ev\n",
      "\n",
      "http://wpo.st/GMCr1\n",
      "\n",
      "http://wapo.st/2b2XJD9\n",
      "\n",
      "https://www.washingtonpost.com/national/health-science/infants-death-is-first-zika-related-fatality-in-texas/2016/08/09/1c8e7974-5e4e-11e6-84c1-6d27287896b5_story.html?postshare=4861470765287003&tid=ss_mail\n",
      "\n",
      "https://www.washingtonpost.com/national/health-science/infants-death-is-first-zika-related-fatality-in-texas/2016/08/09/1c8e7974-5e4e-11e6-84c1-6d27287896b5_story.html?postshare=4861470765287003&tid=ss_mail\n",
      "\n",
      "https://www.washingtonpost.com/national/health-science/infants-death-is-first-zika-related-fatality-in-texas/2016/08/09/1c8e7974-5e4e-11e6-84c1-6d27287896b5_story.html?postshare=4861470765287003&tid=ss_mail\n",
      "\n",
      "http://wapo.st/29yz5KH\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for r in results: \n",
    "    for u in r._json['entities']['urls']:\n",
    "        print u['expanded_url']\n",
    "    print "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://www.nytimes.com/2016/08/10/science/dog-sperm-fertility.html?hp&action=click&pgtype=Homepage&clickSource=story-heading&module=second-column-region&region=top-news&WT.nav=top-news&_r=0 200\n",
      "\n",
      "http://dlvr.it/M0JkQs 200\n",
      "\n",
      "http://dlvr.it/M0JkNh 200\n",
      "\n",
      "http://www.nytimes.com/2016/08/09/science/climate-change-carbon-bogs-peat.html?_r=0 200\n",
      "\n",
      "http://ow.ly/hLjZ302TJcR 200\n",
      "\n",
      "http://goo.gl/fOCwWi 200\n",
      "\n",
      "http://nyti.ms/2a9yEmR 200\n",
      "\n",
      "http://buff.ly/2aeDbHZ 200\n",
      "\n",
      "http://www.nytimes.com/2016/07/20/science/nasa-global-temperatures-2016.html?login=email&rref=collection/sectioncollection/science 200\n",
      "\n",
      "http://fb.me/4rgIAJH9m 200\n",
      "\n",
      "http://nyti.ms/2avQfb6 200\n",
      "\n",
      "http://nyti.ms/29mlfr8 200\n",
      "\n",
      "http://www.nytimes.com/2016/08/10/science/dog-sperm-fertility.html?hp&action=click&pgtype=Homepage&clickSource=story-heading&module=second-column-region&region=top-news&WT.nav=top-news&_r=0 200\n",
      "\n",
      "http://nyti.ms/2b42oTZ 200\n",
      "\n",
      "http://nyti.ms/2bdOPPC 200\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for r in results: \n",
    "    for u in r._json['entities']['urls']:\n",
    "        unshortened_uri, status = unshortenit.unshorten_only(u['expanded_url'])\n",
    "        print unshortened_uri, status\n",
    "    print "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "query=tweepy.Cursor(api.search,  q='washingtonpost.com AND science filter:links').items(10)\n",
    "tweets= [status._json for status in query]\n",
    "tweets=pd.DataFrame(tweets)\n",
    "users=pd.DataFrame(tweets[\"user\"].to_dict()).T\n",
    "users=users.rename(columns= lambda x: 'user:'+x)\n",
    "tweets=pd.concat([tweets,users],axis=1)\n",
    "return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tweepy.cursor.ItemIterator at 0x1043946d0>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "since_id = '652417479537479680'\n",
    "mentions = api.mentions_timeline(count=200, since_id=since_id, include_rts=0)\n",
    "for i, m in enumerate(mentions):\n",
    "    tweet = api.get_status(id=m.id_str)\n",
    "    \n",
    "    with litecon:\n",
    "        litecur = litecon.cursor()        \n",
    "        litecur.execute('INSERT INTO response_data (user_id_str, tweet_id, time_received, tweet_text, tweet) VALUES (?,?,?,?,?)', (tweet.user.id_str, tweet.id_str, tweet.created_at, tweet.text, json.dumps(tweet._json)))\n",
    "        litecon.commit()\n",
    "\n",
    "    if i == 0: \n",
    "        since_id = m.id_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'SFUPubResearch'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "status.user.screen_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit reached. Sleeping for: 174\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "api.followers_ids(id=2270698615)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'mahmd_essam'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u = api.get_user(2270698615)\n",
    "u.screen_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import botornot\n",
    "\n",
    "twitter_app_auth = {\n",
    "    'consumer_key': consumer_key,\n",
    "    'consumer_secret': consumer_secret,\n",
    "    'access_token': access_token,\n",
    "    'access_token_secret': access_token_secret,\n",
    "    'wait_on_rate_limit': True, \n",
    "    'wait_on_rate_limit_notify': True\n",
    "  }\n",
    "bon = botornot.BotOrNot(**twitter_app_auth)\n",
    "bon.twitter_api.wait_on_rate_limit_notify = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bon.twitter_api.wait_on_rate_limit_notify = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tweepy\n",
    "import re\n",
    "import json\n",
    "\n",
    "import sqlite3 as lite\n",
    "\n",
    "import datetime, time, os, sys\n",
    "import argparse, ConfigParser\n",
    "Config = ConfigParser.ConfigParser()\n",
    "Config.read('config.cnf')\n",
    "\n",
    "litecon = lite.connect('data/twitter.db')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1086"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user = api.get_user('juancommander')\n",
    "user_id = user.id\n",
    "ids = []\n",
    "for page in tweepy.Cursor(api.followers_ids, id=user_id).pages():\n",
    "    ids.extend(page)\n",
    "len(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "Config = configparser.ConfigParser()\n",
    "Config.read('config.cnf')\n",
    "\n",
    "litecon = lite.connect('data/lis.db')\n",
    "\n",
    "with litecon:\n",
    "\n",
    "# set up SQL tables\n",
    "    litecur = litecon.cursor()\n",
    "    # the sample, with two columns for either the Tweet itself, or the error in trying to retrieve it\n",
    "    litecur.execute(\"CREATE TABLE IF NOT EXISTS sample (doi TEXT, old_screen_name TEXT, tweet_id TEXT, tweet TEXT, error TEXT, modified TEXT)\")\n",
    "\n",
    "    litecur.execute(\"CREATE INDEX IF NOT EXISTS sample_old_screen_name ON sample (old_screen_name)\")\n",
    "    litecur.execute(\"CREATE INDEX IF NOT EXISTS sample_tweet_id ON sample (tweet_id)\")\n",
    "    litecur.execute(\"CREATE INDEX IF NOT EXISTS sample_modified ON sample (modified)\")\n",
    "\n",
    "    # the users that were found\n",
    "    litecur.execute(\"CREATE TABLE IF NOT EXISTS users (user_id TEXT, screen_name TEXT, user_object TEXT, timeline TEXT, timeline_error TEXT, timeline_modified TEXT, user_modified TEXT)\")\n",
    "    litecur.execute(\"CREATE UNIQUE INDEX IF NOT EXISTS users_user_id ON users (user_id)\")\n",
    "    litecur.execute(\"CREATE INDEX IF NOT EXISTS users_screen_name ON users (screen_name)\")    \n",
    "\n",
    "    litecur.execute(\"CREATE TABLE IF NOT EXISTS friends (user_id TEXT, friend_id TEXT, modified TEXT)\")\n",
    "    litecur.execute(\"CREATE INDEX IF NOT EXISTS friends_user_id ON friends (user_id)\")\n",
    "    litecur.execute(\"CREATE UNIQUE INDEX IF NOT EXISTS friends_user_friend_id ON friends (user_id, friend_id)\")\n",
    "\n",
    "    litecur.execute(\"CREATE TABLE IF NOT EXISTS followers (user_id TEXT, follower_id TEXT, modified TEXT)\")\n",
    "    litecur.execute(\"CREATE INDEX IF NOT EXISTS followers_user_id ON followers (user_id)\")\n",
    "    litecur.execute(\"CREATE UNIQUE INDEX IF NOT EXISTS followers_user_follower_id ON followers (user_id, follower_id)\")\n",
    "\n",
    "\n",
    "consumer_key = Config.get('twittersfupubresearch', 'consumer_key')\n",
    "consumer_secret = Config.get('twittersfupubresearch', 'consumer_secret')\n",
    "access_token = Config.get('twittersfupubresearch', 'access_token')\n",
    "access_token_secret = Config.get('twittersfupubresearch', 'access_token_secret')\n",
    "\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "# set up access to the Twitter API\n",
    "api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)\n",
    "\n",
    "def load_file(filename):\n",
    "    with open(filename, 'r') as f:\n",
    "        l = f.readline()\n",
    "\n",
    "        with litecon:\n",
    "            litecur = litecon.cursor()\n",
    "            for l in f:\n",
    "                l = [x.strip('\"') for x in l.strip().split('\\t')]\n",
    "                doi = l[0]\n",
    "                screenname = l[1]\n",
    "                tweet_id = l[2]\n",
    "\n",
    "                try:\n",
    "                    litecur.execute('INSERT INTO sample (doi, old_screen_name, tweet_id) VALUES (?, ?, ?)', (doi, screenname, tweet_id))\n",
    "                except lite.IntegrityError:\n",
    "                    # don't worry about duplicates\n",
    "                    pass\n",
    "\n",
    "def __save_tweet(tweet_id, tweet, error = None):\n",
    "    '''\n",
    "    Do the actual SQLite update with the info collected\n",
    "    '''\n",
    "    now = datetime.datetime.today().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    with litecon:\n",
    "        litecur = litecon.cursor()\n",
    "\n",
    "        if error: \n",
    "            try: \n",
    "                m = error[0][0]['message']\n",
    "            except:\n",
    "                m = str(error)\n",
    "            litecur.execute('UPDATE sample SET error = ?, modified = ? WHERE tweet_id = ?', (m, now, tweet_id))    \n",
    "\n",
    "        else: \n",
    "            litecur.execute('UPDATE sample SET tweet = ?, modified = ? WHERE tweet_id = ?', (json.dumps(tweet._json), now, tweet_id))\n",
    "            try:\n",
    "                litecur.execute('INSERT INTO users (user_id, screen_name, user_object, user_modified) VALUES (?, ?, ?, ?)', (tweet.user.id, tweet.user.screen_name, json.dumps(tweet.user._json), now))\n",
    "            except lite.IntegrityError:\n",
    "                # don't worry about duplicates\n",
    "                pass\n",
    "\n",
    "def __save_timeline(user_id, timeline, error = None):\n",
    "    '''\n",
    "    Do the actual SQLite update with the info collected\n",
    "    '''\n",
    "    now = datetime.datetime.today().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    with litecon:\n",
    "        litecur = litecon.cursor()\n",
    "\n",
    "        if error: \n",
    "            try: \n",
    "                m = error[0][0]['message']\n",
    "            except:\n",
    "                m = str(error)\n",
    "            litecur.execute('UPDATE users SET timeline_error = ?, timeline_modified = ? WHERE user_id = ?', (m, now, user_id))    \n",
    "\n",
    "        else: \n",
    "            litecur.execute('UPDATE users SET timeline = ?, timeline_modified = ? WHERE user_id = ?', (json.dumps([s._json for s in timeline]), now, user_id))\n",
    "\n",
    "\n",
    "def get_tweets_in_sample():\n",
    "    '''\n",
    "    Find all the tweets in the sample that have not been fetched yet\n",
    "    and make individual calls to find the users associated with them\n",
    "    '''\n",
    "    with litecon:\n",
    "        litecur = litecon.cursor()\n",
    "        litecur.execute(\"SELECT tweet_id, old_screen_name FROM sample WHERE tweet IS NULL\")\n",
    "        sampled = litecur.fetchall()\n",
    "\n",
    "        for s in sampled:\n",
    "            tweet_id = s[0]\n",
    "            try:\n",
    "                tweet = api.get_status(tweet_id)\n",
    "                __save_tweet(tweet_id, tweet)\n",
    "\n",
    "            except tweepy.TweepError as error: \n",
    "                # bit hacky, we're passing a tweet_id instead of a tweet here\n",
    "                try: \n",
    "                    user = api.get_user(screen_name=s[1])\n",
    "                    try:\n",
    "                        error[0][0]['message'] += '; Found by screen_name'\n",
    "                        __save_tweet(tweet_id, None, error)\n",
    "                        now = datetime.datetime.today().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "                        litecur.execute('INSERT INTO users (user_id, screen_name, user_object, user_modified) VALUES (?, ?, ?, ?)', (user.id, user.screen_name, json.dumps(user._json), now))\n",
    "                    except lite.IntegrityError:\n",
    "                        # don't worry about duplicates\n",
    "                        pass\n",
    "\n",
    "                except tweepy.TweepError as error2: \n",
    "                    __save_tweet(tweet_id, None, error) # leave the original error\n",
    "\n",
    "        # now try all the errors one more time\n",
    "        litecur.execute('SELECT tweet_id FROM sample WHERE tweet IS NULL AND error IS NOT NULL')\n",
    "        sampled = litecur.fetchall()\n",
    "\n",
    "def get_tweets_in_sample_batch():\n",
    "    '''\n",
    "    Find all the tweets in the sample that have not been fetched yet\n",
    "    and make a batch call to find the users associated with them\n",
    "    '''\n",
    "    with litecon:\n",
    "        litecur = litecon.cursor()\n",
    "        litecur.execute('SELECT tweet_id FROM sample WHERE tweet IS NULL AND error IS NULL')\n",
    "\n",
    "        i = 0\n",
    "        while (True):\n",
    "            tweet_ids = [t[0] for t in litecur.fetchmany(100)]\n",
    "            if not tweet_ids: break\n",
    "\n",
    "            try:\n",
    "                statuses = api.statuses_lookup(tweet_ids)\n",
    "                for tweet in statuses:\n",
    "                    __save_tweet(tweet.id, tweet)\n",
    "                    i+=1\n",
    "\n",
    "            except tweepy.TweepError as error: \n",
    "                print( error)\n",
    "                exit(1)\n",
    "            print(\"Done %s\" % i)\n",
    "            \n",
    "            \n",
    "\n",
    "def get_timelines_batch():\n",
    "    '''\n",
    "    Get all the timeline JSON objects for users we know exist (we've saved)\n",
    "    '''\n",
    "    while (True):\n",
    "        with litecon:\n",
    "            litecur = litecon.cursor()\n",
    "            litecur.execute('SELECT u.user_id FROM users u WHERE timeline IS NULL AND timeline_error IS NULL')\n",
    "\n",
    "            # go 100 at a time so we're not hitting the DB so much\n",
    "            user_ids = [u[0] for u in litecur.fetchmany(100)]\n",
    "            if not user_ids: break\n",
    "\n",
    "            for user_id in user_ids: \n",
    "                try:\n",
    "                    timeline = api.user_timeline(user_id)\n",
    "                    __save_timeline(user_id, timeline)\n",
    "\n",
    "                except tweepy.TweepError as error: \n",
    "                    __save_timeline(user_id, None, error)\n",
    "\n",
    "def __save_network(endpoint, user_id, ids, error = None):\n",
    "    '''\n",
    "    Do the actual SQLite update with the info collected\n",
    "    '''\n",
    "    now = datetime.datetime.today().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "    with litecon:\n",
    "        litecur = litecon.cursor()\n",
    "        \n",
    "        if error: \n",
    "            try: \n",
    "                m = error[0][0]['message']\n",
    "            except:\n",
    "                m = str(error)\n",
    "                \n",
    "            print( \"Error: \", user_id, m)\n",
    "            litecur.execute('INSERT INTO %s (user_id, %s_id, modified) VALUES (?, ?, ?)' % (endpoint, endpoint[:-1]), (user_id, -1, now))\n",
    "        else:\n",
    "            print( 'saving', len(ids))\n",
    "            for f in ids:\n",
    "                try: \n",
    "                    litecur.execute('INSERT INTO %s (user_id, %s_id, modified) VALUES (?, ?, ?)' % (endpoint, endpoint[:-1]), (user_id, f, now))\n",
    "                except lite.IntegrityError:\n",
    "                    pass # ignore duplicates, they wont change the network\n",
    "                    \n",
    "def get_friends(user_id = None):\n",
    "    get_network('friends', user_id)\n",
    "\n",
    "def get_followers(user_id = None):\n",
    "    get_network('followers', user_id)\n",
    "            \n",
    "def get_network(endpoint, user_id = None):\n",
    "    '''\n",
    "    Get the friends/followers list for all users (or for a specific user\n",
    "    '''\n",
    "    if user_id is None:       \n",
    "        while (True):\n",
    "            with litecon:\n",
    "                litecur = litecon.cursor()\n",
    "                litecur.execute('SELECT u.user_id FROM users u LEFT JOIN %s f ON (u.user_id = f.user_id) WHERE f.user_id IS NULL' % endpoint)\n",
    "        \n",
    "            # go 100 at a time so we're not hitting the DB so much\n",
    "            users = [u[0] for u in litecur.fetchmany(100)]\n",
    "            if not users: break\n",
    "\n",
    "            for user_id in users:\n",
    "                ids = get_network(endpoint, user_id)\n",
    "    else: \n",
    "        try:\n",
    "            ids = []\n",
    "            # a user_id was passed in, fetch it and return a friends list\n",
    "            if endpoint == 'friends':\n",
    "                for page in tweepy.Cursor(api.friends_ids, id=user_id).pages():\n",
    "                    ids.extend(page)\n",
    "            elif endpoint == 'followers':\n",
    "                for page in tweepy.Cursor(api.followers_ids, id=user_id).pages():\n",
    "                    ids.extend(page)\n",
    "\n",
    "            # put in something so that we know we've gone after this user\n",
    "            if len(ids) == 0:\n",
    "                __save_network(endpoint, user_id, None, 'No %s found' % endpoint)\n",
    "\n",
    "            __save_network(endpoint, user_id, ids)\n",
    "\n",
    "        except tweepy.TweepError as error: \n",
    "            __save_network(endpoint, user_id, None, error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "err = None\n",
    "tweet_id = '600296933484326913';\n",
    "try:\n",
    "    tweet = api.get_status(tweet_id)\n",
    "except tweepy.TweepError as error: \n",
    "    err = error\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'No status found with that ID.'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "err.args[0][0]['message']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n"
     ]
    }
   ],
   "source": [
    "screen_names = ['sage_time', 'RamonDavisMark', 'HarrySpoelstra', 'NatRevNeurol', 'mzkhalil', 'Bill_Bl4ck', 'ewydh', 'annabelgillfi', 'viogibsix', 'SaraDivinorum', 'Migraine_Wisdom', 'TheLancet']\n",
    "print len(screen_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "user_ids = []\n",
    "for screen_name in screen_names:\n",
    "    try:\n",
    "        user = api.get_user(screen_name)\n",
    "        user_ids.append(user.id)\n",
    "    except tweepy.TweepError, error:\n",
    "        print screen_name, error\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving 11\n",
      "saving 0\n",
      "saving 116\n",
      "saving 458\n",
      "saving 560\n",
      "saving 927\n",
      "saving 9687\n",
      "saving 122\n",
      "saving 887\n",
      "saving 400\n",
      "saving 111\n",
      "saving 223\n",
      "saving 476\n",
      "saving 179\n",
      "saving 241\n",
      "saving 1095\n",
      "saving 181\n",
      "saving 545\n",
      "saving 140\n",
      "saving 283\n"
     ]
    }
   ],
   "source": [
    "for user_id in user_ids:\n",
    "    get_followers(user_id)\n",
    "    get_friends(user_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4232"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0+ 458+ 927+ 122+ 400+ 223+ 179+ 1095+ 545+ 283"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with litecon:\n",
    "    litecur = litecon.cursor()\n",
    "    litecur.execute('SELECT user_id, screen_name FROM users WHERE botornot IS NULL LIMIT 2')\n",
    "\n",
    "    fetchedmany = litecur.fetchmany(100)\n",
    "# go 100 at a time so we're not hitting the DB so much\n",
    "#     users = [(u[0], u[1]) for u in ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "def tweetHarvester(thisString,hits):\n",
    "    query=tweepy.Cursor(api.search,  q=thisString+' filter:links').items(hits)\n",
    "    tweets= [status._json for status in query]\n",
    "    tweets=pd.DataFrame(tweets)\n",
    "    entities=pd.DataFrame(tweets[\"entities\"].to_dict()).T\n",
    "    entities=entities.rename(columns= lambda x: 'entities:'+x)\n",
    "    tweets=pd.concat([tweets,entities],axis=1)\n",
    "    metadata=pd.DataFrame(tweets[\"metadata\"].to_dict()).T\n",
    "    metadata=metadata.rename(columns= lambda x: 'metadata:'+x)\n",
    "    tweets=pd.concat([tweets,metadata],axis=1)\n",
    "    users=pd.DataFrame(tweets[\"user\"].to_dict()).T\n",
    "    users=users.rename(columns= lambda x: 'user:'+x)\n",
    "    tweets=pd.concat([tweets,users],axis=1)\n",
    "    tweets=tweets.drop(['entities','metadata','user'],axis=1)\n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = tweetHarvester('newyorker.com', 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "contributors\n",
      "coordinates\n",
      "created_at\n",
      "favorite_count\n",
      "favorited\n",
      "geo\n",
      "id\n",
      "id_str\n",
      "in_reply_to_screen_name\n",
      "in_reply_to_status_id\n",
      "in_reply_to_status_id_str\n",
      "in_reply_to_user_id\n",
      "in_reply_to_user_id_str\n",
      "is_quote_status\n",
      "lang\n",
      "place\n",
      "possibly_sensitive\n",
      "retweet_count\n",
      "retweeted\n",
      "retweeted_status\n",
      "source\n",
      "text\n",
      "truncated\n",
      "entities:hashtags\n",
      "entities:symbols\n",
      "entities:urls\n",
      "entities:user_mentions\n",
      "metadata:iso_language_code\n",
      "metadata:result_type\n",
      "user:contributors_enabled\n",
      "user:created_at\n",
      "user:default_profile\n",
      "user:default_profile_image\n",
      "user:description\n",
      "user:entities\n",
      "user:favourites_count\n",
      "user:follow_request_sent\n",
      "user:followers_count\n",
      "user:following\n",
      "user:friends_count\n",
      "user:geo_enabled\n",
      "user:has_extended_profile\n",
      "user:id\n",
      "user:id_str\n",
      "user:is_translation_enabled\n",
      "user:is_translator\n",
      "user:lang\n",
      "user:listed_count\n",
      "user:location\n",
      "user:name\n",
      "user:notifications\n",
      "user:profile_background_color\n",
      "user:profile_background_image_url\n",
      "user:profile_background_image_url_https\n",
      "user:profile_background_tile\n",
      "user:profile_banner_url\n",
      "user:profile_image_url\n",
      "user:profile_image_url_https\n",
      "user:profile_link_color\n",
      "user:profile_sidebar_border_color\n",
      "user:profile_sidebar_fill_color\n",
      "user:profile_text_color\n",
      "user:profile_use_background_image\n",
      "user:protected\n",
      "user:screen_name\n",
      "user:statuses_count\n",
      "user:time_zone\n",
      "user:url\n",
      "user:utc_offset\n",
      "user:verified\n"
     ]
    }
   ],
   "source": [
    "for c in df.columns:\n",
    "    print c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    api.update_status('dup')\n",
    "except tweepy.TweepError, error: \n",
    "    err = error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "screen_names = ['AnandWilson91', 'Firefly_fan', 'OnCritical', 'P1NDSTER', 'Paul_in_Aber', 'Whole9SoPacific', 'albertinquiet', 'johnpane', 'markushinka', 'nacho_zizou', 'satbhambra', 'seitics', 'bemyprimate', 'ACountyGurl', 'All4thelight', 'peter_makin', 'wt3']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AnandWilson91 User not found.\n",
      "Firefly_fan User not found.\n",
      "OnCritical User not found.\n",
      "P1NDSTER User not found.\n",
      "Paul_in_Aber User not found.\n",
      "Whole9SoPacific User not found.\n",
      "albertinquiet User not found.\n",
      "johnpane User not found.\n",
      "markushinka User not found.\n",
      "nacho_zizou User not found.\n",
      "satbhambra User not found.\n",
      "seitics User not found.\n",
      "bemyprimate User not found.\n",
      "ACountyGurl User not found.\n",
      "All4thelight User not found.\n",
      "peter_makin User not found.\n",
      "wt3 User not found.\n"
     ]
    }
   ],
   "source": [
    "responses = {}\n",
    "for screen_name in screen_names:\n",
    "    try:\n",
    "        responses[screen_name] = api.get_user(screen_name=screen_names)\n",
    "    except tweepy.TweepError, error: \n",
    "        print screen_name, error.message[0]['message']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ACountyGurl': u'User not found.',\n",
       " 'All4thelight': u'User not found.',\n",
       " 'AnandWilson91': u'User not found.',\n",
       " 'Firefly_fan': u'User not found.',\n",
       " 'OnCritical': u'User not found.',\n",
       " 'P1NDSTER': u'User not found.',\n",
       " 'Paul_in_Aber': u'User not found.',\n",
       " 'Whole9SoPacific': u'User not found.',\n",
       " 'albertinquiet': u'User not found.',\n",
       " 'bemyprimate': u'User not found.',\n",
       " 'johnpane': u'User not found.',\n",
       " 'markushinka': u'User not found.',\n",
       " 'nacho_zizou': u'User not found.',\n",
       " 'peter_makin': u'User not found.',\n",
       " 'satbhambra': u'User not found.',\n",
       " 'seitics': u'User not found.',\n",
       " 'wt3': u'User not found.'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}