run_download_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "import os\n",
    "import sys\n",
    "import pandas\n",
    "import requests\n",
    "import json\n",
    "import time\n",
    "import re\n",
    "import gzip\n",
    "import shutil\n",
    "import tarfile\n",
    "import stat\n",
    "try:\n",
    "  import wget\n",
    "except:\n",
    "  pass\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_project_df(profID):\n",
    "  '''Wrapper to API at www.ebi.ac.uk/ena'''\n",
    "  import requests\n",
    "  import json\n",
    "  import pandas\n",
    "\n",
    "  url = 'https://www.ebi.ac.uk/ena/portal/api/filereport'\n",
    "  params = {'accession':profID,\n",
    "            'format':'json',\n",
    "            'download':'false',\n",
    "            'result':'read_run'}\n",
    "  headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64)\"+\n",
    "             \" AppleWebKit/537.36 (KHTML, like Gecko)\"+\n",
    "             \" Chrome/70.0.3538.77 Safari/537.36\"}\n",
    "\n",
    "  print(\"Accessing: \", url)\n",
    "  r = requests.get(url,\n",
    "                  params = params,\n",
    "                  headers = headers\n",
    "                  ) \n",
    "  print(\"\\tMaking Request To: \", r.url) \n",
    "  my_json = json.loads(r.text)\n",
    "  df = pandas.json_normalize(json.loads(r.text))\n",
    "  return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/55040442/how-to-register-gz-format-in-shutil-register-archive-format-to-use-same-format\n",
    "\n",
    "def gunzip_something(gzipped_file_name, work_dir):\n",
    "  import os\n",
    "  import re\n",
    "  import gzip\n",
    "  import shutil\n",
    "  \"gunzip the given gzipped file\"\n",
    "\n",
    "  # see warning about filename\n",
    "  filename = os.path.split(gzipped_file_name)[-1]\n",
    "  filename = re.sub(r\"\\.gz$\", \"\", filename, flags=re.IGNORECASE)\n",
    "\n",
    "  with gzip.open(gzipped_file_name, 'rb') as f_in:  # <<========== extraction happens here\n",
    "      with open(os.path.join(work_dir, filename), 'wb') as f_out:\n",
    "          shutil.copyfileobj(f_in, f_out)\n",
    "try:\n",
    "  shutil.register_unpack_format('gz',\n",
    "                              ['.gz', ],\n",
    "                              gunzip_something)\n",
    "except:\n",
    "  pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_selected_runs(df, selected_runs, datapath=None, remove_gz=True):\n",
    "  import time\n",
    "  import shutil\n",
    "  import os\n",
    "  import requests\n",
    "  installed_wget = True\n",
    "  try:\n",
    "    import wget\n",
    "  except ModuleNotFoundError:\n",
    "    installed_wget = False\n",
    "  import subprocess\n",
    "  \n",
    "  single_read = False\n",
    "  for i in range(len(selected_runs)):\n",
    "    subset = df[df['run_accession']==selected_runs[i]]\n",
    "    if ';' in subset['fastq_ftp'].values[0]:\n",
    "        urls = subset['fastq_ftp'].values[0].split(\";\")\n",
    "    else:\n",
    "        urls = [subset['fastq_ftp'].values[0]]\n",
    "        single_read = True\n",
    "    for url in urls:\n",
    "      if not url.startswith(\"https://\"):\n",
    "        url = \"https://\" + url\n",
    "      print('Fetching: {}'.format(url))\n",
    "      headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64)\"+\n",
    "             \" AppleWebKit/537.36 (KHTML, like Gecko)\"+\n",
    "             \" Chrome/70.0.3538.77 Safari/537.36\"}\n",
    "      filename = url.split(\"/\")[-1]\n",
    "      cmd = \"wget {} --no-check-certificate -U Mozilla/5.0\".format(url)\n",
    "      print(\"Running Command: \",cmd)\n",
    "      !{cmd}\n",
    "      #### TODO Find more pythonic way to download data\n",
    "      #cmd = [\"wget\",\"{}\".format(url),\"-U\",\"Mozilla/5.0\"]\n",
    "      #process = subprocess.Popen(cmd, \n",
    "      #                      stdout=subprocess.PIPE, \n",
    "      #                      stderr=subprocess.STDOUT)\n",
    "      #for line in process.stdout:\n",
    "      #  sys.stdout.write(line)\n",
    "      #with open(filename, \"wb\") as f:\n",
    "      #  r = requests.get(url, headers = headers )\n",
    "      #  f.write(r.content)\n",
    "      #if installed_wget: \n",
    "      #  wget.download(url)\n",
    "      #else:\n",
    "      #  subprocess.run([\"wget\", url])\n",
    "        #!wget {url}\n",
    "      time.sleep(2)\n",
    "  \n",
    "  if not os.path.exists(datapath):\n",
    "    os.makedirs(datapath)\n",
    "  for datafile in selected_runs:\n",
    "    if single_read:\n",
    "        mygz= ''.join([datafile,'.fastq.gz'])\n",
    "        myfastq= ''.join([datafile,'.fastq'])\n",
    "        \n",
    "        if os.path.exists(mygz):\n",
    "          print('Unpacking: ' + mygz )\n",
    "          #!gzip -d {mygz} && mv {myfastq} {datapath}\n",
    "          shutil.unpack_archive(mygz, datapath,\"gz\")\n",
    "        if os.path.exists(mygz.replace(\".fastq.gz\",\"_1.fastq.gz\")):\n",
    "          mygz = mygz.replace(\".fastq.gz\",\"_1.fastq.gz\")\n",
    "          print('Unpacking: ' + mygz )\n",
    "          #!gzip -d {mygz} && mv {myfastq} {datapath}\n",
    "          shutil.unpack_archive(mygz, datapath,\"gz\")\n",
    "        if remove_gz:\n",
    "          os.remove(mygz)\n",
    "    else:\n",
    "        mygz= ''.join([datafile,'_1.fastq.gz'])\n",
    "        myfastq= ''.join([datafile,'_1.fastq'])\n",
    "        print('Unpacking: ' + mygz )\n",
    "        if os.path.exists(mygz):\n",
    "          #!gzip -d {mygz} && mv {myfastq} {datapath}\n",
    "          shutil.unpack_archive(mygz, datapath,\"gz\")\n",
    "        if remove_gz:\n",
    "          os.remove(mygz)\n",
    "\n",
    "        mygz= ''.join([datafile,'_2.fastq.gz'])\n",
    "        myfastq= ''.join([datafile,'_2.fastq'])\n",
    "        print('Unpacking: ' + mygz )\n",
    "        if os.path.exists(mygz):\n",
    "      #!gzip -d {mygz} && mv {myfastq} {datapath}\n",
    "          shutil.unpack_archive(mygz, datapath,\"gz\")\n",
    "        if remove_gz:\n",
    "          os.remove(mygz)\n",
    "  return single_read"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to Save Data\n",
    "my_local_path = \"/gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/\"\n",
    "my_local_path = my_local_path + \"data/\"\n",
    "datapath = \"/gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Project to download\n",
    "dataset = 'PRJEB35615'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "overwrite = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Call to ena api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accessing:  https://www.ebi.ac.uk/ena/portal/api/filereport\n",
      "\tMaking Request To:  https://www.ebi.ac.uk/ena/portal/api/filereport?accession=PRJEB35615&format=json&download=false&result=read_run\n"
     ]
    }
   ],
   "source": [
    "df = fetch_project_df(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select Runs to Download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_runs = sorted(df['run_accession'].to_list()) # Get all Runs\n",
    "\n",
    "selected_runs = [ 'ERR3698081'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected Runs:  ['ERR3698081']\n"
     ]
    }
   ],
   "source": [
    "print(\"Selected Runs: \",selected_runs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating Path /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/PRJEB35615/\n"
     ]
    }
   ],
   "source": [
    "mydatapath = ''.join([my_local_path,\"{}/\".format(dataset)])\n",
    "print(\"Generating Path {}\".format(mydatapath))\n",
    "if not os.path.exists(mydatapath):\n",
    "    os.makedirs(mydatapath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start Download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Fetching:  ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_1.fastq.gz\n",
      "Fetching: https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_1.fastq.gz\n",
      "Running Command:  wget https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_1.fastq.gz --no-check-certificate -U Mozilla/5.0\n",
      "--2022-07-01 08:30:29--  https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_1.fastq.gz\n",
      "Resolving ftp.sra.ebi.ac.uk (ftp.sra.ebi.ac.uk)... 193.62.193.138\n",
      "Connecting to ftp.sra.ebi.ac.uk (ftp.sra.ebi.ac.uk)|193.62.193.138|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1577298754 (1.5G) [application/octet-stream]\n",
      "Saving to: ‘ERR3698081_1.fastq.gz’\n",
      "\n",
      "39% [==============>                        ] 615,823,908  175KB/s   in 9m 10s \n",
      "\n",
      "2022-07-01 08:39:40 (1.07 MB/s) - Connection closed at byte 615823908. Retrying.\n",
      "\n",
      "--2022-07-01 08:39:41--  (try: 2)  https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_1.fastq.gz\n",
      "Connecting to ftp.sra.ebi.ac.uk (ftp.sra.ebi.ac.uk)|193.62.193.138|:443... connected.\n",
      "HTTP request sent, awaiting response... 206 Partial Content\n",
      "Length: 1577298754 (1.5G), 961474846 (917M) remaining [application/octet-stream]\n",
      "Saving to: ‘ERR3698081_1.fastq.gz’\n",
      "\n",
      "39% [+++++++++++++++                        ] 627,461,504  181KB/s   in 79s    \n",
      "\n",
      "2022-07-01 08:41:00 (143 KB/s) - Connection closed at byte 627461504. Retrying.\n",
      "\n",
      "--2022-07-01 08:41:02--  (try: 3)  https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_1.fastq.gz\n",
      "Connecting to ftp.sra.ebi.ac.uk (ftp.sra.ebi.ac.uk)|193.62.193.138|:443... connected.\n",
      "HTTP request sent, awaiting response... 206 Partial Content\n",
      "Length: 1577298754 (1.5G), 949837250 (906M) remaining [application/octet-stream]\n",
      "Saving to: ‘ERR3698081_1.fastq.gz’\n",
      "\n",
      "39% [+++++++++++++++                        ] 627,530,904 54.3KB/s   in 54s    \n",
      "\n",
      "2022-07-01 08:41:57 (1.26 KB/s) - Connection closed at byte 627530904. Retrying.\n",
      "\n",
      "--2022-07-01 08:42:00--  (try: 4)  https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_1.fastq.gz\n",
      "Connecting to ftp.sra.ebi.ac.uk (ftp.sra.ebi.ac.uk)|193.62.193.138|:443... connected.\n",
      "HTTP request sent, awaiting response... 206 Partial Content\n",
      "Length: 1577298754 (1.5G), 949767850 (906M) remaining [application/octet-stream]\n",
      "Saving to: ‘ERR3698081_1.fastq.gz’\n",
      "\n",
      "39% [+++++++++++++++                        ] 627,572,544 35.0KB/s   in 3m 24s \n",
      "\n",
      "2022-07-01 08:45:24 (204 B/s) - Connection closed at byte 627572544. Retrying.\n",
      "\n",
      "--2022-07-01 08:45:28--  (try: 5)  https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_1.fastq.gz\n",
      "Connecting to ftp.sra.ebi.ac.uk (ftp.sra.ebi.ac.uk)|193.62.193.138|:443... connected.\n",
      "HTTP request sent, awaiting response... 206 Partial Content\n",
      "Length: 1577298754 (1.5G), 949726210 (906M) remaining [application/octet-stream]\n",
      "Saving to: ‘ERR3698081_1.fastq.gz’\n",
      "\n",
      "100%[++++++++++++++======================>] 1,577,298,754  420KB/s   in 17m 2s \n",
      "\n",
      "2022-07-01 09:02:31 (908 KB/s) - ‘ERR3698081_1.fastq.gz’ saved [1577298754/1577298754]\n",
      "\n",
      "Fetching: https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_2.fastq.gz\n",
      "Running Command:  wget https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_2.fastq.gz --no-check-certificate -U Mozilla/5.0\n",
      "--2022-07-01 09:02:33--  https://ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_2.fastq.gz\n",
      "Resolving ftp.sra.ebi.ac.uk (ftp.sra.ebi.ac.uk)... 193.62.193.138\n",
      "Connecting to ftp.sra.ebi.ac.uk (ftp.sra.ebi.ac.uk)|193.62.193.138|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1924822065 (1.8G) [application/octet-stream]\n",
      "Saving to: ‘ERR3698081_2.fastq.gz’\n",
      "\n",
      "100%[====================================>] 1,924,822,065  699KB/s   in 25m 56s\n",
      "\n",
      "2022-07-01 09:28:30 (1.18 MB/s) - ‘ERR3698081_2.fastq.gz’ saved [1924822065/1924822065]\n",
      "\n",
      "Unpacking: ERR3698081_1.fastq.gz\n",
      "Unpacking: ERR3698081_2.fastq.gz\n",
      "\n",
      "\n",
      "Fetching:  ftp.sra.ebi.ac.uk/vol1/fastq/ERR369/001/ERR3698081/ERR3698081_2.fastq.gz\n",
      "ERR3698081_2.fastq Exists, Will Not Download\n",
      "\tPath: /gpfs/gpfs0/project/ds6011-sp22-wiki-data/ds6011-sp22-dod/data/PRJEB35615/ERR3698081_2.fastq\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing as mp\n",
    "for run in selected_runs:\n",
    "    subset = df[df['run_accession']==run].fastq_ftp.values[0].split(\";\")\n",
    "    reads = [x.split(\"/\")[-1].replace(\".gz\",\"\") for x in subset]\n",
    "    for i in range(len(reads)):\n",
    "        print(\"\\n\")\n",
    "        read = reads[i]\n",
    "        print(\"Fetching: \", subset[i])\n",
    "        dataread = ''.join([mydatapath,read])\n",
    "  \n",
    "        if overwrite == True:\n",
    "            if os.path.exists(dataread):\n",
    "                try: os.remove(dataread)\n",
    "                except: shutil.rmtree(dataread) \n",
    "        \n",
    "        # Prevent download if file exists\n",
    "        if os.path.exists(dataread):\n",
    "            print(\"{} Exists, Will Not Download\".format(read))\n",
    "            print(\"\\tPath: {}\".format(dataread))\n",
    "            continue\n",
    "        else:\n",
    "            singleread = fetch_selected_runs(df, [run], datapath = mydatapath, remove_gz = True)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}