bikroy_splash_request_V2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "if (!(\"Notification\" in window)) {\n",
       "    alert(\"This browser does not support desktop notifications, so the %%notify magic will not work.\");\n",
       "} else if (Notification.permission !== 'granted' && Notification.permission !== 'denied') {\n",
       "    Notification.requestPermission(function (permission) {\n",
       "        if(!('permission' in Notification)) {\n",
       "            Notification.permission = permission;\n",
       "        }\n",
       "    })\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import requests\n",
    "import time \n",
    "import json\n",
    "from requests_html import HTMLSession\n",
    "from PIL import Image\n",
    "from io import StringIO, BytesIO\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from IPython.display import display, clear_output\n",
    "\n",
    "%load_ext jupyternotify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: selenium in c:\\users\\kalim\\anaconda3\\lib\\site-packages (4.1.2)\n",
      "Requirement already satisfied: urllib3[secure,socks]~=1.26 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from selenium) (1.26.7)\n",
      "Requirement already satisfied: trio-websocket~=0.9 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from selenium) (0.9.2)\n",
      "Requirement already satisfied: trio~=0.17 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from selenium) (0.20.0)\n",
      "Requirement already satisfied: outcome in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.1.0)\n",
      "Requirement already satisfied: sniffio in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.2.0)\n",
      "Requirement already satisfied: async-generator>=1.9 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.10)\n",
      "Requirement already satisfied: idna in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (3.2)\n",
      "Requirement already satisfied: cffi>=1.14 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.14.6)\n",
      "Requirement already satisfied: attrs>=19.2.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (21.2.0)\n",
      "Requirement already satisfied: sortedcontainers in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (2.4.0)\n",
      "Requirement already satisfied: pycparser in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from cffi>=1.14->trio~=0.17->selenium) (2.20)\n",
      "Requirement already satisfied: wsproto>=0.14 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from trio-websocket~=0.9->selenium) (1.1.0)\n",
      "Requirement already satisfied: PySocks!=1.5.7,<2.0,>=1.5.6 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (1.7.1)\n",
      "Requirement already satisfied: pyOpenSSL>=0.14 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (21.0.0)\n",
      "Requirement already satisfied: cryptography>=1.3.4 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (3.4.8)\n",
      "Requirement already satisfied: certifi in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (2021.10.8)\n",
      "Requirement already satisfied: six>=1.5.2 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from pyOpenSSL>=0.14->urllib3[secure,socks]~=1.26->selenium) (1.16.0)\n",
      "Requirement already satisfied: h11<1,>=0.9.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.13.0)\n"
     ]
    }
   ],
   "source": [
    "! pip install --upgrade selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: jupyternotify in c:\\users\\kalim\\anaconda3\\lib\\site-packages (0.1.15)\n",
      "Requirement already satisfied: ipython in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyternotify) (7.29.0)\n",
      "Requirement already satisfied: jupyter in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyternotify) (1.0.0)\n",
      "Requirement already satisfied: decorator in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (5.1.0)\n",
      "Requirement already satisfied: setuptools>=18.5 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (58.0.4)\n",
      "Requirement already satisfied: matplotlib-inline in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (0.1.2)\n",
      "Requirement already satisfied: pickleshare in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (0.7.5)\n",
      "Requirement already satisfied: backcall in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (0.2.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (3.0.20)\n",
      "Requirement already satisfied: jedi>=0.16 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (0.18.0)\n",
      "Requirement already satisfied: pygments in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (2.10.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (0.4.4)\n",
      "Requirement already satisfied: traitlets>=4.2 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipython->jupyternotify) (5.1.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jedi>=0.16->ipython->jupyternotify) (0.8.2)\n",
      "Requirement already satisfied: wcwidth in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->jupyternotify) (0.2.5)\n",
      "Requirement already satisfied: ipywidgets in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter->jupyternotify) (7.6.5)\n",
      "Requirement already satisfied: qtconsole in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter->jupyternotify) (5.1.1)\n",
      "Requirement already satisfied: notebook in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter->jupyternotify) (6.4.5)\n",
      "Requirement already satisfied: nbconvert in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter->jupyternotify) (6.1.0)\n",
      "Requirement already satisfied: jupyter-console in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter->jupyternotify) (6.4.0)\n",
      "Requirement already satisfied: ipykernel in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter->jupyternotify) (6.4.1)\n",
      "Requirement already satisfied: tornado<7.0,>=4.2 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipykernel->jupyter->jupyternotify) (6.1)\n",
      "Requirement already satisfied: debugpy<2.0,>=1.0.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipykernel->jupyter->jupyternotify) (1.4.1)\n",
      "Requirement already satisfied: ipython-genutils in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipykernel->jupyter->jupyternotify) (0.2.0)\n",
      "Requirement already satisfied: jupyter-client<8.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipykernel->jupyter->jupyternotify) (6.1.12)\n",
      "Requirement already satisfied: pyzmq>=13 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter-client<8.0->ipykernel->jupyter->jupyternotify) (22.2.1)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter-client<8.0->ipykernel->jupyter->jupyternotify) (2.8.2)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter-client<8.0->ipykernel->jupyter->jupyternotify) (4.8.1)\n",
      "Requirement already satisfied: pywin32>=1.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jupyter-core>=4.6.0->jupyter-client<8.0->ipykernel->jupyter->jupyternotify) (227)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from python-dateutil>=2.1->jupyter-client<8.0->ipykernel->jupyter->jupyternotify) (1.16.0)\n",
      "Requirement already satisfied: widgetsnbextension~=3.5.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipywidgets->jupyter->jupyternotify) (3.5.1)\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipywidgets->jupyter->jupyternotify) (1.0.0)\n",
      "Requirement already satisfied: nbformat>=4.2.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from ipywidgets->jupyter->jupyternotify) (5.1.3)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbformat>=4.2.0->ipywidgets->jupyter->jupyternotify) (3.2.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter->jupyternotify) (0.18.0)\n",
      "Requirement already satisfied: attrs>=17.4.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter->jupyternotify) (21.2.0)\n",
      "Requirement already satisfied: argon2-cffi in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from notebook->jupyter->jupyternotify) (20.1.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from notebook->jupyter->jupyternotify) (0.9.4)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from notebook->jupyter->jupyternotify) (2.11.3)\n",
      "Requirement already satisfied: Send2Trash>=1.5.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from notebook->jupyter->jupyternotify) (1.8.0)\n",
      "Requirement already satisfied: prometheus-client in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from notebook->jupyter->jupyternotify) (0.11.0)\n",
      "Requirement already satisfied: pywinpty>=0.5 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from terminado>=0.8.3->notebook->jupyter->jupyternotify) (0.5.7)\n",
      "Requirement already satisfied: cffi>=1.0.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from argon2-cffi->notebook->jupyter->jupyternotify) (1.14.6)\n",
      "Requirement already satisfied: pycparser in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from cffi>=1.0.0->argon2-cffi->notebook->jupyter->jupyternotify) (2.20)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from jinja2->notebook->jupyter->jupyternotify) (1.1.1)\n",
      "Requirement already satisfied: testpath in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbconvert->jupyter->jupyternotify) (0.5.0)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbconvert->jupyter->jupyternotify) (1.4.3)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbconvert->jupyter->jupyternotify) (0.5.3)\n",
      "Requirement already satisfied: defusedxml in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbconvert->jupyter->jupyternotify) (0.7.1)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbconvert->jupyter->jupyternotify) (0.3)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbconvert->jupyter->jupyternotify) (0.8.4)\n",
      "Requirement already satisfied: jupyterlab-pygments in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbconvert->jupyter->jupyternotify) (0.1.2)\n",
      "Requirement already satisfied: bleach in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbconvert->jupyter->jupyternotify) (4.0.0)\n",
      "Requirement already satisfied: async-generator in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert->jupyter->jupyternotify) (1.10)\n",
      "Requirement already satisfied: nest-asyncio in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert->jupyter->jupyternotify) (1.5.1)\n",
      "Requirement already satisfied: webencodings in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from bleach->nbconvert->jupyter->jupyternotify) (0.5.1)\n",
      "Requirement already satisfied: packaging in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from bleach->nbconvert->jupyter->jupyternotify) (21.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from packaging->bleach->nbconvert->jupyter->jupyternotify) (3.0.4)\n",
      "Requirement already satisfied: qtpy in c:\\users\\kalim\\anaconda3\\lib\\site-packages (from qtconsole->jupyter->jupyternotify) (1.10.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install jupyternotify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = HTMLSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "241.41733026504517\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "$(document).ready(\n",
       "    function() {\n",
       "        function appendUniqueDiv(){\n",
       "            // append a div with our uuid so we can check that it's already\n",
       "            // been sent and avoid duplicates on page reload\n",
       "            var notifiedDiv = document.createElement(\"div\")\n",
       "            notifiedDiv.id = \"7fe4137e-73be-43f4-ba06-403ba154770b\"\n",
       "            element.append(notifiedDiv)\n",
       "        }\n",
       "\n",
       "        // only send notifications if the pageload is complete; this will\n",
       "        // help stop extra notifications when a saved notebook is loaded,\n",
       "        // which during testing gives us state \"interactive\", not \"complete\"\n",
       "        if (document.readyState === 'complete') {\n",
       "            // check for the div that signifies that the notification\n",
       "            // was already sent\n",
       "            if (document.getElementById(\"7fe4137e-73be-43f4-ba06-403ba154770b\") === null) {\n",
       "                var notificationPayload = {\"requireInteraction\": false, \"icon\": \"/static/base/images/favicon.ico\", \"body\": \"Cell execution has finished!\"};\n",
       "                if (Notification.permission !== 'denied') {\n",
       "                    if (Notification.permission !== 'granted') { \n",
       "                        Notification.requestPermission(function (permission) {\n",
       "                            if(!('permission' in Notification)) {\n",
       "                                Notification.permission = permission\n",
       "                            }\n",
       "                        })\n",
       "                    }\n",
       "                    if (Notification.permission === 'granted') {\n",
       "                    var notification = new Notification(\"Jupyter Notebook\", notificationPayload)\n",
       "                    appendUniqueDiv()\n",
       "                    notification.onclick = function () {\n",
       "                        window.focus();\n",
       "                        this.close();\n",
       "                        };\n",
       "                    } \n",
       "                }     \n",
       "            }\n",
       "        }\n",
       "    }\n",
       ")\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%notify\n",
    "start = time.time()\n",
    "pages = 100\n",
    "single_page_urls = []\n",
    "page_cnt, url_cnt = 0, 0\n",
    "category = 'electronics'\n",
    "for p in range(1, pages+1):\n",
    "    bikroy_url = f'https://bikroy.com/en/ads/bangladesh/{category}?sort=date&order=desc&buy_now=0&urgent=0&page={p}'\n",
    "    print(bikroy_url)\n",
    "    r = session.get('http://localhost:8055/render.html', params={'url': bikroy_url, 'wait': 1})\n",
    "\n",
    "    page_cnt += 1\n",
    "    for i in range (1, 28):\n",
    "        try:\n",
    "#             single_url = driver.find_element(by=By.XPATH, value=f'//*[@id=\"app-wrapper\"]/div[1]/div[3]/div/div[2]/div[4]/div[2]/div/div[1]/div[1]/ul/li[{i}]/a').get_attribute('href')\n",
    "            single_url = r.html.find(f'#app-wrapper > div.container--297Nx.all > div:nth-child(3) > div > div.row--3Vhjr.main-container--28aG5.justify-content-flex-start--1Xozy.align-items-normal--vaTgD.flex-wrap-wrap--2PCx8.flex-direction-row--27fh1.flex--3fKk1 > div.sm-col-12--30zDS.lg-col-9--20qCf.block--3v-Ow > div.ad-list-container--1UnyA > div > div.ad-list--2Y3ql > div.list-wrapper--t_A02 > ul > li:nth-child({i}) > a')\n",
    "            single_url = single_url[0].attrs['href']\n",
    "            single_page_urls.append(single_url)\n",
    "            url_cnt += 1\n",
    "            print(\"Page: \" + str(page_cnt) + \" URL: \" + str(url_cnt))\n",
    "        except:\n",
    "            print('Not details link')\n",
    "    \n",
    "    clear_output(wait=True)\n",
    "print(time.time()-start)\n",
    "# driver.maximize_window()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2700"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(single_page_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'category_url/{category}.txt', 'w') as f:\n",
    "    for s in single_page_urls:\n",
    "        f.write(s)\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def features_collection(text):\n",
    "    data_dic = {}\n",
    "    ad_data = text[\"adDetail\"][\"data\"][\"ad\"]\n",
    "    title = ad_data[\"title\"]\n",
    "    description = ad_data[\"description\"]\n",
    "    properties = ad_data[\"properties\"]\n",
    "    features = {}\n",
    "    for p in properties:\n",
    "        features[p[\"label\"]] = p[\"value\"]\n",
    "#     print('Properties', properties)\n",
    "#     print('Features', features)\n",
    "    is_member = ad_data[\"isMember\"]\n",
    "    is_auth_dealer = ad_data[\"isAuthDealer\"]\n",
    "    is_featured_member = ad_data[\"isFeaturedMember\"]\n",
    "    membership_level = ad_data[\"membershipLevel\"]\n",
    "    is_verified = ad_data[\"isVerified\"]\n",
    "    door_step_delivery = ad_data[\"isDoorstepDelivery\"]\n",
    "    dsd_applicable = ad_data[\"isDSDApplicable\"]\n",
    "    timestamp = ad_data[\"timestamp\"]\n",
    "    date = ad_data[\"adDate\"]\n",
    "    deactivates = ad_data[\"deactivates\"]\n",
    "    location_data = ad_data[\"location\"]\n",
    "    location = location_data[\"name\"]\n",
    "    geography_data = location_data[\"geography\"][\"simple_polygons\"][0][\"coordinates\"]\n",
    "    geo_polygon_no = len(geography_data)\n",
    "\n",
    "    category_name = ad_data[\"category\"][\"name\"]\n",
    "    area = ad_data[\"area\"][\"name\"]\n",
    "    contact_name = ad_data[\"contactCard\"][\"name\"]\n",
    "    contact_number = ad_data[\"contactCard\"][\"phoneNumbers\"]\n",
    "    phone_number = [p[\"number\"] for p in contact_number]\n",
    "\n",
    "    shop_name = ad_data[\"shop\"][\"name\"]\n",
    "    shop_email = ad_data[\"shop\"][\"email\"]\n",
    "\n",
    "    price = ad_data[\"money\"][\"amount\"]\n",
    "    delivery_free = ad_data[\"isDeliveryFree\"]\n",
    "\n",
    "    similar_ads = text[\"adDetail\"][\"data\"][\"similarAds\"]\n",
    "\n",
    "    img_path = []\n",
    "    image_urls =  text[\"image\"]\n",
    "    for img_url in image_urls:\n",
    "        r = requests.get(img_url)\n",
    "        i = Image.open(BytesIO(r.content))\n",
    "        img_name = '@'.join(img_url.split('/')[3:5])\n",
    "        i.save(f\"img/{category}/{img_name}.jpg\")\n",
    "        img_path.append(img_name)\n",
    "    \n",
    "    data_dic['title'] = title\n",
    "    data_dic['description'] = description\n",
    "    data_dic['features'] = features\n",
    "    data_dic['is_member'] = is_member\n",
    "    data_dic['is_auth_dealer'] = is_auth_dealer\n",
    "    data_dic['is_featured_member'] = is_featured_member\n",
    "    data_dic['membership_level'] = membership_level\n",
    "    data_dic['is_verified'] = is_verified\n",
    "    data_dic['door_step_delivery'] = door_step_delivery\n",
    "    data_dic['dsd_applicable'] = dsd_applicable\n",
    "    data_dic['timestamp'] = timestamp\n",
    "    data_dic['date'] = date\n",
    "    data_dic['deactivates'] = deactivates\n",
    "    data_dic['geography_data'] = geography_data\n",
    "    data_dic['geo_polygon_no'] = geo_polygon_no\n",
    "    data_dic['area'] = area\n",
    "    data_dic['contact_name'] = contact_name\n",
    "    data_dic['phone_number'] = phone_number\n",
    "    data_dic['shop_name'] = shop_name\n",
    "    data_dic['shop_email'] = shop_email\n",
    "    data_dic['price'] = price\n",
    "    data_dic['delivery_free'] = delivery_free\n",
    "    data_dic['similar_ads'] = similar_ads\n",
    "    data_dic['image_url'] = image_urls\n",
    "    data_dic['image_path'] = img_path\n",
    "    \n",
    "    return data_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 https://bikroy.com/en/ad/oneplus-9-pro-12gb-256gb-global-new-for-sale-chattogram\n",
      "200\n",
      "=============================================\n",
      "2 https://bikroy.com/en/ad/oppo-a57-3-32-new-for-sale-barishal-7\n",
      "200\n",
      "2 https://bikroy.com/en/ad/oppo-a57-3-32-new-for-sale-barishal-7\n",
      "Failed\n",
      "------------------\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-j6-3-32-used-for-sale-dhaka-770\n",
      "200\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-j6-3-32-used-for-sale-dhaka-770\n",
      "Failed\n",
      "------------------\n",
      "2 https://bikroy.com/en/ad/realme-c11-used-for-sale-sylhet-division-12\n",
      "200\n",
      "2 https://bikroy.com/en/ad/realme-c11-used-for-sale-sylhet-division-12\n",
      "Failed\n",
      "------------------\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-a7-2018-used-for-sale-dhaka-1198\n",
      "200\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-a7-2018-used-for-sale-dhaka-1198\n",
      "Failed\n",
      "------------------\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-j2-15gb-32gb-used-for-sale-khulna-division\n",
      "200\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-j2-15gb-32gb-used-for-sale-khulna-division\n",
      "Failed\n",
      "------------------\n",
      "2 https://bikroy.com/en/ad/xiaomi-redmi-note-7-pro-4-64-used-for-sale-khulna-327\n",
      "200\n",
      "2 https://bikroy.com/en/ad/xiaomi-redmi-note-7-pro-4-64-used-for-sale-khulna-327\n",
      "Failed\n",
      "------------------\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-a20s-4-64-used-for-sale-chattogram-division-14\n",
      "200\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-a20s-4-64-used-for-sale-chattogram-division-14\n",
      "Failed\n",
      "------------------\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-s9-plus-2018-used-for-sale-dhaka-95\n",
      "200\n",
      "2 https://bikroy.com/en/ad/samsung-galaxy-s9-plus-2018-used-for-sale-dhaka-95\n",
      "Failed\n",
      "------------------\n",
      "2 https://bikroy.com/en/ad/symphony-b6-used-for-sale-dhaka-595\n",
      "200\n",
      "2 https://bikroy.com/en/ad/symphony-b6-used-for-sale-dhaka-595\n",
      "Failed\n",
      "------------------\n"
     ]
    }
   ],
   "source": [
    "category = 'mobile'\n",
    "session = HTMLSession()\n",
    "\n",
    "with open(f'category_url/{category}.txt', 'r') as f:\n",
    "    single_url = f.readlines()\n",
    "    titles, locations, prices, img_lst, meta_lst,  feature_lst, desc_lst = [], [], [], [], [], [], []\n",
    "    cnt = 1\n",
    "#     print(single_url)\n",
    "    all_data = []\n",
    "    for s in single_url[:10]:\n",
    "        try:\n",
    "            s = s.replace('\\n', '')\n",
    "#             s = 'https://bikroy.com'+s\n",
    "            print(cnt, s)\n",
    "            r = session.get('http://localhost:8055/render.html', params={'url': s, 'wait': 5})\n",
    "            print(r.status_code)       \n",
    "            text = r.html.text \n",
    "\n",
    "            a_split = text.split('window.initialData = ')[1][:-1]\n",
    "            b_split = a_split.split('\"@context\":') #\"@context\": \"http://schema.org\",\n",
    "            data = b_split[0][:-3] + ',' + '\"@context\": ' + b_split[1][:-2]\n",
    "            data = data.replace('\\\\', '')\n",
    "            jtext = json.JSONDecoder().decode(data)\n",
    "\n",
    "            # attributes extraction\n",
    "            data_dic = features_collection(jtext)\n",
    "            data_dic['url'] = s\n",
    "            all_data.append(data_dic)\n",
    "    #         if title:\n",
    "            cnt += 1\n",
    "            print(\"=============================================\")\n",
    "#             clear_output(wait=True)\n",
    "        except:\n",
    "            print(cnt, s)\n",
    "            print('Failed')\n",
    "            print('------------------')\n",
    "#             pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'locale': 'en',\n",
       " 'marketLocales': ['en', 'bn'],\n",
       " 'locations': {'type': 'NotAsked'},\n",
       " 'categories': {'type': 'NotAsked'},\n",
       " 'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       " 'browserInfo': {'fakeWidth': 1200, 'webpSupported': False},\n",
       " 'jobsVertical': {'filters': {'type': 'NotAsked'}},\n",
       " 'serp': {'ads': {'type': 'NotAsked'},\n",
       "  'staticFilters': {'sort': 'date_desc',\n",
       "   'posterType': 'all_posters',\n",
       "   'isDoorstepDelivery': False,\n",
       "   'isUrgent': False,\n",
       "   'isFeaturedMember': False,\n",
       "   'location': -1,\n",
       "   'category': -1,\n",
       "   'categorySlug': '',\n",
       "   'locationSlug': 'bangladesh',\n",
       "   'page': 1,\n",
       "   'query': ''},\n",
       "  'dynamicFilters': {},\n",
       "  'snapshot': None},\n",
       " 'verticals': [{'key': 'jobs',\n",
       "   'name': 'Jobs',\n",
       "   'categories': [{'id': 700},\n",
       "    {'id': 1401},\n",
       "    {'id': 1402},\n",
       "    {'id': 1404},\n",
       "    {'id': 1405},\n",
       "    {'id': 1406},\n",
       "    {'id': 1407},\n",
       "    {'id': 1408},\n",
       "    {'id': 1409},\n",
       "    {'id': 1410},\n",
       "    {'id': 1411},\n",
       "    {'id': 1412},\n",
       "    {'id': 1413},\n",
       "    {'id': 1414},\n",
       "    {'id': 1415},\n",
       "    {'id': 1487},\n",
       "    {'id': 1416},\n",
       "    {'id': 1417},\n",
       "    {'id': 1488},\n",
       "    {'id': 1489},\n",
       "    {'id': 1501},\n",
       "    {'id': 1418},\n",
       "    {'id': 1419},\n",
       "    {'id': 1420},\n",
       "    {'id': 1421},\n",
       "    {'id': 1422},\n",
       "    {'id': 1423},\n",
       "    {'id': 1424},\n",
       "    {'id': 1425},\n",
       "    {'id': 1426},\n",
       "    {'id': 1427},\n",
       "    {'id': 1490},\n",
       "    {'id': 1428},\n",
       "    {'id': 1429},\n",
       "    {'id': 1430},\n",
       "    {'id': 1482},\n",
       "    {'id': 1483},\n",
       "    {'id': 1431},\n",
       "    {'id': 1432},\n",
       "    {'id': 1433},\n",
       "    {'id': 1434},\n",
       "    {'id': 1491},\n",
       "    {'id': 1492},\n",
       "    {'id': 1486},\n",
       "    {'id': 1493},\n",
       "    {'id': 1435},\n",
       "    {'id': 1436},\n",
       "    {'id': 1438},\n",
       "    {'id': 1439},\n",
       "    {'id': 1440},\n",
       "    {'id': 1494},\n",
       "    {'id': 1495},\n",
       "    {'id': 1441},\n",
       "    {'id': 1442},\n",
       "    {'id': 1443},\n",
       "    {'id': 1444},\n",
       "    {'id': 1445},\n",
       "    {'id': 1446},\n",
       "    {'id': 1447},\n",
       "    {'id': 1484},\n",
       "    {'id': 1449},\n",
       "    {'id': 1450},\n",
       "    {'id': 1452},\n",
       "    {'id': 1453},\n",
       "    {'id': 1481},\n",
       "    {'id': 1454},\n",
       "    {'id': 1455},\n",
       "    {'id': 1456},\n",
       "    {'id': 1457},\n",
       "    {'id': 1458},\n",
       "    {'id': 1459},\n",
       "    {'id': 1485},\n",
       "    {'id': 1496},\n",
       "    {'id': 1497},\n",
       "    {'id': 1460},\n",
       "    {'id': 1461},\n",
       "    {'id': 1462},\n",
       "    {'id': 1463},\n",
       "    {'id': 1502},\n",
       "    {'id': 1464},\n",
       "    {'id': 1466},\n",
       "    {'id': 1467},\n",
       "    {'id': 1498},\n",
       "    {'id': 1468},\n",
       "    {'id': 1499},\n",
       "    {'id': 1469},\n",
       "    {'id': 1500},\n",
       "    {'id': 1470},\n",
       "    {'id': 1471},\n",
       "    {'id': 1472},\n",
       "    {'id': 1473},\n",
       "    {'id': 1474},\n",
       "    {'id': 1476},\n",
       "    {'id': 1477},\n",
       "    {'id': 1478},\n",
       "    {'id': 1479},\n",
       "    {'id': 1480},\n",
       "    {'id': 1451}]}],\n",
       " 'isGhanaWeb': False,\n",
       " 'market': {'integrations': []},\n",
       " 'experimentCookie': '',\n",
       " 'adDetail': {'type': 'Success',\n",
       "  'data': {'ad': {'id': '622ece28087489004895900e',\n",
       "    'account': {'id': '512aedee8d1ca33b0c000279', 'traits': {'member': True}},\n",
       "    'details': '',\n",
       "    'slug': 'macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka',\n",
       "    'title': 'MacBook Air A1466/A1369 & A1465/A1370 Order LCD',\n",
       "    'description': 'Address ;nBranch 1 : Daymon Computer bd nShop : 901nLevel : 9nMaltiplan center, New Elephant Road, DhakanBranch 2 : Dapple nHouse : 32nLevel : 6nRoad : 7nBlock :F Banani.nProduct : MacBook Air LCD Display nModel : A1466/A1369 & A1465/A1370 nWarranty : One Month nQuality : Original Apple Products nAlways available this products....',\n",
       "    'properties': [{'label': 'Condition',\n",
       "      'value': 'New',\n",
       "      'key': 'condition',\n",
       "      'value_key': 'new'},\n",
       "     {'label': 'Item type',\n",
       "      'value': 'Monitors',\n",
       "      'key': 'item_type',\n",
       "      'value_key': 'monitor'}],\n",
       "    'isMember': True,\n",
       "    'isAuthDealer': False,\n",
       "    'isFeaturedMember': False,\n",
       "    'type': 'for_sale',\n",
       "    'status': 'published',\n",
       "    'membershipLevel': 'plus',\n",
       "    'memberSince': 'June 2021',\n",
       "    'isVerified': False,\n",
       "    'postedDate': 0,\n",
       "    'isDoorstepDelivery': False,\n",
       "    'isDSDApplicable': True,\n",
       "    'timestamp': '14 Mar 11:26 am',\n",
       "    'adDate': '2022-03-14T11:26:50+06:00',\n",
       "    'noExpiration': True,\n",
       "    'deactivates': '2022-05-13T05:10:00.000Z',\n",
       "    'location': {'id': 1235,\n",
       "     'name': 'Elephant Road',\n",
       "     'children': [],\n",
       "     'geography': {'simple_polygons': [{'coordinates': [{'latitude': 23.738978,\n",
       "          'longitude': 90.390465},\n",
       "         {'latitude': 23.73892, 'longitude': 90.383535},\n",
       "         {'latitude': 23.732615, 'longitude': 90.385209},\n",
       "         {'latitude': 23.73284, 'longitude': 90.387182},\n",
       "         {'latitude': 23.73406, 'longitude': 90.387831},\n",
       "         {'latitude': 23.735335, 'longitude': 90.388684},\n",
       "         {'latitude': 23.738447, 'longitude': 90.391259}]}]},\n",
       "     'geo_region': 'BD-13',\n",
       "     'slug': 'elephant-road',\n",
       "     'link': '/ads/elephant-road',\n",
       "     'parentId': 1211,\n",
       "     'parent': {'id': 1211,\n",
       "      'children': [1255,\n",
       "       1292,\n",
       "       1259,\n",
       "       1281,\n",
       "       1232,\n",
       "       1242,\n",
       "       1215,\n",
       "       1224,\n",
       "       4068,\n",
       "       1221,\n",
       "       1235,\n",
       "       1276,\n",
       "       1254,\n",
       "       1260,\n",
       "       1251,\n",
       "       1217,\n",
       "       1253,\n",
       "       4076,\n",
       "       4082,\n",
       "       1291,\n",
       "       1279,\n",
       "       4069,\n",
       "       3580,\n",
       "       1230,\n",
       "       4080,\n",
       "       3582,\n",
       "       1226,\n",
       "       1272,\n",
       "       4081,\n",
       "       1290,\n",
       "       1249,\n",
       "       1220,\n",
       "       1294,\n",
       "       1275,\n",
       "       4079,\n",
       "       4078,\n",
       "       1227,\n",
       "       1241,\n",
       "       3579,\n",
       "       1231,\n",
       "       1233,\n",
       "       1245,\n",
       "       1263,\n",
       "       1218,\n",
       "       1256,\n",
       "       4088,\n",
       "       1284],\n",
       "      'name': 'Dhaka',\n",
       "      'geography': {'simple_polygons': [{'coordinates': [{'latitude': 23.836923,\n",
       "           'longitude': 90.248428},\n",
       "          {'latitude': 23.758725, 'longitude': 90.245876},\n",
       "          {'latitude': 23.764749, 'longitude': 90.322621},\n",
       "          {'latitude': 23.652392, 'longitude': 90.386777},\n",
       "          {'latitude': 23.712127, 'longitude': 90.51735},\n",
       "          {'latitude': 23.900967, 'longitude': 90.464476},\n",
       "          {'latitude': 23.922007, 'longitude': 90.391296},\n",
       "          {'latitude': 24.024114, 'longitude': 90.242784},\n",
       "          {'latitude': 24.013722, 'longitude': 90.196741},\n",
       "          {'latitude': 23.999115, 'longitude': 90.188486},\n",
       "          {'latitude': 23.980117, 'longitude': 90.193279},\n",
       "          {'latitude': 23.970899, 'longitude': 90.207053},\n",
       "          {'latitude': 23.962934, 'longitude': 90.221513},\n",
       "          {'latitude': 23.948258, 'longitude': 90.226397},\n",
       "          {'latitude': 23.914515, 'longitude': 90.230667},\n",
       "          {'latitude': 23.872597, 'longitude': 90.237666}]}]},\n",
       "      'geo_region': 'BD-13',\n",
       "      'slug': 'dhaka',\n",
       "      'link': '/ads/dhaka',\n",
       "      'parentId': -1}},\n",
       "    'category': {'id': 897,\n",
       "     'name': 'Laptop & Computer Accessories',\n",
       "     'children': [],\n",
       "     'ad_types': ['for_sale', 'to_buy'],\n",
       "     'serp_type': 'ads',\n",
       "     'available_serp_layouts': ['classic', 'fluid'],\n",
       "     'default_serp_layout': 'fluid',\n",
       "     'default_ad_type': 'for_sale',\n",
       "     'slug': 'laptop-computer-accessories',\n",
       "     'form_view_version': '1',\n",
       "     'parentId': 229,\n",
       "     'parent': {'id': 229,\n",
       "      'children': [893, 896, 897, 894, 851, 850, 855, 846, 848, 242, 895, 244],\n",
       "      'name': 'Electronics',\n",
       "      'ad_types': [],\n",
       "      'serp_type': 'ads',\n",
       "      'available_serp_layouts': ['classic', 'fluid'],\n",
       "      'default_serp_layout': 'fluid',\n",
       "      'type': 'electronics',\n",
       "      'slug': 'electronics',\n",
       "      'form_view_version': '1',\n",
       "      'parentId': -1}},\n",
       "    'area': {'id': 1211, 'name': 'Dhaka'},\n",
       "    'contactCard': {'name': 'Daymon Computer (BD)',\n",
       "     'phoneNumbers': [{'number': '01722298111', 'verified': True},\n",
       "      {'number': '01744403111', 'verified': True}],\n",
       "     'chatEnabled': True},\n",
       "    'images': {'meta': [{'src': 'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/d75e6d44-d156-40d8-8f26-13aaac6979b4',\n",
       "       'alt': 'MacBook Air A1466/A1369 & A1465/A1370 Order LCD in Dhaka'},\n",
       "      {'src': 'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/ee70968b-bb0c-45ff-8959-9eeaa50f3aa5',\n",
       "       'alt': 'MacBook Air A1466/A1369 & A1465/A1370 Order LCD in Elephant Road'},\n",
       "      {'src': 'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/450ef6eb-1493-4e8e-85f3-1d817200c537',\n",
       "       'alt': 'MacBook Air A1466/A1369 & A1465/A1370 Order LCD Laptop & Computer Accessories'},\n",
       "      {'src': 'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/e031ccc0-cfbb-4311-ab69-7a1de57a0c5d',\n",
       "       'alt': 'MacBook Air A1466/A1369 & A1465/A1370 Order LCD Monitors'},\n",
       "      {'src': 'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/56f7a0e3-9a63-41ee-bcdc-0f1155b1ab58',\n",
       "       'alt': 'MacBook Air A1466/A1369 & A1465/A1370 Order LCD'}],\n",
       "     'base_uri': 'https://i.bikroy-st.com',\n",
       "     'metadata': []},\n",
       "    'shop': {'id': '56f27f389d232a00015d2c6a',\n",
       "     'slug': 'daymoncomputer',\n",
       "     'name': 'Daymon Computer(BD)',\n",
       "     'email': 'daymon_computer@yahoo.com',\n",
       "     'logo': 'https://i.bikroy-st.com/u/daymoncomputer/656ad4ab-81c4-42d7-9910-1fecfa1d087b/48/36/cropped.jpg',\n",
       "     'contactSectionLogo': 'https://i.bikroy-st.com/u/daymoncomputer/656ad4ab-81c4-42d7-9910-1fecfa1d087b/72/54/cropped.jpg'},\n",
       "    'money': {'label': 'Price', 'amount': 'Tk 15,000'},\n",
       "    'statistics': {},\n",
       "    'job': {},\n",
       "    'isDeliveryFree': False,\n",
       "    'memberLogo': 'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/d75e6d44-d156-40d8-8f26-13aaac6979b4/72/54/cropped.jpg',\n",
       "    'imgUrl': 'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/d75e6d44-d156-40d8-8f26-13aaac6979b4/142/107/cropped.jpg',\n",
       "    'isLocalJob': False},\n",
       "   'similarAds': [{'id': '62049bd086f565001a90e653',\n",
       "     'slug': 'macbook-air-a1370-a1465-and-a1369-a1466-original-keyboard-usuk-for-sale-dhaka',\n",
       "     'title': 'MacBook Air A1370/A1465 & A1369/A1466 Original keyboard US,UK',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-air-a1370-a1465-and-a1369-a1466-original-keyboard-usuk-for-sale-dhaka/ead694f0-d27e-41a3-a0e5-bec77dde278b/142/107/cropped.jpg',\n",
       "     'price': 'Tk 5,000',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '30 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '622d82787a643b00754cff94',\n",
       "     'slug': 'macbook-air-a1466-a1369-and-a1465-a1370-original-battery-for-sale-dhaka',\n",
       "     'title': 'MacBook Air A1466/A1369 & A1465/A1370 Original Battery',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-original-battery-for-sale-dhaka/ef3470d7-37e2-4f39-82a3-f8acb7921f54/142/107/cropped.jpg',\n",
       "     'price': 'Tk 5,500',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '1 day',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '62287ab1a972df367b669b64',\n",
       "     'slug': 'macbook-pro-a2338-and-air-a2337-display-for-sale-dhaka',\n",
       "     'title': 'MacBook Pro A2338 & Air A2337 Display',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a2338-and-air-a2337-display-for-sale-dhaka/00894c42-8e48-4832-a461-8ba1f459ad68/142/107/cropped.jpg',\n",
       "     'price': 'Tk 50,000',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '5 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '621b0255ae1f3c001a806dfd',\n",
       "     'slug': 'macbook-pro-a1706-new-lcd-screen-for-sale-dhaka',\n",
       "     'title': 'MacBook Pro A1706 new LCD Screen',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1706-new-lcd-screen-for-sale-dhaka/b69a76aa-12ad-4dcf-b8bf-c542424e2efc/142/107/cropped.jpg',\n",
       "     'price': 'Tk 25,000',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '15 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '6221accb44130900474423ef',\n",
       "     'slug': 'macbook-pro-a1286-and-a1278-original-lcd-for-sale-dhaka',\n",
       "     'title': 'MacBook Pro A1286 & A1278 Original LCD',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1286-and-a1278-original-lcd-for-sale-dhaka/2e7fc5ae-b6cf-47cc-ba5c-76450257f481/142/107/cropped.jpg',\n",
       "     'price': 'Tk 15,000',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '10 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '621775f0595fa3001abf64e9',\n",
       "     'slug': 'macbook-ssd-converter-for-sale-dhaka-1',\n",
       "     'title': 'MacBook SSD Converter',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-ssd-converter-for-sale-dhaka-1/a8c411fb-7437-4f03-92ac-f2d727e3fba1/142/107/cropped.jpg',\n",
       "     'price': 'Tk 3,000',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '18 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '62073c2ecc8d030076b3422f',\n",
       "     'slug': 'macbook-pro-a1502-original-speaker-for-sale-dhaka-3',\n",
       "     'title': 'MacBook Pro A1502 Original Speaker',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1502-original-speaker-for-sale-dhaka-3/1dcb2b27-c4b2-44a5-a0e4-1db067100035/142/107/cropped.jpg',\n",
       "     'price': 'Tk 5,500',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '30 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '621f9c561b718612dbf0e007',\n",
       "     'slug': 'macbook-pro-a1278-original-battery-for-sale-dhaka-5',\n",
       "     'title': 'MacBook Pro A1278 original Battery',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1278-original-battery-for-sale-dhaka-5/748f02f9-def3-42d1-ae8a-a9f03947026b/142/107/cropped.jpg',\n",
       "     'price': 'Tk 4,500',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '11 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '6226df64a9f081001935e444',\n",
       "     'slug': 'macbook-pro-a1708-original-battery-for-sale-dhaka-6',\n",
       "     'title': 'MacBook Pro A1708 Original Battery',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1708-original-battery-for-sale-dhaka-6/a36c922c-e2ca-423d-92af-982df30758c9/142/107/cropped.jpg',\n",
       "     'price': 'Tk 9,500',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '6 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '62258ed369b6810075d21525',\n",
       "     'slug': 'macbook-pro-a1707-original-battery-for-sale-dhaka-4',\n",
       "     'title': 'MacBook Pro A1707 Original Battery',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1707-original-battery-for-sale-dhaka-4/8b19f068-1f6d-451f-bcbb-2e249f94f4a6/142/107/cropped.jpg',\n",
       "     'price': 'Tk 12,500',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '7 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '622abe3977b0ab0019505215',\n",
       "     'slug': 'macbook-pro-a1286-original-keyboard-for-sale-dhaka',\n",
       "     'title': 'MacBook Pro A1286 Original Keyboard',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1286-original-keyboard-for-sale-dhaka/830bc764-0b51-4fd7-b5ec-36d49e6953e5/142/107/cropped.jpg',\n",
       "     'price': 'Tk 4,000',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '3 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '621777946064720019e3c4cf',\n",
       "     'slug': 'macbook-pro-a1502-new-motherboard-for-sale-dhaka',\n",
       "     'title': 'MacBook Pro A1502 New Motherboard',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1502-new-motherboard-for-sale-dhaka/efa555b7-f025-4498-bb43-18a3947460b8/142/107/cropped.jpg',\n",
       "     'price': 'Tk 25,000',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '18 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '621b0134ba64e20048ce688a',\n",
       "     'slug': 'macbook-pro-a1706-original-keyboard-for-sale-dhaka',\n",
       "     'title': 'MacBook Pro A1706 Original Keyboard',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1706-original-keyboard-for-sale-dhaka/b491c6f6-d9a4-4dcc-a8f2-57b7c41d2656/142/107/cropped.jpg',\n",
       "     'price': 'Tk 9,500',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '15 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '62230031200bfa00779c5ff9',\n",
       "     'slug': 'macbook-pro-a1278-original-speaker-for-sale-dhaka-6',\n",
       "     'title': 'MacBook Pro A1278 Original Speaker',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1278-original-speaker-for-sale-dhaka-6/5b4b14c5-2d52-47e3-bbc8-9653326bb447/142/107/cropped.jpg',\n",
       "     'price': 'Tk 3,500',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '9 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '622c2151ae030e00440224ee',\n",
       "     'slug': 'macbook-pro-a1286-original-battery-for-sale-dhaka-7',\n",
       "     'title': 'MacBook Pro A1286 Original Battery',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1286-original-battery-for-sale-dhaka-7/f37e0ac6-fcc5-420c-8e14-29d42c3d3694/142/107/cropped.jpg',\n",
       "     'price': 'Tk 5,000',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '2 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False},\n",
       "    {'id': '61f4c5926d8705007c3b825d',\n",
       "     'slug': 'macbook-pro-a1708-original-battery-for-sale-dhaka-5',\n",
       "     'title': 'MacBook Pro A1708 Original Battery',\n",
       "     'description': 'Dhaka, Laptop & Computer Accessories',\n",
       "     'details': '',\n",
       "     'imgUrl': 'https://i.bikroy-st.com/macbook-pro-a1708-original-battery-for-sale-dhaka-5/fe289cf6-2080-4eca-815d-bba7c215bdfd/142/107/cropped.jpg',\n",
       "     'price': 'Tk 9,500',\n",
       "     'discount': 0,\n",
       "     'mrp': 0,\n",
       "     'isMember': True,\n",
       "     'isAuthDealer': False,\n",
       "     'isFeaturedMember': False,\n",
       "     'membershipLevel': 'plus',\n",
       "     'shopName': 'Daymon Computer(BD)',\n",
       "     'isDoorstepDelivery': False,\n",
       "     'isDeliveryFree': False,\n",
       "     'isTopAd': False,\n",
       "     'isUrgentAd': False,\n",
       "     'timeStamp': '44 days',\n",
       "     'category': {'id': 897, 'name': 'Laptop & Computer Accessories'},\n",
       "     'isVerified': False,\n",
       "     'isJobAd': False,\n",
       "     'location': 'Dhaka',\n",
       "     'adType': 'for_sale',\n",
       "     'isLocalJob': False}],\n",
       "   'safetyTips': ['Avoid offers that look unrealistic',\n",
       "    'Chat with seller to clarify item details',\n",
       "    'Meet in a safe & public place',\n",
       "    'Check the item before buying it',\n",
       "    'Don’t pay in advance'],\n",
       "   'title': 'MacBook Air A1466/A1369 & A1465/A1370 Order LCD in Elephant Road | Bikroy.com',\n",
       "   'description': 'Address ;nBranch 1 : Daymon Computer bd nShop : 901nLevel : 9nMaltiplan center, New Elephant Road, DhakanBranch 2 : Dapple nHouse : 32nLevel : 6nRoad : 7nBlock :F Banani....'}},\n",
       " 'subShop': {'type': 'NotAsked'},\n",
       " 'shop': {'type': 'NotAsked'},\n",
       " 'shopSerp': {'type': 'NotAsked'},\n",
       " 'banners': {'type': 'Success',\n",
       "  'data': {'web': {'skyscraper': {'slot': '/134461134/bikroy_desktop_detailview_skyscraper',\n",
       "     'sizes': [[300, 250], [300, 600]],\n",
       "     'key_values': [{'key': 'url',\n",
       "       'values': ['macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka']},\n",
       "      {'key': 'page', 'values': ['detail']},\n",
       "      {'key': 'locale', 'values': ['en']},\n",
       "      {'key': 'by_paying_member', 'values': [True]},\n",
       "      {'key': 'by_authorized_dealer', 'values': [False]},\n",
       "      {'key': 'buy_now', 'values': [False]},\n",
       "      {'key': 'shop_id', 'values': ['56f27f389d232a00015d2c6a']},\n",
       "      {'key': 'by_featured_member', 'values': [False]},\n",
       "      {'key': 'l1_category', 'values': ['electronics']},\n",
       "      {'key': 'l2_category', 'values': ['computer_accessories']},\n",
       "      {'key': 'bikroy_l1_location', 'values': ['dhaka']},\n",
       "      {'key': 'bikroy_l2_location', 'values': ['elephant_road']},\n",
       "      {'key': 'bikroy_price', 'values': ['10-20k']},\n",
       "      {'key': 'condition', 'values': ['new']},\n",
       "      {'key': 'Ad_Format', 'values': ['skyscraper']},\n",
       "      {'key': 'Platform', 'values': ['Desktop']}]},\n",
       "    'leaderboard': {'slot': '/134461134/bikroy_desktop_detailview_leaderboard',\n",
       "     'sizes': [[970, 90]],\n",
       "     'key_values': [{'key': 'url',\n",
       "       'values': ['macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka']},\n",
       "      {'key': 'page', 'values': ['detail']},\n",
       "      {'key': 'locale', 'values': ['en']},\n",
       "      {'key': 'by_paying_member', 'values': [True]},\n",
       "      {'key': 'by_authorized_dealer', 'values': [False]},\n",
       "      {'key': 'buy_now', 'values': [False]},\n",
       "      {'key': 'shop_id', 'values': ['56f27f389d232a00015d2c6a']},\n",
       "      {'key': 'by_featured_member', 'values': [False]},\n",
       "      {'key': 'l1_category', 'values': ['electronics']},\n",
       "      {'key': 'l2_category', 'values': ['computer_accessories']},\n",
       "      {'key': 'bikroy_l1_location', 'values': ['dhaka']},\n",
       "      {'key': 'bikroy_l2_location', 'values': ['elephant_road']},\n",
       "      {'key': 'bikroy_price', 'values': ['10-20k']},\n",
       "      {'key': 'condition', 'values': ['new']},\n",
       "      {'key': 'Ad_Format', 'values': ['leaderboard']},\n",
       "      {'key': 'Platform', 'values': ['Desktop']}]},\n",
       "    'lowerboard': {'slot': '/134461134/bikroy_desktop_detailview_lowerboard',\n",
       "     'sizes': [[970, 90]],\n",
       "     'key_values': [{'key': 'url',\n",
       "       'values': ['macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka']},\n",
       "      {'key': 'page', 'values': ['detail']},\n",
       "      {'key': 'locale', 'values': ['en']},\n",
       "      {'key': 'by_paying_member', 'values': [True]},\n",
       "      {'key': 'by_authorized_dealer', 'values': [False]},\n",
       "      {'key': 'buy_now', 'values': [False]},\n",
       "      {'key': 'shop_id', 'values': ['56f27f389d232a00015d2c6a']},\n",
       "      {'key': 'by_featured_member', 'values': [False]},\n",
       "      {'key': 'l1_category', 'values': ['electronics']},\n",
       "      {'key': 'l2_category', 'values': ['computer_accessories']},\n",
       "      {'key': 'bikroy_l1_location', 'values': ['dhaka']},\n",
       "      {'key': 'bikroy_l2_location', 'values': ['elephant_road']},\n",
       "      {'key': 'bikroy_price', 'values': ['10-20k']},\n",
       "      {'key': 'condition', 'values': ['new']},\n",
       "      {'key': 'Ad_Format', 'values': ['lowerboard']},\n",
       "      {'key': 'Platform', 'values': ['Desktop']}]}},\n",
       "   'mobile_web': {'leaderboard': {'slot': '/134461134/bikroy_mobile_detailview_leaderboard',\n",
       "     'sizes': [[320, 100]],\n",
       "     'key_values': [{'key': 'url',\n",
       "       'values': ['macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka']},\n",
       "      {'key': 'page', 'values': ['detail']},\n",
       "      {'key': 'locale', 'values': ['en']},\n",
       "      {'key': 'by_paying_member', 'values': [True]},\n",
       "      {'key': 'by_authorized_dealer', 'values': [False]},\n",
       "      {'key': 'buy_now', 'values': [False]},\n",
       "      {'key': 'shop_id', 'values': ['56f27f389d232a00015d2c6a']},\n",
       "      {'key': 'by_featured_member', 'values': [False]},\n",
       "      {'key': 'l1_category', 'values': ['electronics']},\n",
       "      {'key': 'l2_category', 'values': ['computer_accessories']},\n",
       "      {'key': 'bikroy_l1_location', 'values': ['dhaka']},\n",
       "      {'key': 'bikroy_l2_location', 'values': ['elephant_road']},\n",
       "      {'key': 'bikroy_price', 'values': ['10-20k']},\n",
       "      {'key': 'condition', 'values': ['new']},\n",
       "      {'key': 'Ad_Format', 'values': ['leaderboard']},\n",
       "      {'key': 'Platform', 'values': ['Mobile']}]},\n",
       "    'middle': {'slot': '/134461134/bikroy_mobile_detailview_middle',\n",
       "     'sizes': [[300, 250], [320, 100], [320, 250]],\n",
       "     'key_values': [{'key': 'url',\n",
       "       'values': ['macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka']},\n",
       "      {'key': 'page', 'values': ['detail']},\n",
       "      {'key': 'locale', 'values': ['en']},\n",
       "      {'key': 'by_paying_member', 'values': [True]},\n",
       "      {'key': 'by_authorized_dealer', 'values': [False]},\n",
       "      {'key': 'buy_now', 'values': [False]},\n",
       "      {'key': 'shop_id', 'values': ['56f27f389d232a00015d2c6a']},\n",
       "      {'key': 'by_featured_member', 'values': [False]},\n",
       "      {'key': 'l1_category', 'values': ['electronics']},\n",
       "      {'key': 'l2_category', 'values': ['computer_accessories']},\n",
       "      {'key': 'bikroy_l1_location', 'values': ['dhaka']},\n",
       "      {'key': 'bikroy_l2_location', 'values': ['elephant_road']},\n",
       "      {'key': 'bikroy_price', 'values': ['10-20k']},\n",
       "      {'key': 'condition', 'values': ['new']},\n",
       "      {'key': 'Ad_Format', 'values': ['middle']},\n",
       "      {'key': 'Platform', 'values': ['Mobile']}]},\n",
       "    'lowerboard': {'slot': '/134461134/bikroy_mobile_detailview_lowerboard',\n",
       "     'sizes': [[300, 250], [320, 100], [320, 250]],\n",
       "     'key_values': [{'key': 'url',\n",
       "       'values': ['macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka']},\n",
       "      {'key': 'page', 'values': ['detail']},\n",
       "      {'key': 'locale', 'values': ['en']},\n",
       "      {'key': 'by_paying_member', 'values': [True]},\n",
       "      {'key': 'by_authorized_dealer', 'values': [False]},\n",
       "      {'key': 'buy_now', 'values': [False]},\n",
       "      {'key': 'shop_id', 'values': ['56f27f389d232a00015d2c6a']},\n",
       "      {'key': 'by_featured_member', 'values': [False]},\n",
       "      {'key': 'l1_category', 'values': ['electronics']},\n",
       "      {'key': 'l2_category', 'values': ['computer_accessories']},\n",
       "      {'key': 'bikroy_l1_location', 'values': ['dhaka']},\n",
       "      {'key': 'bikroy_l2_location', 'values': ['elephant_road']},\n",
       "      {'key': 'bikroy_price', 'values': ['10-20k']},\n",
       "      {'key': 'condition', 'values': ['new']},\n",
       "      {'key': 'Ad_Format', 'values': ['lowerboard']},\n",
       "      {'key': 'Platform', 'values': ['Mobile']}]}}}},\n",
       " 'user': {'favorites': {'type': 'NotAsked'},\n",
       "  'ads': {'type': 'NotAsked'},\n",
       "  'id': None,\n",
       "  'sessionToken': None,\n",
       "  'account': {'type': 'NotAsked'},\n",
       "  'signup': {'type': 'NotAsked'}},\n",
       " 'availableProducts': {'type': 'NotAsked'},\n",
       " 'paymentDetails': {'type': 'NotAsked'},\n",
       " 'categoryBreakdown': {'type': 'NotAsked'},\n",
       " 'doorstep': {'featuredProducts': {'type': 'NotAsked'},\n",
       "  'popularProducts': {'type': 'NotAsked'},\n",
       "  'doorstepBreakdown': {'type': 'NotAsked'},\n",
       "  'featuredShops': {'type': 'NotAsked'},\n",
       "  'recentlyViewed': {'type': 'NotAsked'}},\n",
       " 'seo': {'type': 'NotAsked'},\n",
       " '@context': 'https://schema.org/',\n",
       " '@type': 'Product',\n",
       " 'name': 'MacBook Air A1466/A1369 & A1465/A1370 Order LCD',\n",
       " 'image': ['https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/d75e6d44-d156-40d8-8f26-13aaac6979b4/1200/800/fitted.jpg',\n",
       "  'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/ee70968b-bb0c-45ff-8959-9eeaa50f3aa5/1200/800/fitted.jpg',\n",
       "  'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/450ef6eb-1493-4e8e-85f3-1d817200c537/1200/800/fitted.jpg',\n",
       "  'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/e031ccc0-cfbb-4311-ab69-7a1de57a0c5d/1200/800/fitted.jpg',\n",
       "  'https://i.bikroy-st.com/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka/56f7a0e3-9a63-41ee-bcdc-0f1155b1ab58/1200/800/fitted.jpg'],\n",
       " 'description': 'Address ;nBranch 1 : Daymon Computer bd nShop : 901nLevel : 9nMaltiplan center, New Elephant Road, DhakanBranch 2 : Dapple nHouse : 32nLevel : 6nRoad : 7nBlock ',\n",
       " 'offers': {'@type': 'Offer',\n",
       "  'url': 'https://bikroy.com/en/ad/macbook-air-a1466-a1369-and-a1465-a1370-order-lcd-for-sale-dhaka',\n",
       "  'priceCurrency': 'BDT',\n",
       "  'price': '15000',\n",
       "  'itemCondition': 'https://schema.org/NewCondition',\n",
       "  'availability': 'https://schema.org/InStock',\n",
       "  'seller': {'@type': 'Organization', 'name': 'Daymon Computer(BD)'}}}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jtext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>features</th>\n",
       "      <th>is_member</th>\n",
       "      <th>is_auth_dealer</th>\n",
       "      <th>is_featured_member</th>\n",
       "      <th>membership_level</th>\n",
       "      <th>is_verified</th>\n",
       "      <th>door_step_delivery</th>\n",
       "      <th>dsd_applicable</th>\n",
       "      <th>...</th>\n",
       "      <th>contact_name</th>\n",
       "      <th>phone_number</th>\n",
       "      <th>shop_name</th>\n",
       "      <th>shop_email</th>\n",
       "      <th>price</th>\n",
       "      <th>delivery_free</th>\n",
       "      <th>similar_ads</th>\n",
       "      <th>image_url</th>\n",
       "      <th>image_path</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OnePlus 9 Pro 12GB 256GB global (New)</td>\n",
       "      <td>@@ new and pack❤️😍nn14 days Replacement Guaran...</td>\n",
       "      <td>{'Condition': 'New', 'Brand': 'OnePlus', 'Mode...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>plus</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>Marvel</td>\n",
       "      <td>[01884002525]</td>\n",
       "      <td>Marvel</td>\n",
       "      <td>marvelbd422@gmail.com</td>\n",
       "      <td>Tk 63,990</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'id': '62249d02981cc0007af806c4', 'slug': 'o...</td>\n",
       "      <td>[https://i.bikroy-st.com/oneplus-9-pro-12gb-25...</td>\n",
       "      <td>[oneplus-9-pro-12gb-256gb-global-new-for-sale-...</td>\n",
       "      <td>https://bikroy.com/en/ad/oneplus-9-pro-12gb-25...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   title  \\\n",
       "0  OnePlus 9 Pro 12GB 256GB global (New)   \n",
       "\n",
       "                                         description  \\\n",
       "0  @@ new and pack❤️😍nn14 days Replacement Guaran...   \n",
       "\n",
       "                                            features  is_member  \\\n",
       "0  {'Condition': 'New', 'Brand': 'OnePlus', 'Mode...       True   \n",
       "\n",
       "   is_auth_dealer  is_featured_member membership_level  is_verified  \\\n",
       "0           False               False             plus        False   \n",
       "\n",
       "   door_step_delivery  dsd_applicable  ... contact_name   phone_number  \\\n",
       "0               False            True  ...       Marvel  [01884002525]   \n",
       "\n",
       "  shop_name             shop_email      price delivery_free  \\\n",
       "0    Marvel  marvelbd422@gmail.com  Tk 63,990         False   \n",
       "\n",
       "                                         similar_ads  \\\n",
       "0  [{'id': '62249d02981cc0007af806c4', 'slug': 'o...   \n",
       "\n",
       "                                           image_url  \\\n",
       "0  [https://i.bikroy-st.com/oneplus-9-pro-12gb-25...   \n",
       "\n",
       "                                          image_path  \\\n",
       "0  [oneplus-9-pro-12gb-256gb-global-new-for-sale-...   \n",
       "\n",
       "                                                 url  \n",
       "0  https://bikroy.com/en/ad/oneplus-9-pro-12gb-25...  \n",
       "\n",
       "[1 rows x 26 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(all_data)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['https://i.bikroy-st.com/oneplus-9-pro-12gb-256gb-global-new-for-sale-chattogram/60e710cc-3497-417c-8f40-42630b9e4d97/1200/800/fitted.jpg']\n"
     ]
    }
   ],
   "source": [
    "for d in df['image_url']:\n",
    "    print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://bikroy.com/en/ad/symphony-b6-used-for-sale-dhaka-595\n"
     ]
    }
   ],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Location</th>\n",
       "      <th>Price</th>\n",
       "      <th>Images</th>\n",
       "      <th>Meta Data</th>\n",
       "      <th>Feature</th>\n",
       "      <th>Description</th>\n",
       "      <th>URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Realme C11 (Used)</td>\n",
       "      <td>Posted on 07 Mar 11:15 pm, Maulvi Bazar, Sylhe...</td>\n",
       "      <td>Tk 6,000</td>\n",
       "      <td>[realme-c11-used-for-sale-sylhet-division-12@1...</td>\n",
       "      <td>{'Condition': 'Used', 'Brand': 'Realme', 'Mode...</td>\n",
       "      <td>N/A</td>\n",
       "      <td>N/A</td>\n",
       "      <td>https://bikroy.com/en/ad/oneplus-9-pro-12gb-25...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Samsung Galaxy A7 . (Used)</td>\n",
       "      <td>Posted on 07 Mar 11:15 pm, Dhanmondi, Dhaka</td>\n",
       "      <td>Tk 9,000</td>\n",
       "      <td>[samsung-galaxy-a7-2018-used-for-sale-dhaka-11...</td>\n",
       "      <td>{'Condition': 'Used', 'Brand': 'Samsung', 'Mod...</td>\n",
       "      <td>Bluetooth, Camera, Dual SIM, Expandable Memory...</td>\n",
       "      <td>N/A</td>\n",
       "      <td>https://bikroy.com/en/ad/oppo-a57-3-32-new-for...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Title  \\\n",
       "0           Realme C11 (Used)   \n",
       "1  Samsung Galaxy A7 . (Used)   \n",
       "\n",
       "                                            Location     Price  \\\n",
       "0  Posted on 07 Mar 11:15 pm, Maulvi Bazar, Sylhe...  Tk 6,000   \n",
       "1        Posted on 07 Mar 11:15 pm, Dhanmondi, Dhaka  Tk 9,000   \n",
       "\n",
       "                                              Images  \\\n",
       "0  [realme-c11-used-for-sale-sylhet-division-12@1...   \n",
       "1  [samsung-galaxy-a7-2018-used-for-sale-dhaka-11...   \n",
       "\n",
       "                                           Meta Data  \\\n",
       "0  {'Condition': 'Used', 'Brand': 'Realme', 'Mode...   \n",
       "1  {'Condition': 'Used', 'Brand': 'Samsung', 'Mod...   \n",
       "\n",
       "                                             Feature Description  \\\n",
       "0                                                N/A         N/A   \n",
       "1  Bluetooth, Camera, Dual SIM, Expandable Memory...         N/A   \n",
       "\n",
       "                                                 URL  \n",
       "0  https://bikroy.com/en/ad/oneplus-9-pro-12gb-25...  \n",
       "1  https://bikroy.com/en/ad/oppo-a57-3-32-new-for...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_lst = [titles, locations, prices, img_lst, meta_lst,  feature_lst, desc_lst, single_url[:2]]\n",
    "column_lst = ['Title', 'Location', 'Price', 'Images', 'Meta Data', 'Feature', 'Description', 'URL']\n",
    "df = pd.DataFrame (df_lst).T\n",
    "df.columns = column_lst\n",
    "df.to_csv(f'csv/{category}.csv', index=False)\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Realme C11 (Used)', 'Samsung Galaxy A7 . (Used)']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}