location_based_groupings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import MongoConnectionManager, get_redis_instance\n",
    "from dotenv import load_dotenv\n",
    "import jsonlines\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def retrieve_segment_data(bounding_box, view_name, fields):\n",
    "    database_name = os.environ.get('DATABASE')\n",
    "    collection_name = f'{view_name}_collection'\n",
    "    \n",
    "    latitudes = [coordinate[1] for coordinate in bounding_box]\n",
    "    longitudes = [coordinate[0] for coordinate in bounding_box]\n",
    "    match_query = {\n",
    "        'LAT': {'$gte': min(latitudes), '$lte': max(latitudes)},\n",
    "        'LON': {'$gte': min(longitudes), '$lte': max(longitudes)},\n",
    "    }\n",
    "    \n",
    "    facet_query = {}\n",
    "    check_query = {'$or': []}\n",
    "    for field, field_type in fields.items():\n",
    "        field_name = field.upper()\n",
    "        if field_type == 'categorical':\n",
    "            facet_query[field] = [\n",
    "                {'$match': {field_name: {'$ne': None}}},\n",
    "                {'$group': {'_id': f'${field_name}', 'count': {'$sum': 1}}},\n",
    "                {'$group': {'_id': None, 'values': {'$push': {'k': '$_id', 'v': '$count'}}}},\n",
    "                {'$replaceRoot': {'newRoot': {'$arrayToObject': '$values'}}},\n",
    "            ]\n",
    "            check_query['$or'].append({field: {'$ne': []}})\n",
    "        else:\n",
    "            facet_query[field] = [\n",
    "                {'$match': {field_name: {'$ne': None}}},\n",
    "                {'$group': {'_id': None, 'max': {'$max': f'${field_name}'}, 'min': {'$min': f'${field_name}'}}},\n",
    "                {'$project': {'_id': 0, 'max': 1, 'min': 1}}\n",
    "            ]\n",
    "    \n",
    "    pipeline = [{'$match': match_query}, {'$facet': facet_query}, {'$match': check_query}]\n",
    "    with MongoConnectionManager(database_name, collection_name) as collection:\n",
    "        data = list(collection.aggregate(pipeline=pipeline, allowDiskUse=True))\n",
    "    \n",
    "    if data:\n",
    "        data = data[0]\n",
    "    else:\n",
    "        return {'nodata': 'no data in this grid'}\n",
    "    \n",
    "    properties = {}\n",
    "    for field in fields:\n",
    "        if data[field]:\n",
    "            properties[field] = data[field][0]\n",
    "        else:\n",
    "            properties[field] = {}\n",
    "    \n",
    "    document = {\n",
    "        'properties': properties,\n",
    "        'geometry': [{\n",
    "            'type': 'Polygon',\n",
    "            'coordinates': [bounding_box],\n",
    "        }]\n",
    "    }\n",
    "    \n",
    "    with jsonlines.open(f'{view_name}-grid-object-data.jsonl', 'a') as writer:\n",
    "        writer.write(document)\n",
    "    \n",
    "    writer_collection = f'{view_name}_object_clusters'\n",
    "    with MongoConnectionManager(database_name, writer_collection) as collection:\n",
    "        collection.insert_one(document)\n",
    "    \n",
    "    del document['_id']\n",
    "    return document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_filter_fields(section_name):\n",
    "    filter_key = f'{section_name}_skeleton'\n",
    "    redis_instance = get_redis_instance()\n",
    "    filters = json.loads(redis_instance.get(filter_key))['filters']\n",
    "    \n",
    "    fields = {}\n",
    "    for key, value in filters.items():\n",
    "        if value['input']:\n",
    "            if value['type'] == 'select':\n",
    "                fields[key] = 'categorical'\n",
    "            elif key not in {'lat', 'lon'}:\n",
    "                fields[key] = 'numeric'\n",
    "    \n",
    "    return fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_data(view_name):\n",
    "    with open('static/processed-small-grids.geojson', 'r') as reader:\n",
    "        data = json.loads(reader.read())\n",
    "    \n",
    "    processed = []\n",
    "    fields = get_filter_fields(view_name)\n",
    "    count = 0\n",
    "    \n",
    "    for feature in data['features']:\n",
    "        bbox = feature['geometry']['coordinates'][0]\n",
    "        doc = retrieve_segment_data(bbox, view_name, fields)\n",
    "        \n",
    "        if 'nodata' not in doc:\n",
    "            processed.append(doc)\n",
    "        count += 1\n",
    "        break\n",
    "    \n",
    "    processed_dump = json.dumps(processed)\n",
    "    with open(f'data/processed-{view_name}.json', 'w') as writer:\n",
    "        writer.write(processed_dump)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv()\n",
    "for view in os.environ.get('SECTIONS').split(','):\n",
    "    prepare_data(view)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.6 ('mongo-env': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "13d1778114034d34b564d3f7c2ae061006db3957e9fd1b7b2a9fbf02ccf80dfc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}