From 6456c12e3428d4f45bc550d4ed18e08a081e67b9 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Fri, 10 May 2024 18:35:03 +0530
Subject: [PATCH 01/59] feat: adding azure-connector

---
 object_store_connector/provider/azure.py | 228 +++++++++++++++++++++++
 1 file changed, 228 insertions(+)
 create mode 100644 object_store_connector/provider/azure.py

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
new file mode 100644
index 0000000..738f937
--- /dev/null
+++ b/object_store_connector/provider/azure.py
@@ -0,0 +1,228 @@
+
+from typing import Dict, List, Any
+from azure.storage.blob import ContainerClient, BlobClient, BlobServiceClient
+from pyspark.sql import DataFrame, SparkSession
+from provider.blob_provider import BlobProvider
+from models.object_info import ObjectInfo,Tag
+from pyspark.conf import SparkConf
+from obsrv.job.batch import get_base_conf
+from obsrv.connector import MetricsCollector
+from obsrv.common import ObsrvException
+from obsrv.models import ErrorData
+import json
+
+
+
+
+class AzureBlobStorage(BlobProvider):
+    def __init__(self, connector_config: str)-> None:
+        super().__init__()
+        self.config=connector_config
+        self.account_name = self.config["credentials"]["account_name"]
+        self.account_key = self.config["credentials"]["account_key"]
+        self.container_name = self.config["containername"]
+        self.blob_endpoint = self.config["blob_endpoint"]
+
+        self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
+
+    # def _get_spark_session(self):
+    #     spark = SparkSession.builder\
+    #         .appName("ObsrvAzureBlobStoreConnector")\
+    #         .config("spark.jars.packages", "org.apache.hadoop:hadoop-azure:3.3.1") \
+    #         .getOrCreate()
+    #     return spark
+    def get_spark_config(self, connector_config) -> SparkConf:
+        conf = get_base_conf()
+        conf.setAppName("ObsrvObjectStoreConnector")
+        conf.set("spark.jars.packages", "org.apache.hadoop:hadoop-azure:3.3.1")
+        conf.set("fs.azure.storage.accountAuthType", "SharedKey")
+        conf.set("fs.azure.storage.accountKey", connector_config["credentials"]["account_key"])
+        return conf
+
+    
+
+    def fetch_objects(self,metrics_collector: MetricsCollector) -> List[ObjectInfo]:
+        
+        try:
+            objects = self._list_blobs_in_container(metrics_collector=metrics_collector)
+            print("objects:",objects)
+            objects_info=[]
+            if objects==None:
+                raise Exception("No objects found")
+    
+            for obj in objects:
+                #print("in",obj['size'])
+                blob_client = BlobClient.from_connection_string(
+                    conn_str=self.connection_string, container_name=self.container_name, blob_name=obj
+                )
+                account_name = self.connection_string.split(";")[1].split("=")[-1]
+                blob_location = f"wasb://{self.container_name}@storageemulator/{obj}"
+                properties = blob_client.get_blob_properties()
+
+                object_info = ObjectInfo(
+                file_size_kb=properties.size,
+                location= blob_location,
+                format=obj.split(".")[-1],
+                tags= self.fetch_tags(obj, metrics_collector)
+            )
+        
+                objects_info.append(object_info.to_json())
+            print("--------",objects_info)
+        except Exception as e:
+            print("Exception: 1",e)
+            #exit()
+        return objects_info
+    
+    def read_object(self, object_path: str, sc: SparkSession, metrics_collector: MetricsCollector,file_format: str) -> DataFrame:
+        labels = [
+            {"key": "request_method", "value": "GET"},
+            {"key": "method_name", "value": "getObject"},
+            {"key": "object_path", "value": object_path}
+        ]
+        print("o------bj path----\n",object_path)
+        # file_format = self.connector_config.get("fileFormat", {}).get("type", "jsonl")
+        api_calls, errors, records_count = 0, 0, 0
+
+        try:
+            if file_format == "jsonl":
+                df = sc.read.format("json").load(object_path)
+            elif file_format == "json":
+                df = sc.read.format("json").option("multiLine", False).load(object_path)
+            elif file_format == "json.gz":
+                df = sc.read.format("json").option("compression", "gzip").option("multiLine", True).load(object_path)
+            elif file_format == "csv":
+                df = sc.read.format("csv").option("header", True).load(object_path)
+            elif file_format == "csv.gz":
+                df = sc.read.format("csv").option("header", True).option("compression", "gzip").load(object_path)
+            else:
+                raise ObsrvException(ErrorData("UNSUPPORTED_FILE_FORMAT", f"unsupported file format: {file_format}"))
+            records_count = df.count()
+            api_calls += 1
+
+            metrics_collector.collect({"num_api_calls": api_calls, "num_records": records_count}, addn_labels=labels)
+            print("\n")
+            print("dataframe",df)
+            return df
+        except Exception as e:
+             print(f"Failed to read data from Blob {e}")
+        
+
+
+    def _list_blobs_in_container(self,metrics_collector )->List :
+        try:
+            container = ContainerClient.from_connection_string(
+                conn_str=self.connection_string, container_name=self.container_name
+            )
+            if container.exists():
+                print("Container exists")
+            else:
+                raise Exception("Container does not exist")
+                
+
+            blob_list = container.list_blobs()
+            print(" ")
+            print("Listing blobs...\n")
+            blob_metadata = []
+            for blob in blob_list:
+                blob_metadata.append(blob.name)
+            print(blob_metadata, "\n")
+
+            for blob_name in blob_metadata:
+                self._get_properties(self.container_name, blob_name)
+            if blob_metadata==None:
+                raise Exception("NO objs")
+            
+            return blob_metadata
+        except Exception as ex:
+            print("Exception: 2", ex)
+            #exit()
+            
+        
+
+    
+
+    def _get_spark_session(self):
+        return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
+
+    def _get_properties(self, container_name, blob_name):
+        try:
+            blob_client = BlobClient.from_connection_string(
+                conn_str=self.connection_string, container_name=container_name, blob_name=blob_name
+            )
+            properties = blob_client.get_blob_properties()
+
+            account_name = self.connection_string.split(";")[1].split("=")[-1]
+            blob_location = f"wasb://{account_name}/{container_name}/{blob_name}"
+
+            print(f"Blob location: {blob_location}")
+            print(f"Blob name: {blob_name.split('.')[0]}")
+            print(f"File Type: {blob_name.split('.')[-1]}")
+            print(f"Container: {properties.container}")
+            print(f"Created Time:{properties.creation_time}")
+            print(f"Blob file_size: {properties.size}")
+            print(f"Blob metadata: {properties.metadata}")
+            print(f"Blob etag: {properties.etag}")
+            print(" ")
+
+        except Exception as ex:
+            print("Exception:3", ex)
+
+    def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> List[Tag]:
+        obj=object_path.split("//")[-1].split("/")[-1]
+        labels = [
+            {"key": "request_method", "value": "GET"},
+            {"key": "method_name", "value": "getObjectTagging"},
+            {"key": "object_path", "value": object_path}
+        ]
+
+        api_calls, errors = 0, 0
+        print("obj :",obj)
+        try:
+            blob_client = BlobClient.from_connection_string(
+                conn_str=self.connection_string, container_name=self.container_name, blob_name=obj
+            )
+            tags = blob_client.get_blob_tags()
+            print("Blob tags:")
+            
+            for k, v in tags.items():
+                print(k, v)
+            print("\n")  
+            #return tag_objects
+            
+        except (ValueError, IOError) as e:
+            print(f"Error retrieving tags: {e}")
+        except Exception as ex:
+            print("Exception:4", ex)
+        return [Tag(k,v) for k,v in tags.items()]
+    
+
+    def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCollector) -> bool:
+        labels = [
+            {"key": "request_method", "value": "PUT"},
+            {"key": "method_name", "value": "setObjectTagging"},
+            {"key": "object_path", "value": object.get('location')}
+        ]
+        initial_tags = object.get('tags')
+        new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
+        updated_tags = [Tag(tag.get('key'), tag.get('value')).to_azure() for tag in new_tags]
+        try:
+    
+
+            values = list(object.values())
+            obj = values[3].split('//')[-1].split('/')[-1]
+            
+            print("jsdjab",obj)
+            blob_client = BlobClient.from_connection_string(
+                conn_str=self.connection_string, container_name=self.container_name, blob_name=obj
+            )
+            existing_tags = blob_client.get_blob_tags() or {}
+            existing_tags.update(tags)
+            blob_client.set_blob_tags(existing_tags)
+            print("Blob tags updated!")
+        except (ValueError, IOError) as e:
+            print(f"Error setting tags: {e}")
+        except Exception as ex:
+            print("Exception:5", ex)
+        
+        print("updated tags",updated_tags)
+

From ec218c536ad59d50b5c86d3afe9daf0628890585 Mon Sep 17 00:00:00 2001
From: i-manaswiniee <143373040+i-manaswiniee@users.noreply.github.com>
Date: Fri, 10 May 2024 18:49:00 +0530
Subject: [PATCH 02/59] Initial Commit

---
 object_store_connector/provider/gcs.py | 389 +++++++++++++++++++++++++
 1 file changed, 389 insertions(+)
 create mode 100644 object_store_connector/provider/gcs.py

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
new file mode 100644
index 0000000..ba05c9a
--- /dev/null
+++ b/object_store_connector/provider/gcs.py
@@ -0,0 +1,389 @@
+from google.oauth2 import service_account
+from google.cloud import storage
+from pyspark.sql import DataFrame
+from typing import List
+from uuid import uuid4
+from pyspark.conf import SparkConf
+from models.object_info import Tag,ObjectInfo
+from pyspark.sql import SparkSession
+from obsrv.connector import MetricsCollector
+import json
+from obsrv.common import ObsrvException
+from obsrv.job.batch import get_base_conf
+from google.api_core.exceptions import ClientError
+from provider.blob_provider import BlobProvider
+
+class GCS(BlobProvider):
+    def __init__(self, connector_config) -> None:
+            self.key_path = "/tmp/key.json"
+            with open(self.key_path, "w") as f:
+                f.write(json.dumps(connector_config["credentials"]))
+            super().__init__()
+            self.connector_config=connector_config
+            self.bucket = connector_config['bucket']
+            self.credentials = connector_config['credentials']
+            self.prefix = connector_config.get('prefix', '/')
+            self.obj_prefix = f"gs://{self.bucket}/"
+            self.gcs_client = self._get_client()
+
+    def _get_client(self) -> storage.Client:
+        client = storage.Client(
+            credentials=service_account.Credentials.from_service_account_info({
+                "type": self.credentials["type"],
+                "project_id": self.credentials["project_id"],
+                "private_key_id": self.credentials["private_key_id"],
+                "private_key": self.credentials["private_key"],
+                "client_email": self.credentials["client_email"],
+                "client_id": self.credentials["client_id"],
+                "auth_uri": self.credentials["auth_uri"],
+                "token_uri": self.credentials["token_uri"],
+                "auth_provider_x509_cert_url": self.credentials["auth_provider_x509_cert_url"],
+                "client_x509_cert_url": self.credentials["client_x509_cert_url"]
+            })
+        )
+        try:
+            return client
+        except Exception as e:
+            print(f"Error creating GCS client: {str(e)}")
+
+    def get_spark_config(self,connector_config)->SparkConf:
+            conf = get_base_conf()
+            conf.setAppName("ObsrvObjectStoreConnector") 
+            conf.set("spark.jars", "/root/spark/jars/gcs-connector-hadoop3-2.2.22-shaded.jar") 
+            conf.set("spark.hadoop.google.cloud.auth.service.account.json.keyfile", self.key_path) 
+            conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "true") 
+            conf.set("spark.hadoop.fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem") 
+            conf.set("spark.hadoop.fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS") 
+            return conf
+
+
+    def fetch_tags(self, object_path: str) -> List[Tag]:
+        try:
+            bucket = self.gcs_client.bucket(self.bucket)
+            blob = bucket.blob(object_path)
+            if not blob.exists():
+                print(f"Object '{object_path}' not found in bucket '{self.bucket}'.")
+                return []
+            # Fetch metadata from the object
+            blob.reload()
+            metadata = blob.metadata
+            if metadata:
+                # return [Tag(tag['Key'], tag['Value']) for tag in tags]
+                metadata_list = [Tag(key, value) for key, value in metadata.items()]
+
+                print(f"Metadata fetched for object '{object_path}': {metadata_list}")
+                return metadata_list
+            else:
+                print(f"No metadata found for object '{object_path}'.")
+                return []
+        except Exception as e:
+            print(f"Error fetching metadata: {str(e)}")
+            return []
+
+
+    def update_tag(self, object: ObjectInfo, tags: List[Tag],metrics_collector:MetricsCollector) -> bool:
+        bucket=self.connector_config['bucket']
+        labels = [
+            {"key": "request_method", "value": "PUT"},
+            {"key": "method_name", "value": "setObjectTagging"},
+            {"key": "object_path", "value": object.get('location')}
+        ]
+        api_calls=0
+        try:
+            blob = self.gcs_client.bucket(object)
+            # Check if the object exists
+            if not blob.exists():
+                print(f"Object '{object}' not found in bucket '{bucket}'.")
+                return False
+            # Fetch existing metadata
+            blob.reload()
+            existing_metadata = blob.metadata
+            # Check if the key already exists in the existing metadata
+            for item in tags:
+                key = item["key"]
+                value = item["value"]
+                if key in existing_metadata:
+                    print(f"Key '{key}' already exists in existing metadata for object '{object}'.")
+                    return False
+                existing_metadata[key] = value
+            # Update metadata for the object
+            blob.metadata = existing_metadata
+            blob.patch()
+            print(f"Metadata updated for object '{object}'.")
+            return True
+        except Exception as e:
+            metrics_collector.collect("num_errors", addn_labels=labels)
+            print(f"Error updating metadata: {str(e)}")
+            metrics_collector.collect("num_api_calls",api_calls)
+            return False
+     
+
+  
+    def fetch_objects(self,metrics_collector: MetricsCollector) -> List[ObjectInfo]:
+        try:
+            objects = self._list_objects(metrics_collector=metrics_collector)
+            objects_info = []
+            if not objects:
+                print(f"No objects found")
+                return []
+            objects_info = []
+            for obj in objects:
+                # print(obj.name)
+                object_info = ObjectInfo(
+                    id=str(uuid4()),
+                    location=f"{self.obj_prefix}{obj.name}",  # Accessing blob's name
+                    # format=obj['name'].split(".")[-1],  # Extracting format from the name
+                    file_size_kb=obj.size // 1024,
+                    file_hash=obj.etag.strip('"')
+                    # tags=self.gcs_client.fetch_tags(obj['Key'], metrics_collector)
+                )
+            print(f"{self.obj_prefix}")
+            objects_info.append(object_info.to_json())
+            print(objects_info) 
+            return objects_info
+        except Exception as e:
+            print(f"Error fetching objects: {str(e)}")
+            return []
+
+
+
+    def read_object(self, object_path: str, sc: SparkSession,metrics_collector:MetricsCollector, file_format: str) -> DataFrame:
+        labels = [
+             {"key": "request_method", "value": "GET"},
+             {"key": "method_name", "value": "getObject"},
+             {"key": "object_path", "value": object_path}
+         ]
+
+        print("In read object", object_path)
+        
+        api_calls= 0
+        try:
+             # Determine file format based on the provided parameter
+             if file_format == "jsonl":
+                 df = sc.read.format("json").load(object_path)
+             elif file_format == "json":
+                 df = sc.read.format("json").option("multiLine", True).load(object_path)
+             elif file_format == "json.gz":
+                 df = sc.read.format("json").option("compression", "gzip").option("multiLine", True).load(object_path)
+             elif file_format == "csv":
+                 df = sc.read.format("csv").option("header", True).load(object_path)
+             elif file_format == "csv.gz":
+                 df = sc.read.format("csv").option("header", True).option("compression", "gzip").load(object_path)
+             else:
+               raise Exception(f"Unsupported file format for file: {file_format}")
+            #  records_count = df.count()
+             api_calls += 1
+             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
+             return df 
+        except Exception as exception:
+            metrics_collector.collect("num_errors", addn_labels=labels)
+            raise Exception(f"failed to read object from GCS: {str(exception)}")
+
+
+
+    
+    def _list_objects(self,metrics_collector:MetricsCollector):
+        """Lists all the blobs in the bucket."""
+        bucket_name = self.connector_config['bucket']
+        api_calls=0
+        summaries=[]
+        labels = [
+            {"key": "request_method", "value": "GET"},
+            {"key": "method_name", "value": "ListBlobs"}
+        ]
+        try:
+            # Note: Client.list_blobs requires at least package version 1.17.0.
+            objects = self.gcs_client.list_blobs(bucket_name)
+            # Note: The call returns a response only when the iterator is consumed.
+            for obj in objects:
+                # print(blob.name)
+                summaries.append(obj)
+        except ( ClientError) as exception:
+                # errors += 1
+                labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+                metrics_collector.collect("num_errors", addn_labels=labels)
+                raise Exception(f"failed to list objects in GCS: {str(exception)}")
+        metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
+        # print(summaries)
+        return summaries
+
+
+
+
+
+
+
+    # def _list_objects(self, metrics_collector: MetricsCollector) -> list:
+    #     bucket = self.connector_config['bucket']
+    #     summaries = []
+    #     # api_calls = 0
+    #     try:
+    #         print(f"Bucket: {bucket}, Prefix: {self.prefix}")
+    #         bucket_obj = self.gcs_client.bucket(bucket)
+    #         blobs = bucket_obj.list_blobs(prefix=self.prefix if self.prefix != '/' else '')
+    #         for page in blobs.pages:
+    #             # api_calls += 1
+    #             # Extract blob names from the current page
+    #             for blob in page:
+    #                 summaries.append(blob.name)
+    #     except Exception as e:
+    #         print(f"Error listing objects: {str(e)}")
+    #     # metrics_collector.collect("API calls made: ", api_calls)
+    #     return summaries
+
+
+
+# Usage example:
+# gcs = GCS(connector_config)
+# object_names = gcs._list_objects()
+# print("Object names:", object_names)
+
+
+
+    # def _list_objects(self, metrics_collector) -> list:
+    #     bucket_name = self.connector_config['bucket']
+    #     prefix = self.prefix
+    #     summaries = []
+    #     continuation_token = None
+
+    #     # metrics
+    #     api_calls, errors = 0, 0
+
+    #     labels = [
+    #         {"key": "request_method", "value": "GET"},
+    #         {"key": "method_name", "value": "ListBlobs"}
+    #     ]
+
+    #     while True:
+    #         try:
+    #             if continuation_token:
+    #                 objects = self.gcs_client.list_blobs(bucket_name)#, Prefix=prefix, ContinuationToken=continuation_token)
+    #             else:
+    #                 objects = self.gcs_client.list_blobs(bucket_name, prefix=self.prefix)
+    #             api_calls += 1
+    #             blob=next(objects)
+    #             summaries.append(blob.name)
+               
+    #             if not blob.get('IsTruncated'):
+    #                 break
+    #             continuation_token = blob.get('NextContinuationToken')
+    #         except (ClientError) as exception:
+    #             errors += 1
+    #             labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+    #             metrics_collector.collect("num_errors", errors, addn_labels=labels)
+    #             raise Exception( f"failed to list objects in GCS: {str(exception)}")
+
+    #     metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
+    #     return summaries
+
+    # def _list_objects(self, metrics_collector: MetricsCollector) -> list:
+    #     bucket = self.connector_config['bucket']
+    #     summaries = []
+    #     api_calls = 0
+    #     # prefix=self.prefix
+    #     labels = [
+    #         {"key": "request_method", "value": "GET"},
+    #         {"key": "method_name", "value": "ListBlobs"}
+    #     ]
+    #     try:
+    #         print(f"Bucket: {bucket}, Prefix: {self.prefix}")
+    #         bucket_obj = self.gcs_client.bucket(bucket)
+    #         blobs = bucket_obj.list_blobs(prefix=self.prefix if self.prefix != '/' else '')
+            
+    #         # if isinstance(blobs, list):  # To check if blob is a list
+    #         for blob in blobs:
+    #                 # print(f"Blob: {blob}")
+    #                 api_calls += 1
+    #                 summaries.append(blob)
+    #         # else:
+    #         #     for page in blobs.pages:
+    #         #         api_calls += 1
+    #         #         for blob in page:
+    #         #             summaries.append(blob.name)
+    #     except (ClientError) as e:
+    #         labels += [{"key": "error_code", "value": str(e.response['Error']['Code'])}]
+    #         print(f"Error listing objects: {str(e)}")
+    #         print(f"API calls made: {api_calls}")
+    #     metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
+    #     return summaries
+
+
+
+
+
+
+    def _get_spark_session(self):
+             return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
+
+
+# spark = gcs_connector._get_spark_session()
+# keyfile_path ="/root/Cloud/gcs/key.json"
+# jar_path ="/root/spark/jars/gcs-connector-hadoop3-2.2.22-shaded.jar"
+# # gcs_connector=GCS(connector_config)
+# # connector_config = "key.json"
+
+# spark = gcs_connector._get_spark_session()
+# # Define metadata to be updated
+# metadata_to_update = [
+#     {"key":"write","value":"true"}
+# ]
+
+
+# # Fetch custom metadata
+# metadata = gcs_connector.fetch_tags(gcs_connector.object_path)
+
+# # Update metadata for the specified object
+# success = gcs_connector.update_tag(gcs_connector.object_path, metadata_to_update)
+# if success:
+#     print("Metadata update :",success)
+# else:
+#     print("Metadata update :",success)
+
+# # Fetch objects from GCS
+# objects_info = gcs_connector.fetch_objects(gcs_connector.object_path)
+# print(objects_info)
+
+# # Read objects from GCS
+#  data_frame = gcs_connector.read_object(gcs_connector.object_path, spark)
+#  if data_frame:
+#      data_frame.show()
+
+# List objects from GCS bucket
+
+
+# # Connection to the PostgreSQL database
+# conn = psycopg2.connect(
+#     database="postgres",
+#     user="postgres",
+#     password="postgres",
+#     host="localhost",
+#     port=5432
+# )
+
+# # Create a cursor
+# cursor = conn.cursor()
+# # SQL query
+# cursor.execute(" SELECT connector_config FROM connector_instances WHERE id = 'gcs.new-york-taxi-data.1' ")
+# # Fetch the connector_config
+# connector_config= cursor.fetchone()[0]
+# # Close the cursor and connection
+# cursor.close()
+# conn.close()
+
+# print(connector_config)
+
+# # gcs_connector = GCS(connector_config)
+
+# # #Function Call
+# # objects_list, api_calls = gcs_connector._list_objects()
+# # print("Objects List:")
+# # for obj_name in objects_list:
+# #     print(obj_name)
+# # print("API Calls Made:", api_calls)
+
+#     # def _get_spark_session(self):
+#     #         return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
+# object_path="uploaded_file.json"
+# data_frame = gcs_connector.read_object(object_path, spark)
+# if data_frame:
+#       data_frame.show()

From 0274f0f5fed15915217fbcda16579effaa0c6d48 Mon Sep 17 00:00:00 2001
From: i-manaswiniee <143373040+i-manaswiniee@users.noreply.github.com>
Date: Fri, 10 May 2024 18:53:19 +0530
Subject: [PATCH 03/59] initial

---
 object_store_connector/provider/gcs.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index ba05c9a..da7947e 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -13,6 +13,8 @@
 from google.api_core.exceptions import ClientError
 from provider.blob_provider import BlobProvider
 
+
+
 class GCS(BlobProvider):
     def __init__(self, connector_config) -> None:
             self.key_path = "/tmp/key.json"

From 0ef519dd41199e776ae7126589180e02c256ce97 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Fri, 10 May 2024 19:22:30 +0530
Subject: [PATCH 04/59] Update gcs.py

---
 object_store_connector/provider/gcs.py | 177 -------------------------
 1 file changed, 177 deletions(-)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index da7947e..b7657c9 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -209,183 +209,6 @@ def _list_objects(self,metrics_collector:MetricsCollector):
         # print(summaries)
         return summaries
 
-
-
-
-
-
-
-    # def _list_objects(self, metrics_collector: MetricsCollector) -> list:
-    #     bucket = self.connector_config['bucket']
-    #     summaries = []
-    #     # api_calls = 0
-    #     try:
-    #         print(f"Bucket: {bucket}, Prefix: {self.prefix}")
-    #         bucket_obj = self.gcs_client.bucket(bucket)
-    #         blobs = bucket_obj.list_blobs(prefix=self.prefix if self.prefix != '/' else '')
-    #         for page in blobs.pages:
-    #             # api_calls += 1
-    #             # Extract blob names from the current page
-    #             for blob in page:
-    #                 summaries.append(blob.name)
-    #     except Exception as e:
-    #         print(f"Error listing objects: {str(e)}")
-    #     # metrics_collector.collect("API calls made: ", api_calls)
-    #     return summaries
-
-
-
-# Usage example:
-# gcs = GCS(connector_config)
-# object_names = gcs._list_objects()
-# print("Object names:", object_names)
-
-
-
-    # def _list_objects(self, metrics_collector) -> list:
-    #     bucket_name = self.connector_config['bucket']
-    #     prefix = self.prefix
-    #     summaries = []
-    #     continuation_token = None
-
-    #     # metrics
-    #     api_calls, errors = 0, 0
-
-    #     labels = [
-    #         {"key": "request_method", "value": "GET"},
-    #         {"key": "method_name", "value": "ListBlobs"}
-    #     ]
-
-    #     while True:
-    #         try:
-    #             if continuation_token:
-    #                 objects = self.gcs_client.list_blobs(bucket_name)#, Prefix=prefix, ContinuationToken=continuation_token)
-    #             else:
-    #                 objects = self.gcs_client.list_blobs(bucket_name, prefix=self.prefix)
-    #             api_calls += 1
-    #             blob=next(objects)
-    #             summaries.append(blob.name)
-               
-    #             if not blob.get('IsTruncated'):
-    #                 break
-    #             continuation_token = blob.get('NextContinuationToken')
-    #         except (ClientError) as exception:
-    #             errors += 1
-    #             labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
-    #             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-    #             raise Exception( f"failed to list objects in GCS: {str(exception)}")
-
-    #     metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
-    #     return summaries
-
-    # def _list_objects(self, metrics_collector: MetricsCollector) -> list:
-    #     bucket = self.connector_config['bucket']
-    #     summaries = []
-    #     api_calls = 0
-    #     # prefix=self.prefix
-    #     labels = [
-    #         {"key": "request_method", "value": "GET"},
-    #         {"key": "method_name", "value": "ListBlobs"}
-    #     ]
-    #     try:
-    #         print(f"Bucket: {bucket}, Prefix: {self.prefix}")
-    #         bucket_obj = self.gcs_client.bucket(bucket)
-    #         blobs = bucket_obj.list_blobs(prefix=self.prefix if self.prefix != '/' else '')
-            
-    #         # if isinstance(blobs, list):  # To check if blob is a list
-    #         for blob in blobs:
-    #                 # print(f"Blob: {blob}")
-    #                 api_calls += 1
-    #                 summaries.append(blob)
-    #         # else:
-    #         #     for page in blobs.pages:
-    #         #         api_calls += 1
-    #         #         for blob in page:
-    #         #             summaries.append(blob.name)
-    #     except (ClientError) as e:
-    #         labels += [{"key": "error_code", "value": str(e.response['Error']['Code'])}]
-    #         print(f"Error listing objects: {str(e)}")
-    #         print(f"API calls made: {api_calls}")
-    #     metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
-    #     return summaries
-
-
-
-
-
-
     def _get_spark_session(self):
              return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
 
-
-# spark = gcs_connector._get_spark_session()
-# keyfile_path ="/root/Cloud/gcs/key.json"
-# jar_path ="/root/spark/jars/gcs-connector-hadoop3-2.2.22-shaded.jar"
-# # gcs_connector=GCS(connector_config)
-# # connector_config = "key.json"
-
-# spark = gcs_connector._get_spark_session()
-# # Define metadata to be updated
-# metadata_to_update = [
-#     {"key":"write","value":"true"}
-# ]
-
-
-# # Fetch custom metadata
-# metadata = gcs_connector.fetch_tags(gcs_connector.object_path)
-
-# # Update metadata for the specified object
-# success = gcs_connector.update_tag(gcs_connector.object_path, metadata_to_update)
-# if success:
-#     print("Metadata update :",success)
-# else:
-#     print("Metadata update :",success)
-
-# # Fetch objects from GCS
-# objects_info = gcs_connector.fetch_objects(gcs_connector.object_path)
-# print(objects_info)
-
-# # Read objects from GCS
-#  data_frame = gcs_connector.read_object(gcs_connector.object_path, spark)
-#  if data_frame:
-#      data_frame.show()
-
-# List objects from GCS bucket
-
-
-# # Connection to the PostgreSQL database
-# conn = psycopg2.connect(
-#     database="postgres",
-#     user="postgres",
-#     password="postgres",
-#     host="localhost",
-#     port=5432
-# )
-
-# # Create a cursor
-# cursor = conn.cursor()
-# # SQL query
-# cursor.execute(" SELECT connector_config FROM connector_instances WHERE id = 'gcs.new-york-taxi-data.1' ")
-# # Fetch the connector_config
-# connector_config= cursor.fetchone()[0]
-# # Close the cursor and connection
-# cursor.close()
-# conn.close()
-
-# print(connector_config)
-
-# # gcs_connector = GCS(connector_config)
-
-# # #Function Call
-# # objects_list, api_calls = gcs_connector._list_objects()
-# # print("Objects List:")
-# # for obj_name in objects_list:
-# #     print(obj_name)
-# # print("API Calls Made:", api_calls)
-
-#     # def _get_spark_session(self):
-#     #         return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
-# object_path="uploaded_file.json"
-# data_frame = gcs_connector.read_object(object_path, spark)
-# if data_frame:
-#       data_frame.show()

From 3cfeba4a99b73d696618d95fef3020d3f8f97554 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Fri, 10 May 2024 19:23:08 +0530
Subject: [PATCH 05/59] Update gcs.py

---
 object_store_connector/provider/gcs.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index b7657c9..0489e61 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -13,8 +13,6 @@
 from google.api_core.exceptions import ClientError
 from provider.blob_provider import BlobProvider
 
-
-
 class GCS(BlobProvider):
     def __init__(self, connector_config) -> None:
             self.key_path = "/tmp/key.json"
@@ -120,7 +118,6 @@ def update_tag(self, object: ObjectInfo, tags: List[Tag],metrics_collector:Metri
             return False
      
 
-  
     def fetch_objects(self,metrics_collector: MetricsCollector) -> List[ObjectInfo]:
         try:
             objects = self._list_objects(metrics_collector=metrics_collector)
@@ -148,7 +145,6 @@ def fetch_objects(self,metrics_collector: MetricsCollector) -> List[ObjectInfo]:
             return []
 
 
-
     def read_object(self, object_path: str, sc: SparkSession,metrics_collector:MetricsCollector, file_format: str) -> DataFrame:
         labels = [
              {"key": "request_method", "value": "GET"},
@@ -181,8 +177,6 @@ def read_object(self, object_path: str, sc: SparkSession,metrics_collector:Metri
             metrics_collector.collect("num_errors", addn_labels=labels)
             raise Exception(f"failed to read object from GCS: {str(exception)}")
 
-
-
     
     def _list_objects(self,metrics_collector:MetricsCollector):
         """Lists all the blobs in the bucket."""

From 72fb940e466510a8869cfbb223ff89f22a047b62 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Tue, 14 May 2024 16:34:50 +0530
Subject: [PATCH 06/59] Update gcs.py

---
 object_store_connector/provider/gcs.py | 157 +++++++++++++------------
 1 file changed, 83 insertions(+), 74 deletions(-)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index 0489e61..6056b75 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -7,11 +7,10 @@
 from models.object_info import Tag,ObjectInfo
 from pyspark.sql import SparkSession
 from obsrv.connector import MetricsCollector
-import json
-from obsrv.common import ObsrvException
 from obsrv.job.batch import get_base_conf
 from google.api_core.exceptions import ClientError
 from provider.blob_provider import BlobProvider
+import json
 
 class GCS(BlobProvider):
     def __init__(self, connector_config) -> None:
@@ -56,105 +55,109 @@ def get_spark_config(self,connector_config)->SparkConf:
             conf.set("spark.hadoop.fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS") 
             return conf
 
-
-    def fetch_tags(self, object_path: str) -> List[Tag]:
+    
+    def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> List[Tag]:
+        labels = [
+            {"key": "request_method", "value": "GET"},
+            {"key": "method_name", "value": "getObjectTagging"},
+            {"key": "object_path", "value": object_path}
+        ]
+        api_calls, errors = 0, 0
         try:
+            relative_path = object_path.replace("gs://" + self.bucket + "/", "")
             bucket = self.gcs_client.bucket(self.bucket)
-            blob = bucket.blob(object_path)
-            if not blob.exists():
+            blob = bucket.get_blob(relative_path)
+            if blob is None:
                 print(f"Object '{object_path}' not found in bucket '{self.bucket}'.")
                 return []
+            api_calls += 1
             # Fetch metadata from the object
             blob.reload()
             metadata = blob.metadata
+            tags = []
             if metadata:
-                # return [Tag(tag['Key'], tag['Value']) for tag in tags]
-                metadata_list = [Tag(key, value) for key, value in metadata.items()]
-
-                print(f"Metadata fetched for object '{object_path}': {metadata_list}")
-                return metadata_list
+                for key, value in metadata.items():
+                    tags.append({'Key': key, 'Value': value})
+                metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
+                print(f"Tags:{tags}")
             else:
                 print(f"No metadata found for object '{object_path}'.")
                 return []
         except Exception as e:
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
             print(f"Error fetching metadata: {str(e)}")
             return []
+        return tags
 
-
-    def update_tag(self, object: ObjectInfo, tags: List[Tag],metrics_collector:MetricsCollector) -> bool:
-        bucket=self.connector_config['bucket']
+    
+    def update_tag(self, object: ObjectInfo, tags: List[Tag], metrics_collector: MetricsCollector) -> bool:
+        object_path = object.get('location')
+        print(object_path)
         labels = [
             {"key": "request_method", "value": "PUT"},
             {"key": "method_name", "value": "setObjectTagging"},
-            {"key": "object_path", "value": object.get('location')}
+            {"key": "object_path", "value": object_path}
         ]
-        api_calls=0
+        api_calls,errors= 0,0
         try:
-            blob = self.gcs_client.bucket(object)
-            # Check if the object exists
-            if not blob.exists():
-                print(f"Object '{object}' not found in bucket '{bucket}'.")
-                return False
-            # Fetch existing metadata
+            # Get the bucket
+            relative_path = object_path.replace("gs://" + self.bucket + "/", "")
+            bucket = self.gcs_client.bucket(self.bucket)
+            blob = bucket.get_blob(relative_path)
             blob.reload()
             existing_metadata = blob.metadata
-            # Check if the key already exists in the existing metadata
-            for item in tags:
-                key = item["key"]
-                value = item["value"]
+            print(f"Present:{existing_metadata}")
+            # Convert the list of Tag objects into a list of dictionaries
+            tags_dict = [{tag.key: tag.value} for tag in tags]
+            for item in tags_dict:
+                key = list(item.keys())[0]
+                value = item[key]
+                print(item)
                 if key in existing_metadata:
-                    print(f"Key '{key}' already exists in existing metadata for object '{object}'.")
+                    print(f"Key '{key}' already exists in existing metadata for object '.")
                     return False
                 existing_metadata[key] = value
             # Update metadata for the object
             blob.metadata = existing_metadata
             blob.patch()
-            print(f"Metadata updated for object '{object}'.")
+            metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
             return True
-        except Exception as e:
-            metrics_collector.collect("num_errors", addn_labels=labels)
+        except (ClientError) as exception:
+            errors+=1
+            labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+            metrics_collector.collect("num_errors",errors, addn_labels=labels)
             print(f"Error updating metadata: {str(e)}")
-            metrics_collector.collect("num_api_calls",api_calls)
-            return False
-     
-
-    def fetch_objects(self,metrics_collector: MetricsCollector) -> List[ObjectInfo]:
-        try:
-            objects = self._list_objects(metrics_collector=metrics_collector)
-            objects_info = []
-            if not objects:
-                print(f"No objects found")
-                return []
-            objects_info = []
-            for obj in objects:
-                # print(obj.name)
-                object_info = ObjectInfo(
-                    id=str(uuid4()),
-                    location=f"{self.obj_prefix}{obj.name}",  # Accessing blob's name
-                    # format=obj['name'].split(".")[-1],  # Extracting format from the name
-                    file_size_kb=obj.size // 1024,
-                    file_hash=obj.etag.strip('"')
-                    # tags=self.gcs_client.fetch_tags(obj['Key'], metrics_collector)
-                )
-            print(f"{self.obj_prefix}")
-            objects_info.append(object_info.to_json())
-            print(objects_info) 
-            return objects_info
-        except Exception as e:
-            print(f"Error fetching objects: {str(e)}")
+            raise Exception( f"failed to update tags in S3: {str(exception)}")
+            
+  
+    def fetch_objects(self, metrics_collector: MetricsCollector) -> List[ObjectInfo]:
+        objects = self._list_objects(metrics_collector=metrics_collector)
+        objects_info = []
+        if not objects:
+            print(f"No objects found")
             return []
+        for obj in objects:
+            object_info = ObjectInfo(
+                id=str(uuid4()),
+                location=f"{self.obj_prefix}{obj.name}",  # Accessing blob's name
+                format=(obj.name).split(".")[-1],  # Extracting format from the name
+                file_size_kb=obj.size // 1024,
+                file_hash=obj.etag.strip('"'),
+                tags=self.fetch_tags(f"{self.obj_prefix}{obj.name}", metrics_collector)
+            )
+            print(f"INFO:{object_info}")
+            # Append each object_info inside the loop
+            objects_info.append(object_info.to_json())
+        return objects_info
 
-
+    
     def read_object(self, object_path: str, sc: SparkSession,metrics_collector:MetricsCollector, file_format: str) -> DataFrame:
         labels = [
              {"key": "request_method", "value": "GET"},
              {"key": "method_name", "value": "getObject"},
              {"key": "object_path", "value": object_path}
          ]
-
-        print("In read object", object_path)
-        
-        api_calls= 0
+        api_calls,errors,records_count= 0,0,0
         try:
              # Determine file format based on the provided parameter
              if file_format == "jsonl":
@@ -169,24 +172,30 @@ def read_object(self, object_path: str, sc: SparkSession,metrics_collector:Metri
                  df = sc.read.format("csv").option("header", True).option("compression", "gzip").load(object_path)
              else:
                raise Exception(f"Unsupported file format for file: {file_format}")
-            #  records_count = df.count()
+             records_count = df.count()
              api_calls += 1
-             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
-             return df 
+             metrics_collector.collect({"num_api_calls": api_calls, "num_records": records_count}, addn_labels=labels)
+             return df
+        except (ClientError) as exception:
+            errors += 1
+            labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
+            raise Exception( f"failed to read object from S3: {str(exception)}")
         except Exception as exception:
-            metrics_collector.collect("num_errors", addn_labels=labels)
-            raise Exception(f"failed to read object from GCS: {str(exception)}")
+            errors += 1
+            labels += [{"key": "error_code", "value": "S3_READ_ERROR"}]
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
+            raise Exception( f"failed to read object from S3: {str(exception)}")
 
     
     def _list_objects(self,metrics_collector:MetricsCollector):
-        """Lists all the blobs in the bucket."""
         bucket_name = self.connector_config['bucket']
-        api_calls=0
         summaries=[]
         labels = [
             {"key": "request_method", "value": "GET"},
             {"key": "method_name", "value": "ListBlobs"}
         ]
+        api_calls,errors=0,0
         try:
             # Note: Client.list_blobs requires at least package version 1.17.0.
             objects = self.gcs_client.list_blobs(bucket_name)
@@ -195,14 +204,14 @@ def _list_objects(self,metrics_collector:MetricsCollector):
                 # print(blob.name)
                 summaries.append(obj)
         except ( ClientError) as exception:
-                # errors += 1
+                errors += 1
                 labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
-                metrics_collector.collect("num_errors", addn_labels=labels)
-                raise Exception(f"failed to list objects in GCS: {str(exception)}")
+                metrics_collector.collect("num_errors", errors, addn_labels=labels)
+                raise Exception (f"failed to list objects in GCS: {str(exception)}")
         metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
-        # print(summaries)
         return summaries
 
+    
     def _get_spark_session(self):
              return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
 

From 4de96f081b7c4a7abe71aee1c3aebb24274457c7 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Tue, 14 May 2024 16:54:15 +0530
Subject: [PATCH 07/59] feat: added the read form parquet format

---
 object_store_connector/config/config.yaml    |  2 +-
 object_store_connector/connector.py          | 28 +++++++++++---------
 object_store_connector/models/object_info.py |  2 +-
 object_store_connector/provider/azure.py     | 13 +++------
 4 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/object_store_connector/config/config.yaml b/object_store_connector/config/config.yaml
index 23d21df..6d786b9 100644
--- a/object_store_connector/config/config.yaml
+++ b/object_store_connector/config/config.yaml
@@ -15,7 +15,7 @@ kafka:
 
 obsrv_encryption_key: random_thirty_two_encrypt_string
 
-connector_instance_id: s3.new-york-taxi-data.1
+connector_instance_id: azure.new-york-taxi-data.1
 
 building-block: rv-test
 env: local
\ No newline at end of file
diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index 51a9d2f..c263e98 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -13,6 +13,7 @@
 from pyspark.sql.functions import lit
 
 from provider.s3 import S3
+from provider.azure import AzureBlobStorage
 from models.object_info import ObjectInfo
 
 class ObjectStoreConnector(ISourceConnector):
@@ -38,22 +39,25 @@ def get_spark_conf(self, connector_config) -> SparkConf:
     def _get_provider(self, connector_config: Dict[Any, Any]):
         if connector_config["type"] == "s3":
             self.provider = S3(connector_config)
+        elif connector_config["type"] == "azure_blob":
+            self.provider = AzureBlobStorage(connector_config)
+            print("---",connector_config)
         else:
             raise ObsrvException(ErrorData("INVALID_PROVIDER", "provider not supported: {}".format(connector_config["type"])))
 
     def _get_objects_to_process(self, ctx: ConnectorContext, metrics_collector: MetricsCollector) -> None:
-        objects = ctx.state.get_state("to_process", list())
-        if ctx.building_block is not None and ctx.env is not None:
-            self.dedupe_tag = "{}-{}".format(ctx.building_block, ctx.env)
-        else:
-            raise ObsrvException(ErrorData("INVALID_CONTEXT", "building_block or env not found in context"))
-
-        if not len(objects):
-            objects = self.provider.fetch_objects(metrics_collector)
-            objects = self._exclude_processed_objects(objects)
-            metrics_collector.collect("new_objects_discovered", len(objects))
-            ctx.state.put_state("to_process", objects)
-            ctx.state.save_state()
+        # objects = ctx.state.get_state("to_process", list())
+        # if ctx.building_block is not None and ctx.env is not None:
+        #     self.dedupe_tag = "{}-{}".format(ctx.building_block, ctx.env)
+        # else:
+        #     raise ObsrvException(ErrorData("INVALID_CONTEXT", "building_block or env not found in context"))
+
+        # if not len(objects):
+        objects = self.provider.fetch_objects(metrics_collector)
+        # objects = self._exclude_processed_objects(objects)
+        # metrics_collector.collect("new_objects_discovered", len(objects))
+        # ctx.state.put_state("to_process", objects)
+        # ctx.state.save_state()
 
         self.objects = objects
 
diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
index f5be427..ebb1e8f 100644
--- a/object_store_connector/models/object_info.py
+++ b/object_store_connector/models/object_info.py
@@ -14,7 +14,7 @@ def to_dict(self):
             'value': self.value
         }
 
-    def to_aws(self):
+    def to_azure(self):
         return {
             'Key': self.key,
             'Value': self.value
diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 738f937..fbbc854 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -1,6 +1,6 @@
 
 from typing import Dict, List, Any
-from azure.storage.blob import ContainerClient, BlobClient, BlobServiceClient
+from azure.storage.blob import ContainerClient, BlobClient
 from pyspark.sql import DataFrame, SparkSession
 from provider.blob_provider import BlobProvider
 from models.object_info import ObjectInfo,Tag
@@ -25,12 +25,6 @@ def __init__(self, connector_config: str)-> None:
 
         self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
 
-    # def _get_spark_session(self):
-    #     spark = SparkSession.builder\
-    #         .appName("ObsrvAzureBlobStoreConnector")\
-    #         .config("spark.jars.packages", "org.apache.hadoop:hadoop-azure:3.3.1") \
-    #         .getOrCreate()
-    #     return spark
     def get_spark_config(self, connector_config) -> SparkConf:
         conf = get_base_conf()
         conf.setAppName("ObsrvObjectStoreConnector")
@@ -92,6 +86,8 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
                 df = sc.read.format("json").option("compression", "gzip").option("multiLine", True).load(object_path)
             elif file_format == "csv":
                 df = sc.read.format("csv").option("header", True).load(object_path)
+            elif file_format == "parquet":
+                df = sc.read.parquet(object_path)
             elif file_format == "csv.gz":
                 df = sc.read.format("csv").option("header", True).option("compression", "gzip").load(object_path)
             else:
@@ -138,9 +134,6 @@ def _list_blobs_in_container(self,metrics_collector )->List :
             #exit()
             
         
-
-    
-
     def _get_spark_session(self):
         return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
 

From a11e1b050ab328e95bb21065ba8f168ab671afa5 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Tue, 14 May 2024 19:09:13 +0530
Subject: [PATCH 08/59] feat: structuring the azure code

---
 object_store_connector/provider/azure.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index fbbc854..8db6402 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -25,6 +25,7 @@ def __init__(self, connector_config: str)-> None:
 
         self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
 
+
     def get_spark_config(self, connector_config) -> SparkConf:
         conf = get_base_conf()
         conf.setAppName("ObsrvObjectStoreConnector")
@@ -34,7 +35,6 @@ def get_spark_config(self, connector_config) -> SparkConf:
         return conf
 
     
-
     def fetch_objects(self,metrics_collector: MetricsCollector) -> List[ObjectInfo]:
         
         try:
@@ -66,7 +66,8 @@ def fetch_objects(self,metrics_collector: MetricsCollector) -> List[ObjectInfo]:
             print("Exception: 1",e)
             #exit()
         return objects_info
-    
+
+
     def read_object(self, object_path: str, sc: SparkSession, metrics_collector: MetricsCollector,file_format: str) -> DataFrame:
         labels = [
             {"key": "request_method", "value": "GET"},
@@ -101,7 +102,6 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
             return df
         except Exception as e:
              print(f"Failed to read data from Blob {e}")
-        
 
 
     def _list_blobs_in_container(self,metrics_collector )->List :
@@ -114,7 +114,6 @@ def _list_blobs_in_container(self,metrics_collector )->List :
             else:
                 raise Exception("Container does not exist")
                 
-
             blob_list = container.list_blobs()
             print(" ")
             print("Listing blobs...\n")
@@ -160,6 +159,7 @@ def _get_properties(self, container_name, blob_name):
         except Exception as ex:
             print("Exception:3", ex)
 
+
     def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> List[Tag]:
         obj=object_path.split("//")[-1].split("/")[-1]
         labels = [
@@ -186,7 +186,7 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             print(f"Error retrieving tags: {e}")
         except Exception as ex:
             print("Exception:4", ex)
-        return [Tag(k,v) for k,v in tags.items()]
+        return [Tag(key,value) for key,value in tags.items()]
     
 
     def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCollector) -> bool:
@@ -199,8 +199,6 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
         new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
         updated_tags = [Tag(tag.get('key'), tag.get('value')).to_azure() for tag in new_tags]
         try:
-    
-
             values = list(object.values())
             obj = values[3].split('//')[-1].split('/')[-1]
             

From 2c044006906206eeed2f4841b727a043e5b1782c Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Wed, 15 May 2024 01:00:19 +0530
Subject: [PATCH 09/59] Update gcs.py

---
 object_store_connector/provider/gcs.py | 69 ++++++++++----------------
 1 file changed, 25 insertions(+), 44 deletions(-)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index 6056b75..7795189 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -52,14 +52,15 @@ def get_spark_config(self,connector_config)->SparkConf:
             conf.set("spark.hadoop.google.cloud.auth.service.account.json.keyfile", self.key_path) 
             conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "true") 
             conf.set("spark.hadoop.fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem") 
-            conf.set("spark.hadoop.fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS") 
+            conf.set("spark.hadoop.fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS")
+            conf.set("spark.sql.debug.maxToStringFields", 1000) 
             return conf
 
-    
+
     def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> List[Tag]:
         labels = [
             {"key": "request_method", "value": "GET"},
-            {"key": "method_name", "value": "getObjectTagging"},
+            {"key": "method_name", "value": "getObjectMetadata"},
             {"key": "object_path", "value": object_path}
         ]
         api_calls, errors = 0, 0
@@ -80,55 +81,40 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
                     tags.append({'Key': key, 'Value': value})
                 metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
                 print(f"Tags:{tags}")
-            else:
-                print(f"No metadata found for object '{object_path}'.")
-                return []
-        except Exception as e:
+        except Exception as exception:
+            errors += 1
+            labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            print(f"Error fetching metadata: {str(e)}")
-            return []
+            raise Exception(f"failed to fetch tags from S3: {str(exception)}")
         return tags
 
-    
+
     def update_tag(self, object: ObjectInfo, tags: List[Tag], metrics_collector: MetricsCollector) -> bool:
         object_path = object.get('location')
-        print(object_path)
         labels = [
             {"key": "request_method", "value": "PUT"},
-            {"key": "method_name", "value": "setObjectTagging"},
+            {"key": "method_name", "value": "updateObjectMetadata"},
             {"key": "object_path", "value": object_path}
         ]
-        api_calls,errors= 0,0
+        api_calls,errors=0,0
         try:
-            # Get the bucket
-            relative_path = object_path.replace("gs://" + self.bucket + "/", "")
-            bucket = self.gcs_client.bucket(self.bucket)
-            blob = bucket.get_blob(relative_path)
-            blob.reload()
-            existing_metadata = blob.metadata
-            print(f"Present:{existing_metadata}")
-            # Convert the list of Tag objects into a list of dictionaries
-            tags_dict = [{tag.key: tag.value} for tag in tags]
-            for item in tags_dict:
-                key = list(item.keys())[0]
-                value = item[key]
-                print(item)
-                if key in existing_metadata:
-                    print(f"Key '{key}' already exists in existing metadata for object '.")
-                    return False
-                existing_metadata[key] = value
-            # Update metadata for the object
-            blob.metadata = existing_metadata
-            blob.patch()
+            initial_tags = object.get('tags')
+            print(f"Initial:{initial_tags}")
+            new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
+            print(f"New tags:{new_tags}")
+            # Filter out tags with missing key or value
+            valid_tags = [tag for tag in new_tags if 'key' in tag and 'value' in tag]
+            updated_tags = [Tag(tag['key'], tag['value']).to_gcs() for tag in valid_tags]
+            print(f"Updated: {updated_tags}")
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
             return True
         except (ClientError) as exception:
-            errors+=1
+            errors += 1
             labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
-            metrics_collector.collect("num_errors",errors, addn_labels=labels)
-            print(f"Error updating metadata: {str(e)}")
-            raise Exception( f"failed to update tags in S3: {str(exception)}")
-            
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
+            print(f"Error updating metadata: {str(exception)}")
+            raise Exception(f"failed to update tags in GCS: {str(exception)}")
+
   
     def fetch_objects(self, metrics_collector: MetricsCollector) -> List[ObjectInfo]:
         objects = self._list_objects(metrics_collector=metrics_collector)
@@ -145,12 +131,10 @@ def fetch_objects(self, metrics_collector: MetricsCollector) -> List[ObjectInfo]
                 file_hash=obj.etag.strip('"'),
                 tags=self.fetch_tags(f"{self.obj_prefix}{obj.name}", metrics_collector)
             )
-            print(f"INFO:{object_info}")
-            # Append each object_info inside the loop
             objects_info.append(object_info.to_json())
         return objects_info
 
-    
+
     def read_object(self, object_path: str, sc: SparkSession,metrics_collector:MetricsCollector, file_format: str) -> DataFrame:
         labels = [
              {"key": "request_method", "value": "GET"},
@@ -199,9 +183,7 @@ def _list_objects(self,metrics_collector:MetricsCollector):
         try:
             # Note: Client.list_blobs requires at least package version 1.17.0.
             objects = self.gcs_client.list_blobs(bucket_name)
-            # Note: The call returns a response only when the iterator is consumed.
             for obj in objects:
-                # print(blob.name)
                 summaries.append(obj)
         except ( ClientError) as exception:
                 errors += 1
@@ -211,7 +193,6 @@ def _list_objects(self,metrics_collector:MetricsCollector):
         metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
         return summaries
 
-    
     def _get_spark_session(self):
              return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
 

From 38077628d284ebe069ebcefd96cc0a5161e54a04 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Wed, 15 May 2024 01:01:15 +0530
Subject: [PATCH 10/59] Update gcs.py

---
 object_store_connector/provider/gcs.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index 7795189..169cbc3 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -53,7 +53,6 @@ def get_spark_config(self,connector_config)->SparkConf:
             conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "true") 
             conf.set("spark.hadoop.fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem") 
             conf.set("spark.hadoop.fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS")
-            conf.set("spark.sql.debug.maxToStringFields", 1000) 
             return conf
 
 

From 97fdcd3c2bc3e4cc2c34b8eb344f1192fecd601e Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Wed, 15 May 2024 10:58:47 +0530
Subject: [PATCH 11/59] Update connector.py

---
 object_store_connector/connector.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index 51a9d2f..758f26d 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -38,6 +38,8 @@ def get_spark_conf(self, connector_config) -> SparkConf:
     def _get_provider(self, connector_config: Dict[Any, Any]):
         if connector_config["type"] == "s3":
             self.provider = S3(connector_config)
+        elif connector_config["type"] == "gcs":
+            self.provider = GCS(connector_config)
         else:
             raise ObsrvException(ErrorData("INVALID_PROVIDER", "provider not supported: {}".format(connector_config["type"])))
 
@@ -97,4 +99,4 @@ def _exclude_processed_objects(self, objects):
                 to_be_processed.append(obj)
 
         # return [objects[-1]] #TODO: Remove this line
-        return to_be_processed
\ No newline at end of file
+        return to_be_processed

From fd8ef6bc5749abca00ab68454f9b9873493d70e9 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Wed, 15 May 2024 10:59:13 +0530
Subject: [PATCH 12/59] Update object_info.py

---
 object_store_connector/models/object_info.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
index f5be427..3c34a60 100644
--- a/object_store_connector/models/object_info.py
+++ b/object_store_connector/models/object_info.py
@@ -19,6 +19,11 @@ def to_aws(self):
             'Key': self.key,
             'Value': self.value
         }
+    def to_aws(self):
+        return {
+            'Key': self.key,
+            'Value': self.value
+        }
 
 @dataclass
 class ObjectInfo:
@@ -51,4 +56,4 @@ def to_json(self):
             'file_hash': self.file_hash,
             'num_of_retries': self.num_of_retries,
             'tags': [tag.__dict__ for tag in self.tags]
-        }
\ No newline at end of file
+        }

From add758d50327cef50602a6e85be063de9695f537 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Wed, 15 May 2024 16:00:00 +0530
Subject: [PATCH 13/59] feat: updating the fetch and update tags

---
 object_store_connector/connector.py      | 26 ++++-----
 object_store_connector/provider/azure.py | 72 ++++++++++++++++++++----
 2 files changed, 74 insertions(+), 24 deletions(-)

diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index c263e98..f631045 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -41,23 +41,23 @@ def _get_provider(self, connector_config: Dict[Any, Any]):
             self.provider = S3(connector_config)
         elif connector_config["type"] == "azure_blob":
             self.provider = AzureBlobStorage(connector_config)
-            print("---",connector_config)
+            print("Connector_config",connector_config)
         else:
             raise ObsrvException(ErrorData("INVALID_PROVIDER", "provider not supported: {}".format(connector_config["type"])))
 
     def _get_objects_to_process(self, ctx: ConnectorContext, metrics_collector: MetricsCollector) -> None:
-        # objects = ctx.state.get_state("to_process", list())
-        # if ctx.building_block is not None and ctx.env is not None:
-        #     self.dedupe_tag = "{}-{}".format(ctx.building_block, ctx.env)
-        # else:
-        #     raise ObsrvException(ErrorData("INVALID_CONTEXT", "building_block or env not found in context"))
-
-        # if not len(objects):
-        objects = self.provider.fetch_objects(metrics_collector)
-        # objects = self._exclude_processed_objects(objects)
-        # metrics_collector.collect("new_objects_discovered", len(objects))
-        # ctx.state.put_state("to_process", objects)
-        # ctx.state.save_state()
+        objects = ctx.state.get_state("to_process", list())
+        if ctx.building_block is not None and ctx.env is not None:
+            self.dedupe_tag = "{}-{}".format(ctx.building_block, ctx.env)
+        else:
+            raise ObsrvException(ErrorData("INVALID_CONTEXT", "building_block or env not found in context"))
+
+        if not len(objects):
+            objects = self.provider.fetch_objects(metrics_collector)
+            objects = self._exclude_processed_objects(objects)
+            metrics_collector.collect("new_objects_discovered", len(objects))
+            ctx.state.put_state("to_process", objects)
+            ctx.state.save_state()
 
         self.objects = objects
 
diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 8db6402..67ddb03 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -1,6 +1,6 @@
 
 from typing import Dict, List, Any
-from azure.storage.blob import ContainerClient, BlobClient
+from azure.storage.blob import ContainerClient, BlobClient, BlobServiceClient
 from pyspark.sql import DataFrame, SparkSession
 from provider.blob_provider import BlobProvider
 from models.object_info import ObjectInfo,Tag
@@ -22,8 +22,11 @@ def __init__(self, connector_config: str)-> None:
         self.account_key = self.config["credentials"]["account_key"]
         self.container_name = self.config["containername"]
         self.blob_endpoint = self.config["blob_endpoint"]
+        self.prefix = self.config["prefix"]
+        
 
         self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
+        self.container_client = ContainerClient.from_connection_string(self.connection_string,self.container_name)
 
 
     def get_spark_config(self, connector_config) -> SparkConf:
@@ -105,10 +108,41 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
 
 
     def _list_blobs_in_container(self,metrics_collector )->List :
+        # container_name = self.config['container_name']
+        # prefix = self.prefix
+        # summaries = []
+        # marker = None
+
+        # # metrics
+        # api_calls, errors = 0, 0
+
+        # labels = [
+        #     {"key": "request_method", "value": "GET"},
+        #     {"key": "method_name", "value": "list_blobs"}
+        # ]
+
+        # while True:
+        #     try:
+        #         self.container_client.list_blobs
+        #         blobs = self.container_client.list_blobs(container_name, name_starts_with=prefix, marker=marker)
+        #         api_calls += 1
+        #         summaries.extend(blobs)
+        #         if not blobs.next_marker:
+        #             break
+        #         marker = blobs.next_marker
+        #     except Exception as exception:
+        #         errors += 1
+        #         labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+        #         metrics_collector.collect("num_errors", errors, addn_labels=labels)
+        #         raise ObsrvException(ErrorData('AZURE_BLOB_LIST_ERROR', f"Failed to list objects in Azure Blob Storage: {str(exception)}"))
+
+        # return summaries
+        
         try:
             container = ContainerClient.from_connection_string(
                 conn_str=self.connection_string, container_name=self.container_name
             )
+            
             if container.exists():
                 print("Container exists")
             else:
@@ -131,6 +165,7 @@ def _list_blobs_in_container(self,metrics_collector )->List :
         except Exception as ex:
             print("Exception: 2", ex)
             #exit()
+        
             
         
     def _get_spark_session(self):
@@ -161,7 +196,7 @@ def _get_properties(self, container_name, blob_name):
 
 
     def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> List[Tag]:
-        obj=object_path.split("//")[-1].split("/")[-1]
+        obj=object_path.split("//")[-1].split("/")[-1] #
         labels = [
             {"key": "request_method", "value": "GET"},
             {"key": "method_name", "value": "getObjectTagging"},
@@ -176,17 +211,22 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             )
             tags = blob_client.get_blob_tags()
             print("Blob tags:")
-            
+            api_calls += 1
+            metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
             for k, v in tags.items():
                 print(k, v)
             print("\n")  
-            #return tag_objects
+            return [Tag(key,value) for key,value in tags.items()]
             
-        except (ValueError, IOError) as e:
-            print(f"Error retrieving tags: {e}")
-        except Exception as ex:
-            print("Exception:4", ex)
-        return [Tag(key,value) for key,value in tags.items()]
+        # except (ValueError, IOError) as e:
+        #     print(f"Error retrieving tags: {e}")
+        except Exception as exception:
+            errors += 1
+            labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
+            raise ObsrvException(ErrorData("AZURE_TAG_READ_ERROR", f"failed to fetch tags from AZURE: {str(exception)}"))
+            #print("Exception:4", exception)
+        
     
 
     def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCollector) -> bool:
@@ -195,7 +235,10 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
             {"key": "method_name", "value": "setObjectTagging"},
             {"key": "object_path", "value": object.get('location')}
         ]
+        api_calls, errors = 0, 0
+
         initial_tags = object.get('tags')
+        print("::initials tags::\n",initial_tags)
         new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
         updated_tags = [Tag(tag.get('key'), tag.get('value')).to_azure() for tag in new_tags]
         try:
@@ -210,10 +253,17 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
             existing_tags.update(tags)
             blob_client.set_blob_tags(existing_tags)
             print("Blob tags updated!")
+            metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
+            return True
         except (ValueError, IOError) as e:
             print(f"Error setting tags: {e}")
-        except Exception as ex:
-            print("Exception:5", ex)
+        except Exception as exception:
+            errors += 1
+            labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
+            raise ObsrvException(ErrorData("AZURE_TAG_UPDATE_ERROR", f"failed to update tags in AZURE: {str(exception)}"))
+
+            #print("Exception:5", ex)
         
         print("updated tags",updated_tags)
 

From b42dc6d31878d621091ce5c1ff28897c345052fb Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Wed, 15 May 2024 18:04:07 +0530
Subject: [PATCH 14/59] obsrv-labs/issue-tracker#6: feat: changes made in
 config.yaml and added the paramenters in the fetch_tags

---
 object_store_connector/config/config.yaml |   5 +-
 object_store_connector/connector.py       |   2 +-
 object_store_connector/provider/azure.py  |  16 +-
 poetry.lock                               | 424 +++++-----------------
 pyproject.toml                            |   2 +-
 5 files changed, 111 insertions(+), 338 deletions(-)

diff --git a/object_store_connector/config/config.yaml b/object_store_connector/config/config.yaml
index 6d786b9..d09d89b 100644
--- a/object_store_connector/config/config.yaml
+++ b/object_store_connector/config/config.yaml
@@ -6,15 +6,14 @@ postgres:
   port: 5432
 
 kafka:
-  bootstrap-servers: localhost:9092
+  broker-servers: localhost:9092
   telemetry-topic: obsrv-connectors-telemetry
   connector-metrics-topic: obsrv-connectors-metrics
   producer:
     compression: snappy
     max-request-size: 1000000 # 1MB {1M: 1000000, 10M: 10000000, 5M: 5000000}
 
-obsrv_encryption_key: random_thirty_two_encrypt_string
-
+obsrv_encryption_key: strong_encryption_key_to_encrypt
 connector_instance_id: azure.new-york-taxi-data.1
 
 building-block: rv-test
diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index 40dfd26..e186cbb 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -74,7 +74,7 @@ def get_spark_conf(self, connector_config) -> SparkConf:
     def _get_provider(self, connector_config: Dict[Any, Any]):
         if connector_config["source"]["type"] == "s3":
             self.provider = S3(connector_config)
-        elif connector_config["type"] == "azure_blob":
+        elif connector_config["source"]["type"] == "azure_blob":
             self.provider = AzureBlobStorage(connector_config)
             print("Connector_config",connector_config)
         else:
diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 67ddb03..27bcbca 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -6,7 +6,7 @@
 from models.object_info import ObjectInfo,Tag
 from pyspark.conf import SparkConf
 from obsrv.job.batch import get_base_conf
-from obsrv.connector import MetricsCollector
+from obsrv.connector import ConnectorContext, MetricsCollector
 from obsrv.common import ObsrvException
 from obsrv.models import ErrorData
 import json
@@ -18,11 +18,11 @@ class AzureBlobStorage(BlobProvider):
     def __init__(self, connector_config: str)-> None:
         super().__init__()
         self.config=connector_config
-        self.account_name = self.config["credentials"]["account_name"]
-        self.account_key = self.config["credentials"]["account_key"]
-        self.container_name = self.config["containername"]
-        self.blob_endpoint = self.config["blob_endpoint"]
-        self.prefix = self.config["prefix"]
+        self.account_name = self.config["source"]["credentials"]["account_name"]
+        self.account_key = self.config["source"]["credentials"]["account_key"]
+        self.container_name = self.config["source"]["containername"]
+        self.blob_endpoint = self.config["source"]["blob_endpoint"]
+        self.prefix = self.config["source"]["prefix"]
         
 
         self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
@@ -34,11 +34,11 @@ def get_spark_config(self, connector_config) -> SparkConf:
         conf.setAppName("ObsrvObjectStoreConnector")
         conf.set("spark.jars.packages", "org.apache.hadoop:hadoop-azure:3.3.1")
         conf.set("fs.azure.storage.accountAuthType", "SharedKey")
-        conf.set("fs.azure.storage.accountKey", connector_config["credentials"]["account_key"])
+        conf.set("fs.azure.storage.accountKey", connector_config["source"]["credentials"]["account_key"])
         return conf
 
     
-    def fetch_objects(self,metrics_collector: MetricsCollector) -> List[ObjectInfo]:
+    def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollector) -> List[ObjectInfo]:
         
         try:
             objects = self._list_blobs_in_container(metrics_collector=metrics_collector)
diff --git a/poetry.lock b/poetry.lock
index fb2b630..74a84fd 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,18 +1,18 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
 
 [[package]]
 name = "boto3"
-version = "1.34.82"
+version = "1.34.105"
 description = "The AWS SDK for Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "boto3-1.34.82-py3-none-any.whl", hash = "sha256:6e0ee12e87b37fa81133e9308d0957fce4200c1ff37c96346538dba5e857da18"},
-    {file = "boto3-1.34.82.tar.gz", hash = "sha256:fcdb84936b04d5f78c8c8667b65bf5b9803cf39fd25bb7fe57ba237074e36171"},
+    {file = "boto3-1.34.105-py3-none-any.whl", hash = "sha256:b633e8fbf7145bdb995ce68a27d096bb89fd393185b0e773418d81cd78db5a03"},
+    {file = "boto3-1.34.105.tar.gz", hash = "sha256:f2c11635be0de7b7c06eb606ece1add125e02d6ed521592294a0a21af09af135"},
 ]
 
 [package.dependencies]
-botocore = ">=1.34.82,<1.35.0"
+botocore = ">=1.34.105,<1.35.0"
 jmespath = ">=0.7.1,<2.0.0"
 s3transfer = ">=0.10.0,<0.11.0"
 
@@ -21,13 +21,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
 
 [[package]]
 name = "botocore"
-version = "1.34.82"
+version = "1.34.105"
 description = "Low-level, data-driven core of boto 3."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "botocore-1.34.82-py3-none-any.whl", hash = "sha256:8f839e9a88e7ac7185e406be4cf9926673374e8a6ecc295302f56f7e3c618692"},
-    {file = "botocore-1.34.82.tar.gz", hash = "sha256:2fd14676152f9d64541099090cc64973fdf8232744256454de443583e35e497d"},
+    {file = "botocore-1.34.105-py3-none-any.whl", hash = "sha256:a459d060b541beecb50681e6e8a39313cca981e146a59ba7c5229d62f631a016"},
+    {file = "botocore-1.34.105.tar.gz", hash = "sha256:727d5d3e800ac8b705fca6e19b6fefa1e728a81d62a712df9bd32ed0117c740b"},
 ]
 
 [package.dependencies]
@@ -36,7 +36,7 @@ python-dateutil = ">=2.1,<3.0.0"
 urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}
 
 [package.extras]
-crt = ["awscrt (==0.19.19)"]
+crt = ["awscrt (==0.20.9)"]
 
 [[package]]
 name = "colorama"
@@ -51,68 +51,82 @@ files = [
 
 [[package]]
 name = "coverage"
-version = "7.5.0"
+version = "7.5.1"
 description = "Code coverage measurement for Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "coverage-7.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:432949a32c3e3f820af808db1833d6d1631664d53dd3ce487aa25d574e18ad1c"},
-    {file = "coverage-7.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2bd7065249703cbeb6d4ce679c734bef0ee69baa7bff9724361ada04a15b7e3b"},
-    {file = "coverage-7.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbfe6389c5522b99768a93d89aca52ef92310a96b99782973b9d11e80511f932"},
-    {file = "coverage-7.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39793731182c4be939b4be0cdecde074b833f6171313cf53481f869937129ed3"},
-    {file = "coverage-7.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85a5dbe1ba1bf38d6c63b6d2c42132d45cbee6d9f0c51b52c59aa4afba057517"},
-    {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:357754dcdfd811462a725e7501a9b4556388e8ecf66e79df6f4b988fa3d0b39a"},
-    {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a81eb64feded34f40c8986869a2f764f0fe2db58c0530d3a4afbcde50f314880"},
-    {file = "coverage-7.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:51431d0abbed3a868e967f8257c5faf283d41ec882f58413cf295a389bb22e58"},
-    {file = "coverage-7.5.0-cp310-cp310-win32.whl", hash = "sha256:f609ebcb0242d84b7adeee2b06c11a2ddaec5464d21888b2c8255f5fd6a98ae4"},
-    {file = "coverage-7.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:6782cd6216fab5a83216cc39f13ebe30adfac2fa72688c5a4d8d180cd52e8f6a"},
-    {file = "coverage-7.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e768d870801f68c74c2b669fc909839660180c366501d4cc4b87efd6b0eee375"},
-    {file = "coverage-7.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:84921b10aeb2dd453247fd10de22907984eaf80901b578a5cf0bb1e279a587cb"},
-    {file = "coverage-7.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:710c62b6e35a9a766b99b15cdc56d5aeda0914edae8bb467e9c355f75d14ee95"},
-    {file = "coverage-7.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c379cdd3efc0658e652a14112d51a7668f6bfca7445c5a10dee7eabecabba19d"},
-    {file = "coverage-7.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fea9d3ca80bcf17edb2c08a4704259dadac196fe5e9274067e7a20511fad1743"},
-    {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:41327143c5b1d715f5f98a397608f90ab9ebba606ae4e6f3389c2145410c52b1"},
-    {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:565b2e82d0968c977e0b0f7cbf25fd06d78d4856289abc79694c8edcce6eb2de"},
-    {file = "coverage-7.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cf3539007202ebfe03923128fedfdd245db5860a36810136ad95a564a2fdffff"},
-    {file = "coverage-7.5.0-cp311-cp311-win32.whl", hash = "sha256:bf0b4b8d9caa8d64df838e0f8dcf68fb570c5733b726d1494b87f3da85db3a2d"},
-    {file = "coverage-7.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:9c6384cc90e37cfb60435bbbe0488444e54b98700f727f16f64d8bfda0b84656"},
-    {file = "coverage-7.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fed7a72d54bd52f4aeb6c6e951f363903bd7d70bc1cad64dd1f087980d309ab9"},
-    {file = "coverage-7.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cbe6581fcff7c8e262eb574244f81f5faaea539e712a058e6707a9d272fe5b64"},
-    {file = "coverage-7.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad97ec0da94b378e593ef532b980c15e377df9b9608c7c6da3506953182398af"},
-    {file = "coverage-7.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd4bacd62aa2f1a1627352fe68885d6ee694bdaebb16038b6e680f2924a9b2cc"},
-    {file = "coverage-7.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adf032b6c105881f9d77fa17d9eebe0ad1f9bfb2ad25777811f97c5362aa07f2"},
-    {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ba01d9ba112b55bfa4b24808ec431197bb34f09f66f7cb4fd0258ff9d3711b1"},
-    {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f0bfe42523893c188e9616d853c47685e1c575fe25f737adf473d0405dcfa7eb"},
-    {file = "coverage-7.5.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a9a7ef30a1b02547c1b23fa9a5564f03c9982fc71eb2ecb7f98c96d7a0db5cf2"},
-    {file = "coverage-7.5.0-cp312-cp312-win32.whl", hash = "sha256:3c2b77f295edb9fcdb6a250f83e6481c679335ca7e6e4a955e4290350f2d22a4"},
-    {file = "coverage-7.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:427e1e627b0963ac02d7c8730ca6d935df10280d230508c0ba059505e9233475"},
-    {file = "coverage-7.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9dd88fce54abbdbf4c42fb1fea0e498973d07816f24c0e27a1ecaf91883ce69e"},
-    {file = "coverage-7.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a898c11dca8f8c97b467138004a30133974aacd572818c383596f8d5b2eb04a9"},
-    {file = "coverage-7.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07dfdd492d645eea1bd70fb1d6febdcf47db178b0d99161d8e4eed18e7f62fe7"},
-    {file = "coverage-7.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3d117890b6eee85887b1eed41eefe2e598ad6e40523d9f94c4c4b213258e4a4"},
-    {file = "coverage-7.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6afd2e84e7da40fe23ca588379f815fb6dbbb1b757c883935ed11647205111cb"},
-    {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a9960dd1891b2ddf13a7fe45339cd59ecee3abb6b8326d8b932d0c5da208104f"},
-    {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ced268e82af993d7801a9db2dbc1d2322e786c5dc76295d8e89473d46c6b84d4"},
-    {file = "coverage-7.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7c211f25777746d468d76f11719e64acb40eed410d81c26cefac641975beb88"},
-    {file = "coverage-7.5.0-cp38-cp38-win32.whl", hash = "sha256:262fffc1f6c1a26125d5d573e1ec379285a3723363f3bd9c83923c9593a2ac25"},
-    {file = "coverage-7.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:eed462b4541c540d63ab57b3fc69e7d8c84d5957668854ee4e408b50e92ce26a"},
-    {file = "coverage-7.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d0194d654e360b3e6cc9b774e83235bae6b9b2cac3be09040880bb0e8a88f4a1"},
-    {file = "coverage-7.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:33c020d3322662e74bc507fb11488773a96894aa82a622c35a5a28673c0c26f5"},
-    {file = "coverage-7.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbdf2cae14a06827bec50bd58e49249452d211d9caddd8bd80e35b53cb04631"},
-    {file = "coverage-7.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3235d7c781232e525b0761730e052388a01548bd7f67d0067a253887c6e8df46"},
-    {file = "coverage-7.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2de4e546f0ec4b2787d625e0b16b78e99c3e21bc1722b4977c0dddf11ca84e"},
-    {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4d0e206259b73af35c4ec1319fd04003776e11e859936658cb6ceffdeba0f5be"},
-    {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2055c4fb9a6ff624253d432aa471a37202cd8f458c033d6d989be4499aed037b"},
-    {file = "coverage-7.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:075299460948cd12722a970c7eae43d25d37989da682997687b34ae6b87c0ef0"},
-    {file = "coverage-7.5.0-cp39-cp39-win32.whl", hash = "sha256:280132aada3bc2f0fac939a5771db4fbb84f245cb35b94fae4994d4c1f80dae7"},
-    {file = "coverage-7.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:c58536f6892559e030e6924896a44098bc1290663ea12532c78cef71d0df8493"},
-    {file = "coverage-7.5.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:2b57780b51084d5223eee7b59f0d4911c31c16ee5aa12737c7a02455829ff067"},
-    {file = "coverage-7.5.0.tar.gz", hash = "sha256:cf62d17310f34084c59c01e027259076479128d11e4661bb6c9acb38c5e19bb8"},
+    {file = "coverage-7.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e"},
+    {file = "coverage-7.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f"},
+    {file = "coverage-7.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a"},
+    {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35"},
+    {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e"},
+    {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223"},
+    {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e"},
+    {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146"},
+    {file = "coverage-7.5.1-cp310-cp310-win32.whl", hash = "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228"},
+    {file = "coverage-7.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8"},
+    {file = "coverage-7.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428"},
+    {file = "coverage-7.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746"},
+    {file = "coverage-7.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3"},
+    {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2"},
+    {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca"},
+    {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8"},
+    {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057"},
+    {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987"},
+    {file = "coverage-7.5.1-cp311-cp311-win32.whl", hash = "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136"},
+    {file = "coverage-7.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd"},
+    {file = "coverage-7.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206"},
+    {file = "coverage-7.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34"},
+    {file = "coverage-7.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d"},
+    {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa"},
+    {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e"},
+    {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572"},
+    {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07"},
+    {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7"},
+    {file = "coverage-7.5.1-cp312-cp312-win32.whl", hash = "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19"},
+    {file = "coverage-7.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596"},
+    {file = "coverage-7.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7"},
+    {file = "coverage-7.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90"},
+    {file = "coverage-7.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e"},
+    {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5"},
+    {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661"},
+    {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8"},
+    {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4"},
+    {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d"},
+    {file = "coverage-7.5.1-cp38-cp38-win32.whl", hash = "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41"},
+    {file = "coverage-7.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de"},
+    {file = "coverage-7.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1"},
+    {file = "coverage-7.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece"},
+    {file = "coverage-7.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26"},
+    {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5"},
+    {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601"},
+    {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be"},
+    {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f"},
+    {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668"},
+    {file = "coverage-7.5.1-cp39-cp39-win32.whl", hash = "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981"},
+    {file = "coverage-7.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f"},
+    {file = "coverage-7.5.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312"},
+    {file = "coverage-7.5.1.tar.gz", hash = "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c"},
 ]
 
 [package.extras]
 toml = ["tomli"]
 
+[[package]]
+name = "exceptiongroup"
+version = "1.2.1"
+description = "Backport of PEP 654 (exception groups)"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"},
+    {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"},
+]
+
+[package.extras]
+test = ["pytest (>=6)"]
+
 [[package]]
 name = "iniconfig"
 version = "2.0.0"
@@ -135,46 +149,6 @@ files = [
     {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
 ]
 
-[[package]]
-name = "kafka-python-ng"
-version = "2.2.2"
-description = "Pure Python client for Apache Kafka"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "kafka-python-ng-2.2.2.tar.gz", hash = "sha256:87ad3a766e2c0bec71d9b99bdd9e9c5cda62d96cfda61a8ca16510484d6ad7d4"},
-    {file = "kafka_python_ng-2.2.2-py2.py3-none-any.whl", hash = "sha256:3fab1a03133fade1b6fd5367ff726d980e59031c4aaca9bf02c516840a4f8406"},
-]
-
-[package.extras]
-boto = ["botocore"]
-crc32c = ["crc32c"]
-lz4 = ["lz4"]
-snappy = ["python-snappy"]
-zstd = ["zstandard"]
-
-[[package]]
-name = "obsrv"
-version = "0.1.0"
-description = ""
-optional = false
-python-versions = "<4.0,>=3.10"
-files = [
-    {file = "obsrv-0.1.0-py3-none-any.whl", hash = "sha256:91ae7deec62fcb3ea4043bc9119f717405137d32f489cdf1daf5a3f5d05bf2d1"},
-    {file = "obsrv-0.1.0.tar.gz", hash = "sha256:d087fe8727be18117188a36e9bfbe173dc6a8aa07400944c99f225b4d73ecadb"},
-]
-
-[package.dependencies]
-coverage = ">=7.5.0,<8.0.0"
-kafka-python-ng = ">=2.2.2,<3.0.0"
-psycopg2-binary = ">=2.9.9,<3.0.0"
-pycryptodome = ">=3.20.0,<4.0.0"
-pyspark = {version = ">=3.5.1,<4.0.0", optional = true, markers = "extra == \"batch\""}
-pyyaml = ">=6.0.1,<7.0.0"
-
-[package.extras]
-batch = ["pyspark (>=3.5.1,<4.0.0)"]
-
 [[package]]
 name = "packaging"
 version = "24.0"
@@ -188,191 +162,40 @@ files = [
 
 [[package]]
 name = "pluggy"
-version = "1.4.0"
+version = "1.5.0"
 description = "plugin and hook calling mechanisms for python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"},
-    {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"},
+    {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
+    {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
 ]
 
 [package.extras]
 dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
-[[package]]
-name = "psycopg2-binary"
-version = "2.9.9"
-description = "psycopg2 - Python-PostgreSQL Database Adapter"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"},
-]
-
-[[package]]
-name = "py4j"
-version = "0.10.9.7"
-description = "Enables Python programs to dynamically access arbitrary Java objects"
-optional = false
-python-versions = "*"
-files = [
-    {file = "py4j-0.10.9.7-py2.py3-none-any.whl", hash = "sha256:85defdfd2b2376eb3abf5ca6474b51ab7e0de341c75a02f46dc9b5976f5a5c1b"},
-    {file = "py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb"},
-]
-
-[[package]]
-name = "pycryptodome"
-version = "3.20.0"
-description = "Cryptographic library for Python"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-files = [
-    {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"},
-    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"},
-    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"},
-    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"},
-    {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"},
-    {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"},
-    {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"},
-    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"},
-    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"},
-    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"},
-    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"},
-    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"},
-    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"},
-    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"},
-    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"},
-    {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"},
-]
-
-[[package]]
-name = "pyspark"
-version = "3.5.1"
-description = "Apache Spark Python API"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "pyspark-3.5.1.tar.gz", hash = "sha256:dd6569e547365eadc4f887bf57f153e4d582a68c4b490de475d55b9981664910"},
-]
-
-[package.dependencies]
-py4j = "0.10.9.7"
-
-[package.extras]
-connect = ["googleapis-common-protos (>=1.56.4)", "grpcio (>=1.56.0)", "grpcio-status (>=1.56.0)", "numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"]
-ml = ["numpy (>=1.15)"]
-mllib = ["numpy (>=1.15)"]
-pandas-on-spark = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"]
-sql = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"]
-
 [[package]]
 name = "pytest"
-version = "8.1.1"
+version = "8.2.0"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"},
-    {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"},
+    {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"},
+    {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"},
 ]
 
 [package.dependencies]
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
+exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
-pluggy = ">=1.4,<2.0"
+pluggy = ">=1.5,<2.0"
+tomli = {version = ">=1", markers = "python_version < \"3.11\""}
 
 [package.extras]
-testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
 
 [[package]]
 name = "python-dateutil"
@@ -388,66 +211,6 @@ files = [
 [package.dependencies]
 six = ">=1.5"
 
-[[package]]
-name = "pyyaml"
-version = "6.0.1"
-description = "YAML parser and emitter for Python"
-optional = false
-python-versions = ">=3.6"
-files = [
-    {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"},
-    {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
-    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
-    {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
-    {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
-    {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
-    {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
-    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
-    {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
-    {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
-    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"},
-    {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
-    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
-    {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
-    {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
-    {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
-    {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
-    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
-    {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
-    {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
-    {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
-]
-
 [[package]]
 name = "s3transfer"
 version = "0.10.1"
@@ -476,6 +239,17 @@ files = [
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 
+[[package]]
+name = "tomli"
+version = "2.0.1"
+description = "A lil' TOML parser"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
+    {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
+]
+
 [[package]]
 name = "urllib3"
 version = "2.2.1"
@@ -495,5 +269,5 @@ zstd = ["zstandard (>=0.18.0)"]
 
 [metadata]
 lock-version = "2.0"
-python-versions = "^3.12"
-content-hash = "b221af180d723448b4398e52b7b43ebac510f532d87dbe21de9b12668eb786ce"
+python-versions = "^3.10"
+content-hash = "435ba3dcfaa3224044b34dfbaa4e0b66931931b730fe0c4a029bb2dcf685b5e0"
diff --git a/pyproject.toml b/pyproject.toml
index ea167c9..e2713b8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,7 +6,7 @@ authors = ["Ravi Mula <ravismula@users.noreply.github.com>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.12"
+python = "^3.10"
 boto3 = "^1.34.82"
 coverage = "^7.5.0"
 

From 1ec0f989008d59599bf3e39d7ae50f4b2f0e8d4b Mon Sep 17 00:00:00 2001
From: i-manaswiniee <143373040+i-manaswiniee@users.noreply.github.com>
Date: Thu, 16 May 2024 18:01:33 +0530
Subject: [PATCH 15/59] obsrv-labs/issue-tracker#2: gcs connector enhancements

---
 object_store_connector/connector.py          |   3 +-
 object_store_connector/models/object_info.py |  13 ++-
 object_store_connector/provider/gcs.py       | 104 +++++++++++--------
 3 files changed, 72 insertions(+), 48 deletions(-)

diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index 1fc9339..f53f1d9 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -13,6 +13,7 @@
 from pyspark.conf import SparkConf
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import lit
+from provider.gcs import GCS
 
 logger = LoggerController(__name__)
 
@@ -71,7 +72,7 @@ def get_spark_conf(self, connector_config) -> SparkConf:
     def _get_provider(self, connector_config: Dict[Any, Any]):
         if connector_config["source"]["type"] == "s3":
             self.provider = S3(connector_config)
-        elif connector_config["type"] == "gcs":
+        elif connector_config["source"]["type"] == "gcs":
             self.provider = GCS(connector_config)
         else:
             ObsrvException(
diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
index bf9e576..78fb94d 100644
--- a/object_store_connector/models/object_info.py
+++ b/object_store_connector/models/object_info.py
@@ -10,14 +10,23 @@ class Tag:
     value: str
 
     def to_dict(self):
-        return {"key": self.key, "value": self.value}
+        return {
+            'key': self.key,
+            'value': self.value
+        }
 
     def to_aws(self):
         return {
             'Key': self.key,
             'Value': self.value
         }
-    def to_aws(self):
+    def to_gcs(self):
+        return {
+            'Key': self.key,
+            'Value': self.value
+        }
+
+    def to_gcs(self):
         return {
             'Key': self.key,
             'Value': self.value
diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index 169cbc3..03d7d58 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -6,7 +6,7 @@
 from pyspark.conf import SparkConf
 from models.object_info import Tag,ObjectInfo
 from pyspark.sql import SparkSession
-from obsrv.connector import MetricsCollector
+from obsrv.connector import MetricsCollector,ConnectorContext
 from obsrv.job.batch import get_base_conf
 from google.api_core.exceptions import ClientError
 from provider.blob_provider import BlobProvider
@@ -16,12 +16,17 @@ class GCS(BlobProvider):
     def __init__(self, connector_config) -> None:
             self.key_path = "/tmp/key.json"
             with open(self.key_path, "w") as f:
-                f.write(json.dumps(connector_config["credentials"]))
+                f.write(json.dumps(connector_config["source"]["credentials"]))
             super().__init__()
             self.connector_config=connector_config
-            self.bucket = connector_config['bucket']
-            self.credentials = connector_config['credentials']
-            self.prefix = connector_config.get('prefix', '/')
+            self.bucket = connector_config['source']['bucket']
+            self.credentials = connector_config['source']['credentials']
+            # self.prefix = connector_config.get('prefix', '/')
+            self.prefix = (
+            connector_config["source"]["prefix"]
+            if "prefix" in connector_config["source"]
+            else "/"
+        )
             self.obj_prefix = f"gs://{self.bucket}/"
             self.gcs_client = self._get_client()
 
@@ -64,59 +69,65 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
         ]
         api_calls, errors = 0, 0
         try:
-            relative_path = object_path.replace("gs://" + self.bucket + "/", "")
             bucket = self.gcs_client.bucket(self.bucket)
-            blob = bucket.get_blob(relative_path)
+            blob = bucket.get_blob(object_path)
             if blob is None:
                 print(f"Object '{object_path}' not found in bucket '{self.bucket}'.")
                 return []
             api_calls += 1
-            # Fetch metadata from the object
             blob.reload()
             metadata = blob.metadata
             tags = []
             if metadata:
                 for key, value in metadata.items():
-                    tags.append({'Key': key, 'Value': value})
+                    tag = Tag(key=key, value=value)
+                    tags.append(tag)
                 metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
-                print(f"Tags:{tags}")
+            return tags
         except Exception as exception:
             errors += 1
             labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            raise Exception(f"failed to fetch tags from S3: {str(exception)}")
-        return tags
-
+            raise Exception(f"failed to fetch tags from GCS: {str(exception)}")
+    
 
-    def update_tag(self, object: ObjectInfo, tags: List[Tag], metrics_collector: MetricsCollector) -> bool:
+    def update_tag(self, object: ObjectInfo, tags: list, metrics_collector: MetricsCollector) -> bool:
         object_path = object.get('location')
         labels = [
             {"key": "request_method", "value": "PUT"},
             {"key": "method_name", "value": "updateObjectMetadata"},
-            {"key": "object_path", "value": object_path}
+            {"key": "object_path", "value":  object.get('location')}
         ]
-        api_calls,errors=0,0
+        api_calls, errors = 0, 0
         try:
             initial_tags = object.get('tags')
-            print(f"Initial:{initial_tags}")
             new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
-            print(f"New tags:{new_tags}")
-            # Filter out tags with missing key or value
-            valid_tags = [tag for tag in new_tags if 'key' in tag and 'value' in tag]
-            updated_tags = [Tag(tag['key'], tag['value']).to_gcs() for tag in valid_tags]
-            print(f"Updated: {updated_tags}")
+            updated_tags = [tag for tag in new_tags if 'key' in tag and 'value' in tag]
+            relative_path=object_path.lstrip("gs://").split("/",1)[-1]
+            print(f"PATH:{relative_path}")
+            bucket = self.gcs_client.bucket(self.bucket)
+            blob = bucket.blob(relative_path)
+            if blob is None:
+                print(f"Object '{object_path}' not found in bucket '{self.bucket}'.")
+                return False
+            blob.metadata = {tag['key']: tag['value'] for tag in updated_tags}
+            blob.patch()
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
             return True
-        except (ClientError) as exception:
+        except ClientError as exception:
             errors += 1
-            labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+            labels += [
+                {"key": "error_code", "value": str(exception.response["Error"]["Code"])}
+            ]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            print(f"Error updating metadata: {str(exception)}")
-            raise Exception(f"failed to update tags in GCS: {str(exception)}")
+            Exception(
+                    "GCS_TAG_UPDATE_ERROR",
+                    f"failed to update tags in GCS for object: {str(exception)}",
+                )
+            return False
 
-  
-    def fetch_objects(self, metrics_collector: MetricsCollector) -> List[ObjectInfo]:
-        objects = self._list_objects(metrics_collector=metrics_collector)
+    def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollector) -> List[ObjectInfo]:
+        objects = self._list_objects(ctx,metrics_collector=metrics_collector)
         objects_info = []
         if not objects:
             print(f"No objects found")
@@ -124,25 +135,24 @@ def fetch_objects(self, metrics_collector: MetricsCollector) -> List[ObjectInfo]
         for obj in objects:
             object_info = ObjectInfo(
                 id=str(uuid4()),
-                location=f"{self.obj_prefix}{obj.name}",  # Accessing blob's name
-                format=(obj.name).split(".")[-1],  # Extracting format from the name
+                location=f"{self.obj_prefix}{obj.name}", 
+                format=(obj.name).split(".")[-1],  
                 file_size_kb=obj.size // 1024,
                 file_hash=obj.etag.strip('"'),
-                tags=self.fetch_tags(f"{self.obj_prefix}{obj.name}", metrics_collector)
+                tags=self.fetch_tags(obj.name,metrics_collector)
             )
             objects_info.append(object_info.to_json())
+        print(f"Object_Info:{objects_info}")
         return objects_info
 
-
     def read_object(self, object_path: str, sc: SparkSession,metrics_collector:MetricsCollector, file_format: str) -> DataFrame:
         labels = [
              {"key": "request_method", "value": "GET"},
-             {"key": "method_name", "value": "getObject"},
+             {"key": "method_name", "value": "getObjectMetadata"},
              {"key": "object_path", "value": object_path}
          ]
         api_calls,errors,records_count= 0,0,0
         try:
-             # Determine file format based on the provided parameter
              if file_format == "jsonl":
                  df = sc.read.format("json").load(object_path)
              elif file_format == "json":
@@ -163,28 +173,33 @@ def read_object(self, object_path: str, sc: SparkSession,metrics_collector:Metri
             errors += 1
             labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            raise Exception( f"failed to read object from S3: {str(exception)}")
+            raise Exception( f"failed to read object from GCS: {str(exception)}")
         except Exception as exception:
             errors += 1
-            labels += [{"key": "error_code", "value": "S3_READ_ERROR"}]
+            labels += [{"key": "error_code", "value": "GCS_READ_ERROR"}]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            raise Exception( f"failed to read object from S3: {str(exception)}")
+            raise Exception( f"failed to read object from GCS: {str(exception)}")
 
-    
-    def _list_objects(self,metrics_collector:MetricsCollector):
-        bucket_name = self.connector_config['bucket']
+    def _list_objects(self,ctx: ConnectorContext,metrics_collector:MetricsCollector):
+        bucket_name = self.connector_config['source']['bucket']
         summaries=[]
+        file_formats = {
+            "json": ["json", "json.gz", "json.zip"],
+            "jsonl": ["json", "json.gz", "json.zip"],
+            "csv": ["csv", "csv.gz", "csv.zip"],
+        }
+        file_format = ctx.data_format
         labels = [
             {"key": "request_method", "value": "GET"},
             {"key": "method_name", "value": "ListBlobs"}
         ]
         api_calls,errors=0,0
         try:
-            # Note: Client.list_blobs requires at least package version 1.17.0.
             objects = self.gcs_client.list_blobs(bucket_name)
             for obj in objects:
-                summaries.append(obj)
-        except ( ClientError) as exception:
+                if any(obj.name.endswith(f) for f in file_formats[file_format]):
+                    summaries.append(obj)
+        except ClientError as exception:
                 errors += 1
                 labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
                 metrics_collector.collect("num_errors", errors, addn_labels=labels)
@@ -194,4 +209,3 @@ def _list_objects(self,metrics_collector:MetricsCollector):
 
     def _get_spark_session(self):
              return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
-

From 69771d44b154227f627487861080824b0da49f87 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Thu, 16 May 2024 19:49:19 +0530
Subject: [PATCH 16/59] obsrv-labs/issue-tracker#6: lint and cleanup

---
 object_store_connector/connector.py      |  13 +-
 object_store_connector/provider/azure.py | 245 +++++++++++------------
 2 files changed, 119 insertions(+), 139 deletions(-)

diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index e186cbb..007e001 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -40,12 +40,12 @@ def process(
         connector_config: Dict[Any, Any],
         metrics_collector: MetricsCollector,
     ) -> Iterator[DataFrame]:
-        if (
-            ctx.state.get_state("status", default_value=self.not_running_state)
-            == self.running_state
-        ):
-            logger.info("Connector is already running. Skipping processing.")
-            return
+        # if (
+        #     ctx.state.get_state("status", default_value=self.not_running_state)
+        #     == self.running_state
+        # ):
+        #     logger.info("Connector is already running. Skipping processing.")
+        #     return
 
         ctx.state.put_state("status", self.running_state)
         ctx.state.save_state()
@@ -99,7 +99,6 @@ def _get_objects_to_process(
                     "INVALID_CONTEXT", "building_block or env not found in context"
                 )
             )
-
         if not len(objects):
             num_files_discovered = ctx.stats.get_stat("num_files_discovered", 0)
             objects = self.provider.fetch_objects(ctx, metrics_collector)
diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 27bcbca..c710c10 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -1,4 +1,4 @@
-
+from azure.core.exceptions import AzureError
 from typing import Dict, List, Any
 from azure.storage.blob import ContainerClient, BlobClient, BlobServiceClient
 from pyspark.sql import DataFrame, SparkSession
@@ -40,44 +40,33 @@ def get_spark_config(self, connector_config) -> SparkConf:
     
     def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollector) -> List[ObjectInfo]:
         
-        try:
-            objects = self._list_blobs_in_container(metrics_collector=metrics_collector)
-            print("objects:",objects)
-            objects_info=[]
-            if objects==None:
-                raise Exception("No objects found")
+        objects = self._list_blobs_in_container(ctx,metrics_collector=metrics_collector)
+        print("objects:",objects)
+        objects_info=[]
+        if objects==None:
+            raise Exception("No objects found")
     
-            for obj in objects:
-                #print("in",obj['size'])
-                blob_client = BlobClient.from_connection_string(
-                    conn_str=self.connection_string, container_name=self.container_name, blob_name=obj
+        for obj in objects:
+            blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
+            print("blob location:",blob_location)
+            object_info = ObjectInfo(
+                    location=blob_location,
+                    format=obj["name"].split(".")[-1],
+                    file_size_kb=obj["size"] // 1024,
+                    file_hash=obj["etag"].strip('"'),
+                    tags= self.fetch_tags(obj['name'], metrics_collector)
                 )
-                account_name = self.connection_string.split(";")[1].split("=")[-1]
-                blob_location = f"wasb://{self.container_name}@storageemulator/{obj}"
-                properties = blob_client.get_blob_properties()
-
-                object_info = ObjectInfo(
-                file_size_kb=properties.size,
-                location= blob_location,
-                format=obj.split(".")[-1],
-                tags= self.fetch_tags(obj, metrics_collector)
-            )
-        
-                objects_info.append(object_info.to_json())
-            print("--------",objects_info)
-        except Exception as e:
-            print("Exception: 1",e)
-            #exit()
+            objects_info.append(object_info.to_json())
         return objects_info
 
-
+    
     def read_object(self, object_path: str, sc: SparkSession, metrics_collector: MetricsCollector,file_format: str) -> DataFrame:
         labels = [
             {"key": "request_method", "value": "GET"},
             {"key": "method_name", "value": "getObject"},
             {"key": "object_path", "value": object_path}
         ]
-        print("o------bj path----\n",object_path)
+        
         # file_format = self.connector_config.get("fileFormat", {}).get("type", "jsonl")
         api_calls, errors, records_count = 0, 0, 0
 
@@ -100,114 +89,102 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
             api_calls += 1
 
             metrics_collector.collect({"num_api_calls": api_calls, "num_records": records_count}, addn_labels=labels)
-            print("\n")
-            print("dataframe",df)
             return df
-        except Exception as e:
-             print(f"Failed to read data from Blob {e}")
-
-
-    def _list_blobs_in_container(self,metrics_collector )->List :
-        # container_name = self.config['container_name']
-        # prefix = self.prefix
-        # summaries = []
-        # marker = None
-
-        # # metrics
-        # api_calls, errors = 0, 0
-
-        # labels = [
-        #     {"key": "request_method", "value": "GET"},
-        #     {"key": "method_name", "value": "list_blobs"}
-        # ]
-
-        # while True:
-        #     try:
-        #         self.container_client.list_blobs
-        #         blobs = self.container_client.list_blobs(container_name, name_starts_with=prefix, marker=marker)
-        #         api_calls += 1
-        #         summaries.extend(blobs)
-        #         if not blobs.next_marker:
-        #             break
-        #         marker = blobs.next_marker
-        #     except Exception as exception:
-        #         errors += 1
-        #         labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
-        #         metrics_collector.collect("num_errors", errors, addn_labels=labels)
-        #         raise ObsrvException(ErrorData('AZURE_BLOB_LIST_ERROR', f"Failed to list objects in Azure Blob Storage: {str(exception)}"))
-
-        # return summaries
-        
-        try:
-            container = ContainerClient.from_connection_string(
-                conn_str=self.connection_string, container_name=self.container_name
+        except AzureError as exception:
+            errors += 1
+            labels += [
+                {"key": "error_code", "value": str(exception.message["Error"]["Code"])}
+            ]
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
+            ObsrvException(
+                ErrorData(
+                    "AzureBlobStorage_READ_ERROR", f"failed to read object from AzureBlobStorage: {str(exception)}"
+                )
             )
-            
-            if container.exists():
-                print("Container exists")
-            else:
-                raise Exception("Container does not exist")
+            return None
+        except Exception as exception:
+            errors += 1
+            labels += [{"key": "error_code", "value": "AzureBlobStorage_READ_ERROR"}]
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
+            ObsrvException(
+                ErrorData(
+                    "AzureBlobStorage_READ_ERROR", f"failed to read object from AzureBlobStorage: {str(exception)}"
+                )
+            )
+            return None
+
+
+    def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> list:
+        container_name = self.config['source']['containername']
+        print("container_name\n",container_name)
+        summaries = []
+        continuation_token = None
+        formats = {
+            "json": ["json", "json.gz", "json.zip"],
+            "jsonl": ["json", "json.gz", "json.zip"],
+            "csv": ["csv", "csv.gz", "csv.zip"],
+        }
+        file_format = ctx.data_format
+        # metrics
+        api_calls, errors = 0, 0
+
+        labels = [
+            {"key": "request_method", "value": "GET"},
+            {"key": "method_name", "value": "list_blobs"},
+        ]
+        print("--------------=====================")
+        container_client = ContainerClient.from_connection_string(conn_str=self.connection_string, container_name=container_name)
+        while True:
+            try:      
+                if continuation_token:             
+                    blobs = container_client.list_blobs(results_per_page=1).by_page(continuation_token=continuation_token)
+                else:
+                    blobs= container_client.list_blobs()
+                api_calls += 1
+                
+                for blob in blobs:
+                    # if any(obj["Key"].endswith(f) for f in file_formats[file_format]):
+                    summaries.append(blob)
                 
-            blob_list = container.list_blobs()
-            print(" ")
-            print("Listing blobs...\n")
-            blob_metadata = []
-            for blob in blob_list:
-                blob_metadata.append(blob.name)
-            print(blob_metadata, "\n")
+                if not continuation_token:
+                    break
+                continuation_token = blobs.continuation_token
+            except AzureError as exception:
+                errors += 1
+                labels += [
+                    {
+                        "key": "error_code",
+                        "value": str(exception.message["Error"]["Code"]),
+                    }
+                ]
+                metrics_collector.collect("num_errors", errors, addn_labels=labels)
+                ObsrvException(
+                    ErrorData(
+                        "AZURE_BLOB_LIST_ERROR",
+                        f"failed to list objects in AzureBlobStorage: {str(exception)}",
+                    )
+                )
 
-            for blob_name in blob_metadata:
-                self._get_properties(self.container_name, blob_name)
-            if blob_metadata==None:
-                raise Exception("NO objs")
-            
-            return blob_metadata
-        except Exception as ex:
-            print("Exception: 2", ex)
-            #exit()
-        
+        metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
+        return summaries
             
         
     def _get_spark_session(self):
         return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
 
-    def _get_properties(self, container_name, blob_name):
-        try:
-            blob_client = BlobClient.from_connection_string(
-                conn_str=self.connection_string, container_name=container_name, blob_name=blob_name
-            )
-            properties = blob_client.get_blob_properties()
-
-            account_name = self.connection_string.split(";")[1].split("=")[-1]
-            blob_location = f"wasb://{account_name}/{container_name}/{blob_name}"
-
-            print(f"Blob location: {blob_location}")
-            print(f"Blob name: {blob_name.split('.')[0]}")
-            print(f"File Type: {blob_name.split('.')[-1]}")
-            print(f"Container: {properties.container}")
-            print(f"Created Time:{properties.creation_time}")
-            print(f"Blob file_size: {properties.size}")
-            print(f"Blob metadata: {properties.metadata}")
-            print(f"Blob etag: {properties.etag}")
-            print(" ")
-
-        except Exception as ex:
-            print("Exception:3", ex)
-
 
     def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> List[Tag]:
-        obj=object_path.split("//")[-1].split("/")[-1] #
+        
         labels = [
             {"key": "request_method", "value": "GET"},
-            {"key": "method_name", "value": "getObjectTagging"},
+            {"key": "method_name", "value": "get_blob_tags"},
             {"key": "object_path", "value": object_path}
         ]
 
         api_calls, errors = 0, 0
-        print("obj :",obj)
         try:
             blob_client = BlobClient.from_connection_string(
-                conn_str=self.connection_string, container_name=self.container_name, blob_name=obj
+                conn_str=self.connection_string, container_name=self.container_name, blob_name=object_path
             )
             tags = blob_client.get_blob_tags()
             print("Blob tags:")
@@ -217,35 +194,39 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
                 print(k, v)
             print("\n")  
             return [Tag(key,value) for key,value in tags.items()]
-            
-        # except (ValueError, IOError) as e:
-        #     print(f"Error retrieving tags: {e}")
-        except Exception as exception:
+
+        #Need to check on the exception.message    
+        except AzureError as exception:
             errors += 1
-            labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+            labels += [
+                {"key": "error_code", "value": str(exception.message["Error"]["Code"])}
+            ]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            raise ObsrvException(ErrorData("AZURE_TAG_READ_ERROR", f"failed to fetch tags from AZURE: {str(exception)}"))
-            #print("Exception:4", exception)
+            ObsrvException(
+                ErrorData(
+                    "AzureBlobStorage_TAG_READ_ERROR",
+                    f"failed to fetch tags from AzureBlobStorage: {str(exception)}",
+                )
+            )
+
         
     
 
     def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCollector) -> bool:
         labels = [
             {"key": "request_method", "value": "PUT"},
-            {"key": "method_name", "value": "setObjectTagging"},
+            {"key": "method_name", "value": "set_blob_tags"},
             {"key": "object_path", "value": object.get('location')}
         ]
         api_calls, errors = 0, 0
 
         initial_tags = object.get('tags')
-        print("::initials tags::\n",initial_tags)
         new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
         updated_tags = [Tag(tag.get('key'), tag.get('value')).to_azure() for tag in new_tags]
         try:
             values = list(object.values())
             obj = values[3].split('//')[-1].split('/')[-1]
             
-            print("jsdjab",obj)
             blob_client = BlobClient.from_connection_string(
                 conn_str=self.connection_string, container_name=self.container_name, blob_name=obj
             )
@@ -254,16 +235,16 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
             blob_client.set_blob_tags(existing_tags)
             print("Blob tags updated!")
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
+            print("updated tags",updated_tags)
             return True
         except (ValueError, IOError) as e:
             print(f"Error setting tags: {e}")
-        except Exception as exception:
+        except AzureError as exception:
             errors += 1
-            labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+            labels += [{"key": "error_code", "value": str(exception.message['Error']['Code'])}]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
             raise ObsrvException(ErrorData("AZURE_TAG_UPDATE_ERROR", f"failed to update tags in AZURE: {str(exception)}"))
 
-            #print("Exception:5", ex)
         
-        print("updated tags",updated_tags)
+        
 

From b4d2277e667ea07016a5c9374faab43e6f51f209 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Fri, 17 May 2024 12:49:28 +0530
Subject: [PATCH 17/59] obsrv-labs/issue-tracker#6: fix: Changes made in
 exception and update_tag

---
 object_store_connector/provider/azure.py | 43 +++++++++++++-----------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index c710c10..0ca5533 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -93,7 +93,7 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
         except AzureError as exception:
             errors += 1
             labels += [
-                {"key": "error_code", "value": str(exception.message["Error"]["Code"])}
+                {"key": "error_code", "value": str(exception.exc_msg)}
             ]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
             ObsrvException(
@@ -143,7 +143,7 @@ def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> l
                 api_calls += 1
                 
                 for blob in blobs:
-                    # if any(obj["Key"].endswith(f) for f in file_formats[file_format]):
+                    # if any(obj["name"].endswith(f) for f in file_formats[file_format]):
                     summaries.append(blob)
                 
                 if not continuation_token:
@@ -154,7 +154,7 @@ def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> l
                 labels += [
                     {
                         "key": "error_code",
-                        "value": str(exception.message["Error"]["Code"]),
+                        "value": str(exception.exc_msg),
                     }
                 ]
                 metrics_collector.collect("num_errors", errors, addn_labels=labels)
@@ -199,7 +199,7 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
         except AzureError as exception:
             errors += 1
             labels += [
-                {"key": "error_code", "value": str(exception.message["Error"]["Code"])}
+                {"key": "error_code", "value": str(exception.exc_msg)}
             ]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
             ObsrvException(
@@ -210,8 +210,6 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             )
 
         
-    
-
     def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCollector) -> bool:
         labels = [
             {"key": "request_method", "value": "PUT"},
@@ -220,31 +218,36 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
         ]
         api_calls, errors = 0, 0
 
-        initial_tags = object.get('tags')
-        new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
-        updated_tags = [Tag(tag.get('key'), tag.get('value')).to_azure() for tag in new_tags]
         try:
-            values = list(object.values())
-            obj = values[3].split('//')[-1].split('/')[-1]
-            
+            print("tags parameter\n",tags)
+            new_dict = {tag['key']: tag['value'] for tag in tags}
+        
+            stripped_file_path = object.get("location").lstrip("wasb://")
+            obj = stripped_file_path.split("/")[-1]
+ 
             blob_client = BlobClient.from_connection_string(
                 conn_str=self.connection_string, container_name=self.container_name, blob_name=obj
             )
             existing_tags = blob_client.get_blob_tags() or {}
-            existing_tags.update(tags)
+            existing_tags.update(new_dict)
+            print("existing tags",existing_tags)
             blob_client.set_blob_tags(existing_tags)
             print("Blob tags updated!")
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
-            print("updated tags",updated_tags)
+            print("updated tags",existing_tags)
             return True
-        except (ValueError, IOError) as e:
-            print(f"Error setting tags: {e}")
         except AzureError as exception:
             errors += 1
-            labels += [{"key": "error_code", "value": str(exception.message['Error']['Code'])}]
+            labels += [
+                {"key": "error_code", "value": str(exception.exc_msg)}
+            ]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            raise ObsrvException(ErrorData("AZURE_TAG_UPDATE_ERROR", f"failed to update tags in AZURE: {str(exception)}"))
-
-        
+            ObsrvException(
+                ErrorData(
+                    "AzureBlobStorage_TAG_UPDATE_ERROR",
+                    f"failed to update tags in AzureBlobStorage for object: {str(exception)}",
+                )
+            )
+            return False
         
 

From ccefd537e3dfe8cf967bff5717edb80cd04b66bc Mon Sep 17 00:00:00 2001
From: i-manaswiniee <143373040+i-manaswiniee@users.noreply.github.com>
Date: Fri, 17 May 2024 14:46:36 +0530
Subject: [PATCH 18/59] obsrv-labs/issue-tracker#2: gcs connector enhancements

---
 object_store_connector/provider/gcs.py | 46 ++++++++++++++++----------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index 03d7d58..3c0f3e1 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -135,16 +135,17 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
         for obj in objects:
             object_info = ObjectInfo(
                 id=str(uuid4()),
-                location=f"{self.obj_prefix}{obj.name}", 
-                format=(obj.name).split(".")[-1],  
-                file_size_kb=obj.size // 1024,
+                location=f"gs://{self.bucket}/{obj.name}",
+                format=obj.name.split(".")[-1],
+                file_size_kb=obj.size / 1024,
                 file_hash=obj.etag.strip('"'),
-                tags=self.fetch_tags(obj.name,metrics_collector)
+                tags=self.fetch_tags(obj.name, metrics_collector)
             )
             objects_info.append(object_info.to_json())
-        print(f"Object_Info:{objects_info}")
+        print("Object_Info:", objects_info)
         return objects_info
 
+
     def read_object(self, object_path: str, sc: SparkSession,metrics_collector:MetricsCollector, file_format: str) -> DataFrame:
         labels = [
              {"key": "request_method", "value": "GET"},
@@ -180,12 +181,12 @@ def read_object(self, object_path: str, sc: SparkSession,metrics_collector:Metri
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
             raise Exception( f"failed to read object from GCS: {str(exception)}")
 
-    def _list_objects(self,ctx: ConnectorContext,metrics_collector:MetricsCollector):
+    def _list_objects(self, ctx: ConnectorContext, metrics_collector: MetricsCollector) -> list:
         bucket_name = self.connector_config['source']['bucket']
-        summaries=[]
+        summaries = []
         file_formats = {
             "json": ["json", "json.gz", "json.zip"],
-            "jsonl": ["json", "json.gz", "json.zip"],
+            "jsonl": ["jsonl", "jsonl.gz", "jsonl.zip"],
             "csv": ["csv", "csv.gz", "csv.zip"],
         }
         file_format = ctx.data_format
@@ -193,17 +194,28 @@ def _list_objects(self,ctx: ConnectorContext,metrics_collector:MetricsCollector)
             {"key": "request_method", "value": "GET"},
             {"key": "method_name", "value": "ListBlobs"}
         ]
-        api_calls,errors=0,0
-        try:
-            objects = self.gcs_client.list_blobs(bucket_name)
-            for obj in objects:
-                if any(obj.name.endswith(f) for f in file_formats[file_format]):
-                    summaries.append(obj)
-        except ClientError as exception:
+        api_calls, errors = 0, 0
+        page_token = None
+        page_size = 1000# Set your desired page size here
+        while True:
+            try:
+                    bucket = self.gcs_client.bucket(bucket_name)
+                    if page_token:
+                        blobs_response = bucket.list_blobs(prefix=self.prefix, page_token=page_token, max_results=page_size)
+                    else:
+                        blobs_response = bucket.list_blobs(prefix=self.prefix, max_results=page_size)
+                    api_calls += 1
+                    for blob in blobs_response:
+                        if any(blob.name.endswith(f) for f in file_formats[file_format]):
+                            summaries.append(blob)
+                    if not blobs_response.next_page_token:
+                        break
+                    page_token = blobs_response.next_page_token
+            except ClientError as exception:
                 errors += 1
-                labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
+                labels.append({"key": "error_code", "value": str(exception.response['Error']['Code'])})
                 metrics_collector.collect("num_errors", errors, addn_labels=labels)
-                raise Exception (f"failed to list objects in GCS: {str(exception)}")
+                Exception(f"failed to list objects in GCS: {str(exception)}")
         metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
         return summaries
 

From 1d171cd0ee091ae1098dcdd0ba76362e2f2e3cc3 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Fri, 17 May 2024 15:43:51 +0530
Subject: [PATCH 19/59] obsrv-labs/issue-tracker#6:  feat: lint and cleanup

---
 object_store_connector/connector.py      |  2 +-
 object_store_connector/provider/azure.py | 26 ++++++++++++------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index 007e001..3d2feb4 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -76,7 +76,7 @@ def _get_provider(self, connector_config: Dict[Any, Any]):
             self.provider = S3(connector_config)
         elif connector_config["source"]["type"] == "azure_blob":
             self.provider = AzureBlobStorage(connector_config)
-            print("Connector_config",connector_config)
+            
         else:
             ObsrvException(
                 ErrorData(
diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 0ca5533..c810b9b 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -41,14 +41,14 @@ def get_spark_config(self, connector_config) -> SparkConf:
     def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollector) -> List[ObjectInfo]:
         
         objects = self._list_blobs_in_container(ctx,metrics_collector=metrics_collector)
-        print("objects:",objects)
+        
         objects_info=[]
         if objects==None:
             raise Exception("No objects found")
     
         for obj in objects:
             blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
-            print("blob location:",blob_location)
+            
             object_info = ObjectInfo(
                     location=blob_location,
                     format=obj["name"].split(".")[-1],
@@ -116,10 +116,10 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
 
     def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> list:
         container_name = self.config['source']['containername']
-        print("container_name\n",container_name)
+        
         summaries = []
         continuation_token = None
-        formats = {
+        file_formats = {
             "json": ["json", "json.gz", "json.zip"],
             "jsonl": ["json", "json.gz", "json.zip"],
             "csv": ["csv", "csv.gz", "csv.zip"],
@@ -132,7 +132,7 @@ def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> l
             {"key": "request_method", "value": "GET"},
             {"key": "method_name", "value": "list_blobs"},
         ]
-        print("--------------=====================")
+        
         container_client = ContainerClient.from_connection_string(conn_str=self.connection_string, container_name=container_name)
         while True:
             try:      
@@ -143,8 +143,8 @@ def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> l
                 api_calls += 1
                 
                 for blob in blobs:
-                    # if any(obj["name"].endswith(f) for f in file_formats[file_format]):
-                    summaries.append(blob)
+                    if any(blob["name"].endswith(f) for f in file_formats[file_format]):
+                        summaries.append(blob)
                 
                 if not continuation_token:
                     break
@@ -187,12 +187,12 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
                 conn_str=self.connection_string, container_name=self.container_name, blob_name=object_path
             )
             tags = blob_client.get_blob_tags()
-            print("Blob tags:")
+            
             api_calls += 1
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
             for k, v in tags.items():
                 print(k, v)
-            print("\n")  
+            
             return [Tag(key,value) for key,value in tags.items()]
 
         #Need to check on the exception.message    
@@ -219,7 +219,7 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
         api_calls, errors = 0, 0
 
         try:
-            print("tags parameter\n",tags)
+            
             new_dict = {tag['key']: tag['value'] for tag in tags}
         
             stripped_file_path = object.get("location").lstrip("wasb://")
@@ -230,11 +230,11 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
             )
             existing_tags = blob_client.get_blob_tags() or {}
             existing_tags.update(new_dict)
-            print("existing tags",existing_tags)
+            
             blob_client.set_blob_tags(existing_tags)
-            print("Blob tags updated!")
+            
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
-            print("updated tags",existing_tags)
+            
             return True
         except AzureError as exception:
             errors += 1

From 562f00428f16458f68648f5e69fd46e72180f700 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Fri, 17 May 2024 16:21:31 +0530
Subject: [PATCH 20/59] obsrv-labs/issue-tracker#6: lint and cleanup

---
 object_store_connector/provider/azure.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index c810b9b..e013808 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -115,7 +115,7 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
 
 
     def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> list:
-        container_name = self.config['source']['containername']
+        self.container_name = self.config['source']['containername']
         
         summaries = []
         continuation_token = None
@@ -133,13 +133,13 @@ def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> l
             {"key": "method_name", "value": "list_blobs"},
         ]
         
-        container_client = ContainerClient.from_connection_string(conn_str=self.connection_string, container_name=container_name)
+        # container_client = ContainerClient.from_connection_string(conn_str=self.connection_string, container_name=container_name)
         while True:
             try:      
                 if continuation_token:             
-                    blobs = container_client.list_blobs(results_per_page=1).by_page(continuation_token=continuation_token)
+                    blobs = self.container_client.list_blobs(results_per_page=1).by_page(continuation_token=continuation_token)
                 else:
-                    blobs= container_client.list_blobs()
+                    blobs= self.container_client.list_blobs()
                 api_calls += 1
                 
                 for blob in blobs:

From deae8e3c6b817613d0d46f8b30e2641d9477512a Mon Sep 17 00:00:00 2001
From: i-manaswiniee <143373040+i-manaswiniee@users.noreply.github.com>
Date: Fri, 17 May 2024 16:31:07 +0530
Subject: [PATCH 21/59] obsrv-labs/issue-tracker#2: Implemented Pagination
 Functionality

---
 object_store_connector/provider/gcs.py | 37 ++++++++++----------------
 1 file changed, 14 insertions(+), 23 deletions(-)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index 3c0f3e1..d429f44 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -21,7 +21,6 @@ def __init__(self, connector_config) -> None:
             self.connector_config=connector_config
             self.bucket = connector_config['source']['bucket']
             self.credentials = connector_config['source']['credentials']
-            # self.prefix = connector_config.get('prefix', '/')
             self.prefix = (
             connector_config["source"]["prefix"]
             if "prefix" in connector_config["source"]
@@ -60,7 +59,6 @@ def get_spark_config(self,connector_config)->SparkConf:
             conf.set("spark.hadoop.fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS")
             return conf
 
-
     def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> List[Tag]:
         labels = [
             {"key": "request_method", "value": "GET"},
@@ -90,7 +88,6 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
             raise Exception(f"failed to fetch tags from GCS: {str(exception)}")
     
-
     def update_tag(self, object: ObjectInfo, tags: list, metrics_collector: MetricsCollector) -> bool:
         object_path = object.get('location')
         labels = [
@@ -104,7 +101,6 @@ def update_tag(self, object: ObjectInfo, tags: list, metrics_collector: MetricsC
             new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
             updated_tags = [tag for tag in new_tags if 'key' in tag and 'value' in tag]
             relative_path=object_path.lstrip("gs://").split("/",1)[-1]
-            print(f"PATH:{relative_path}")
             bucket = self.gcs_client.bucket(self.bucket)
             blob = bucket.blob(relative_path)
             if blob is None:
@@ -145,7 +141,6 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
         print("Object_Info:", objects_info)
         return objects_info
 
-
     def read_object(self, object_path: str, sc: SparkSession,metrics_collector:MetricsCollector, file_format: str) -> DataFrame:
         labels = [
              {"key": "request_method", "value": "GET"},
@@ -175,11 +170,6 @@ def read_object(self, object_path: str, sc: SparkSession,metrics_collector:Metri
             labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
             raise Exception( f"failed to read object from GCS: {str(exception)}")
-        except Exception as exception:
-            errors += 1
-            labels += [{"key": "error_code", "value": "GCS_READ_ERROR"}]
-            metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            raise Exception( f"failed to read object from GCS: {str(exception)}")
 
     def _list_objects(self, ctx: ConnectorContext, metrics_collector: MetricsCollector) -> list:
         bucket_name = self.connector_config['source']['bucket']
@@ -196,21 +186,21 @@ def _list_objects(self, ctx: ConnectorContext, metrics_collector: MetricsCollect
         ]
         api_calls, errors = 0, 0
         page_token = None
-        page_size = 1000# Set your desired page size here
+        page_size = 1 
         while True:
             try:
-                    bucket = self.gcs_client.bucket(bucket_name)
-                    if page_token:
-                        blobs_response = bucket.list_blobs(prefix=self.prefix, page_token=page_token, max_results=page_size)
-                    else:
-                        blobs_response = bucket.list_blobs(prefix=self.prefix, max_results=page_size)
-                    api_calls += 1
-                    for blob in blobs_response:
-                        if any(blob.name.endswith(f) for f in file_formats[file_format]):
-                            summaries.append(blob)
-                    if not blobs_response.next_page_token:
-                        break
-                    page_token = blobs_response.next_page_token
+                bucket = self.gcs_client.bucket(bucket_name)
+                if page_token:
+                    blobs_response = bucket.list_blobs(prefix=self.prefix, page_token=page_token, max_results=page_size)
+                else:
+                    blobs_response = bucket.list_blobs(prefix=self.prefix, max_results=page_size)
+                api_calls += 1
+                for blob in blobs_response:
+                    if any(blob.name.endswith(f) for f in file_formats[file_format]):
+                        summaries.append(blob)
+                if not blobs_response.next_page_token:
+                    break
+                page_token = blobs_response.next_page_token
             except ClientError as exception:
                 errors += 1
                 labels.append({"key": "error_code", "value": str(exception.response['Error']['Code'])})
@@ -221,3 +211,4 @@ def _list_objects(self, ctx: ConnectorContext, metrics_collector: MetricsCollect
 
     def _get_spark_session(self):
              return SparkSession.builder.config(conf=self.get_spark_config()).getOrCreate()
+    

From 5a25be0a617ac179bf17fa03c5ef176eeb559fdb Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Fri, 17 May 2024 16:50:07 +0530
Subject: [PATCH 22/59] obsrv-labs/issue-tracker#6: feat: increased the
 results_per_page to 1000.

---
 object_store_connector/provider/azure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index e013808..6d02d2a 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -137,7 +137,7 @@ def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> l
         while True:
             try:      
                 if continuation_token:             
-                    blobs = self.container_client.list_blobs(results_per_page=1).by_page(continuation_token=continuation_token)
+                    blobs = self.container_client.list_blobs(results_per_page=1000).by_page(continuation_token=continuation_token)
                 else:
                     blobs= self.container_client.list_blobs()
                 api_calls += 1

From 3ed0a5063b759f3b77fc534d2ecc1bace85350bc Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Fri, 17 May 2024 16:55:52 +0530
Subject: [PATCH 23/59] Update object_info.py

---
 object_store_connector/models/object_info.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
index 78fb94d..e9cdabb 100644
--- a/object_store_connector/models/object_info.py
+++ b/object_store_connector/models/object_info.py
@@ -26,12 +26,6 @@ def to_gcs(self):
             'Value': self.value
         }
 
-    def to_gcs(self):
-        return {
-            'Key': self.key,
-            'Value': self.value
-        }
-
 @dataclass
 class ObjectInfo:
     id: str = field(default_factory=lambda: str(uuid4()))

From b535dab398d1c79ffb6fee8450d6c5388ba93d0e Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 15:48:32 +0530
Subject: [PATCH 24/59] obsrv-labs/issue-tracker#2: Code Cleanup

---
 object_store_connector/provider/gcs.py | 74 ++++++++++++++++++--------
 1 file changed, 53 insertions(+), 21 deletions(-)

diff --git a/object_store_connector/provider/gcs.py b/object_store_connector/provider/gcs.py
index d429f44..28f6e8f 100644
--- a/object_store_connector/provider/gcs.py
+++ b/object_store_connector/provider/gcs.py
@@ -5,12 +5,15 @@
 from uuid import uuid4
 from pyspark.conf import SparkConf
 from models.object_info import Tag,ObjectInfo
+from obsrv.common import ObsrvException
 from pyspark.sql import SparkSession
 from obsrv.connector import MetricsCollector,ConnectorContext
 from obsrv.job.batch import get_base_conf
+from obsrv.models import ErrorData
 from google.api_core.exceptions import ClientError
 from provider.blob_provider import BlobProvider
 import json
+import os
 
 class GCS(BlobProvider):
     def __init__(self, connector_config) -> None:
@@ -44,15 +47,12 @@ def _get_client(self) -> storage.Client:
                 "client_x509_cert_url": self.credentials["client_x509_cert_url"]
             })
         )
-        try:
-            return client
-        except Exception as e:
-            print(f"Error creating GCS client: {str(e)}")
+        return client
 
     def get_spark_config(self,connector_config)->SparkConf:
             conf = get_base_conf()
             conf.setAppName("ObsrvObjectStoreConnector") 
-            conf.set("spark.jars", "/root/spark/jars/gcs-connector-hadoop3-2.2.22-shaded.jar") 
+            conf.set("spark.jars", os.path.join(os.path.dirname(__file__),"lib/gcs-connector-hadoop3-2.2.22-shaded.jar"))
             conf.set("spark.hadoop.google.cloud.auth.service.account.json.keyfile", self.key_path) 
             conf.set("spark.hadoop.google.cloud.auth.service.account.enable", "true") 
             conf.set("spark.hadoop.fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem") 
@@ -70,7 +70,6 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             bucket = self.gcs_client.bucket(self.bucket)
             blob = bucket.get_blob(object_path)
             if blob is None:
-                print(f"Object '{object_path}' not found in bucket '{self.bucket}'.")
                 return []
             api_calls += 1
             blob.reload()
@@ -86,7 +85,12 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             errors += 1
             labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            raise Exception(f"failed to fetch tags from GCS: {str(exception)}")
+            ObsrvException(
+                ErrorData(
+                    "GCS_TAG_READ_ERROR",
+                    f"failed to fetch tags from GCS: {str(exception)}",
+                )
+            )
     
     def update_tag(self, object: ObjectInfo, tags: list, metrics_collector: MetricsCollector) -> bool:
         object_path = object.get('location')
@@ -99,15 +103,15 @@ def update_tag(self, object: ObjectInfo, tags: list, metrics_collector: MetricsC
         try:
             initial_tags = object.get('tags')
             new_tags = list(json.loads(t) for t in set([json.dumps(t) for t in initial_tags + tags]))
-            updated_tags = [tag for tag in new_tags if 'key' in tag and 'value' in tag]
-            relative_path=object_path.lstrip("gs://").split("/",1)[-1]
+            updated_tags = [Tag(key=tag['key'], value=tag['value']).to_gcs() for tag in new_tags if 'key' in tag and 'value' in tag]
+            relative_path = object_path.lstrip("gs://").split("/", 1)[-1]
             bucket = self.gcs_client.bucket(self.bucket)
             blob = bucket.blob(relative_path)
             if blob is None:
-                print(f"Object '{object_path}' not found in bucket '{self.bucket}'.")
                 return False
-            blob.metadata = {tag['key']: tag['value'] for tag in updated_tags}
+            blob.metadata = {tag['Key']: tag['Value'] for tag in updated_tags}
             blob.patch()
+            api_calls += 1
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
             return True
         except ClientError as exception:
@@ -116,35 +120,35 @@ def update_tag(self, object: ObjectInfo, tags: list, metrics_collector: MetricsC
                 {"key": "error_code", "value": str(exception.response["Error"]["Code"])}
             ]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            Exception(
+            ObsrvException(
+                ErrorData(
                     "GCS_TAG_UPDATE_ERROR",
                     f"failed to update tags in GCS for object: {str(exception)}",
                 )
+            )
             return False
 
     def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollector) -> List[ObjectInfo]:
         objects = self._list_objects(ctx,metrics_collector=metrics_collector)
         objects_info = []
         if not objects:
-            print(f"No objects found")
             return []
         for obj in objects:
             object_info = ObjectInfo(
                 id=str(uuid4()),
                 location=f"gs://{self.bucket}/{obj.name}",
                 format=obj.name.split(".")[-1],
-                file_size_kb=obj.size / 1024,
+                file_size_kb=obj.size // 1024,
                 file_hash=obj.etag.strip('"'),
                 tags=self.fetch_tags(obj.name, metrics_collector)
             )
             objects_info.append(object_info.to_json())
-        print("Object_Info:", objects_info)
         return objects_info
 
     def read_object(self, object_path: str, sc: SparkSession,metrics_collector:MetricsCollector, file_format: str) -> DataFrame:
         labels = [
              {"key": "request_method", "value": "GET"},
-             {"key": "method_name", "value": "getObjectMetadata"},
+             {"key": "method_name", "value": "getObject"},
              {"key": "object_path", "value": object_path}
          ]
         api_calls,errors,records_count= 0,0,0
@@ -160,16 +164,39 @@ def read_object(self, object_path: str, sc: SparkSession,metrics_collector:Metri
              elif file_format == "csv.gz":
                  df = sc.read.format("csv").option("header", True).option("compression", "gzip").load(object_path)
              else:
-               raise Exception(f"Unsupported file format for file: {file_format}")
+                raise ObsrvException(
+                    ErrorData(
+                        "UNSUPPORTED_FILE_FORMAT",
+                        f"unsupported file format: {file_format}",
+                    )
+                )
              records_count = df.count()
              api_calls += 1
-             metrics_collector.collect({"num_api_calls": api_calls, "num_records": records_count}, addn_labels=labels)
+             metrics_collector.collect(
+                {"num_api_calls": api_calls, "num_records": records_count},
+                addn_labels=labels,
+             )
              return df
         except (ClientError) as exception:
             errors += 1
             labels += [{"key": "error_code", "value": str(exception.response['Error']['Code'])}]
             metrics_collector.collect("num_errors", errors, addn_labels=labels)
-            raise Exception( f"failed to read object from GCS: {str(exception)}")
+            ObsrvException(
+                ErrorData(
+                    "GCS_READ_ERROR", f"failed to read object from GCS: {str(exception)}"
+                )
+            )
+            return None
+        except Exception as exception:
+            errors += 1
+            labels += [{"key": "error_code", "value": "GCS_READ_ERROR"}]
+            metrics_collector.collect("num_errors", errors, addn_labels=labels)
+            ObsrvException(
+                ErrorData(
+                    "GCS_READ_ERROR", f"failed to read object from GCS: {str(exception)}"
+                )
+            )
+            return None
 
     def _list_objects(self, ctx: ConnectorContext, metrics_collector: MetricsCollector) -> list:
         bucket_name = self.connector_config['source']['bucket']
@@ -186,7 +213,7 @@ def _list_objects(self, ctx: ConnectorContext, metrics_collector: MetricsCollect
         ]
         api_calls, errors = 0, 0
         page_token = None
-        page_size = 1 
+        page_size = 1000
         while True:
             try:
                 bucket = self.gcs_client.bucket(bucket_name)
@@ -205,7 +232,12 @@ def _list_objects(self, ctx: ConnectorContext, metrics_collector: MetricsCollect
                 errors += 1
                 labels.append({"key": "error_code", "value": str(exception.response['Error']['Code'])})
                 metrics_collector.collect("num_errors", errors, addn_labels=labels)
-                Exception(f"failed to list objects in GCS: {str(exception)}")
+                ObsrvException(
+                    ErrorData(
+                        "GCS_LIST_ERROR",
+                        f"failed to list objects in GCS: {str(exception)}",
+                    )
+                )
         metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
         return summaries
 

From 4787e66bcd89e03173f61871cee8af1cc9847833 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 15:51:03 +0530
Subject: [PATCH 25/59] obsrv-labs/issue-tracker#2: Added GCS Dependencies

---
 poetry.lock | 695 ++++++++++++++++++++++++++++++++++------------------
 1 file changed, 459 insertions(+), 236 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index fb2b630..3e4bd2a 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
 
 [[package]]
 name = "boto3"
@@ -33,11 +33,135 @@ files = [
 [package.dependencies]
 jmespath = ">=0.7.1,<2.0.0"
 python-dateutil = ">=2.1,<3.0.0"
-urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}
+urllib3 = [
+    {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""},
+    {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""},
+]
 
 [package.extras]
 crt = ["awscrt (==0.19.19)"]
 
+[[package]]
+name = "cachetools"
+version = "5.3.3"
+description = "Extensible memoizing collections and decorators"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"},
+    {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"},
+]
+
+[[package]]
+name = "certifi"
+version = "2024.2.2"
+description = "Python package for providing Mozilla's CA Bundle."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"},
+    {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"},
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.3.2"
+description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"},
+    {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"},
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -114,66 +238,252 @@ files = [
 toml = ["tomli"]
 
 [[package]]
-name = "iniconfig"
-version = "2.0.0"
-description = "brain-dead simple config-ini parsing"
+name = "exceptiongroup"
+version = "1.2.1"
+description = "Backport of PEP 654 (exception groups)"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
-    {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
+    {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"},
+    {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"},
 ]
 
+[package.extras]
+test = ["pytest (>=6)"]
+
 [[package]]
-name = "jmespath"
-version = "1.0.1"
-description = "JSON Matching Expressions"
+name = "google-api-core"
+version = "2.19.0"
+description = "Google API client core library"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
-    {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
+    {file = "google-api-core-2.19.0.tar.gz", hash = "sha256:cf1b7c2694047886d2af1128a03ae99e391108a08804f87cfd35970e49c9cd10"},
+    {file = "google_api_core-2.19.0-py3-none-any.whl", hash = "sha256:8661eec4078c35428fd3f69a2c7ee29e342896b70f01d1a1cbcb334372dd6251"},
 ]
 
+[package.dependencies]
+google-auth = ">=2.14.1,<3.0.dev0"
+googleapis-common-protos = ">=1.56.2,<2.0.dev0"
+proto-plus = ">=1.22.3,<2.0.0dev"
+protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0"
+requests = ">=2.18.0,<3.0.0.dev0"
+
+[package.extras]
+grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"]
+grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
+grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
+
 [[package]]
-name = "kafka-python-ng"
-version = "2.2.2"
-description = "Pure Python client for Apache Kafka"
+name = "google-auth"
+version = "2.29.0"
+description = "Google Authentication Library"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.7"
 files = [
-    {file = "kafka-python-ng-2.2.2.tar.gz", hash = "sha256:87ad3a766e2c0bec71d9b99bdd9e9c5cda62d96cfda61a8ca16510484d6ad7d4"},
-    {file = "kafka_python_ng-2.2.2-py2.py3-none-any.whl", hash = "sha256:3fab1a03133fade1b6fd5367ff726d980e59031c4aaca9bf02c516840a4f8406"},
+    {file = "google-auth-2.29.0.tar.gz", hash = "sha256:672dff332d073227550ffc7457868ac4218d6c500b155fe6cc17d2b13602c360"},
+    {file = "google_auth-2.29.0-py2.py3-none-any.whl", hash = "sha256:d452ad095688cd52bae0ad6fafe027f6a6d6f560e810fec20914e17a09526415"},
 ]
 
+[package.dependencies]
+cachetools = ">=2.0.0,<6.0"
+pyasn1-modules = ">=0.2.1"
+rsa = ">=3.1.4,<5"
+
 [package.extras]
-boto = ["botocore"]
-crc32c = ["crc32c"]
-lz4 = ["lz4"]
-snappy = ["python-snappy"]
-zstd = ["zstandard"]
+aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"]
+enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"]
+pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"]
+reauth = ["pyu2f (>=0.1.5)"]
+requests = ["requests (>=2.20.0,<3.0.0.dev0)"]
 
 [[package]]
-name = "obsrv"
-version = "0.1.0"
-description = ""
+name = "google-cloud-core"
+version = "2.4.1"
+description = "Google Cloud API client core library"
 optional = false
-python-versions = "<4.0,>=3.10"
+python-versions = ">=3.7"
+files = [
+    {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"},
+    {file = "google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61"},
+]
+
+[package.dependencies]
+google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev"
+google-auth = ">=1.25.0,<3.0dev"
+
+[package.extras]
+grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"]
+
+[[package]]
+name = "google-cloud-storage"
+version = "2.16.0"
+description = "Google Cloud Storage API client library"
+optional = false
+python-versions = ">=3.7"
 files = [
-    {file = "obsrv-0.1.0-py3-none-any.whl", hash = "sha256:91ae7deec62fcb3ea4043bc9119f717405137d32f489cdf1daf5a3f5d05bf2d1"},
-    {file = "obsrv-0.1.0.tar.gz", hash = "sha256:d087fe8727be18117188a36e9bfbe173dc6a8aa07400944c99f225b4d73ecadb"},
+    {file = "google-cloud-storage-2.16.0.tar.gz", hash = "sha256:dda485fa503710a828d01246bd16ce9db0823dc51bbca742ce96a6817d58669f"},
+    {file = "google_cloud_storage-2.16.0-py2.py3-none-any.whl", hash = "sha256:91a06b96fb79cf9cdfb4e759f178ce11ea885c79938f89590344d079305f5852"},
 ]
 
 [package.dependencies]
-coverage = ">=7.5.0,<8.0.0"
-kafka-python-ng = ">=2.2.2,<3.0.0"
-psycopg2-binary = ">=2.9.9,<3.0.0"
-pycryptodome = ">=3.20.0,<4.0.0"
-pyspark = {version = ">=3.5.1,<4.0.0", optional = true, markers = "extra == \"batch\""}
-pyyaml = ">=6.0.1,<7.0.0"
+google-api-core = ">=2.15.0,<3.0.0dev"
+google-auth = ">=2.26.1,<3.0dev"
+google-cloud-core = ">=2.3.0,<3.0dev"
+google-crc32c = ">=1.0,<2.0dev"
+google-resumable-media = ">=2.6.0"
+requests = ">=2.18.0,<3.0.0dev"
+
+[package.extras]
+protobuf = ["protobuf (<5.0.0dev)"]
+
+[[package]]
+name = "google-crc32c"
+version = "1.5.0"
+description = "A python wrapper of the C library 'Google CRC32C'"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "google-crc32c-1.5.0.tar.gz", hash = "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-win32.whl", hash = "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee"},
+    {file = "google_crc32c-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-win32.whl", hash = "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709"},
+    {file = "google_crc32c-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-win32.whl", hash = "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94"},
+    {file = "google_crc32c-1.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-win32.whl", hash = "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4"},
+    {file = "google_crc32c-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"},
+    {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"},
+    {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"},
+    {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"},
+    {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"},
+    {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"},
+    {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"},
+    {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"},
+    {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"},
+    {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"},
+    {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"},
+    {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"},
+    {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"},
+    {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"},
+    {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"},
+    {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31"},
+    {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93"},
+]
 
 [package.extras]
-batch = ["pyspark (>=3.5.1,<4.0.0)"]
+testing = ["pytest"]
+
+[[package]]
+name = "google-resumable-media"
+version = "2.7.0"
+description = "Utilities for Google Media Downloads and Resumable Uploads"
+optional = false
+python-versions = ">= 3.7"
+files = [
+    {file = "google-resumable-media-2.7.0.tar.gz", hash = "sha256:5f18f5fa9836f4b083162064a1c2c98c17239bfda9ca50ad970ccf905f3e625b"},
+    {file = "google_resumable_media-2.7.0-py2.py3-none-any.whl", hash = "sha256:79543cfe433b63fd81c0844b7803aba1bb8950b47bedf7d980c38fa123937e08"},
+]
+
+[package.dependencies]
+google-crc32c = ">=1.0,<2.0dev"
+
+[package.extras]
+aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"]
+requests = ["requests (>=2.18.0,<3.0.0dev)"]
+
+[[package]]
+name = "googleapis-common-protos"
+version = "1.63.0"
+description = "Common protobufs used in Google APIs"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "googleapis-common-protos-1.63.0.tar.gz", hash = "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e"},
+    {file = "googleapis_common_protos-1.63.0-py2.py3-none-any.whl", hash = "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632"},
+]
+
+[package.dependencies]
+protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0"
+
+[package.extras]
+grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"]
+
+[[package]]
+name = "idna"
+version = "3.7"
+description = "Internationalized Domain Names in Applications (IDNA)"
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"},
+    {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"},
+]
+
+[[package]]
+name = "iniconfig"
+version = "2.0.0"
+description = "brain-dead simple config-ini parsing"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
+    {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
+]
+
+[[package]]
+name = "jmespath"
+version = "1.0.1"
+description = "JSON Matching Expressions"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
+    {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
+]
 
 [[package]]
 name = "packaging"
@@ -202,157 +512,66 @@ dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
 [[package]]
-name = "psycopg2-binary"
-version = "2.9.9"
-description = "psycopg2 - Python-PostgreSQL Database Adapter"
+name = "proto-plus"
+version = "1.23.0"
+description = "Beautiful, Pythonic protocol buffers."
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.6"
 files = [
-    {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"},
-    {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"},
-    {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"},
-    {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"},
-    {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"},
-    {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"},
-    {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"},
-]
-
-[[package]]
-name = "py4j"
-version = "0.10.9.7"
-description = "Enables Python programs to dynamically access arbitrary Java objects"
-optional = false
-python-versions = "*"
-files = [
-    {file = "py4j-0.10.9.7-py2.py3-none-any.whl", hash = "sha256:85defdfd2b2376eb3abf5ca6474b51ab7e0de341c75a02f46dc9b5976f5a5c1b"},
-    {file = "py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb"},
-]
-
-[[package]]
-name = "pycryptodome"
-version = "3.20.0"
-description = "Cryptographic library for Python"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-files = [
-    {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"},
-    {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"},
-    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"},
-    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"},
-    {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"},
-    {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"},
-    {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"},
-    {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"},
-    {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"},
-    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"},
-    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"},
-    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"},
-    {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"},
-    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"},
-    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"},
-    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"},
-    {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"},
-    {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"},
-]
-
-[[package]]
-name = "pyspark"
-version = "3.5.1"
-description = "Apache Spark Python API"
+    {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"},
+    {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"},
+]
+
+[package.dependencies]
+protobuf = ">=3.19.0,<5.0.0dev"
+
+[package.extras]
+testing = ["google-api-core[grpc] (>=1.31.5)"]
+
+[[package]]
+name = "protobuf"
+version = "4.25.3"
+description = ""
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pyspark-3.5.1.tar.gz", hash = "sha256:dd6569e547365eadc4f887bf57f153e4d582a68c4b490de475d55b9981664910"},
+    {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"},
+    {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"},
+    {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"},
+    {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"},
+    {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"},
+    {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"},
+    {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"},
+    {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"},
+    {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"},
+    {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"},
+    {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"},
 ]
 
-[package.dependencies]
-py4j = "0.10.9.7"
+[[package]]
+name = "pyasn1"
+version = "0.6.0"
+description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pyasn1-0.6.0-py2.py3-none-any.whl", hash = "sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473"},
+    {file = "pyasn1-0.6.0.tar.gz", hash = "sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c"},
+]
 
-[package.extras]
-connect = ["googleapis-common-protos (>=1.56.4)", "grpcio (>=1.56.0)", "grpcio-status (>=1.56.0)", "numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"]
-ml = ["numpy (>=1.15)"]
-mllib = ["numpy (>=1.15)"]
-pandas-on-spark = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"]
-sql = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=4.0.0)"]
+[[package]]
+name = "pyasn1-modules"
+version = "0.4.0"
+description = "A collection of ASN.1-based protocols modules"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pyasn1_modules-0.4.0-py3-none-any.whl", hash = "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b"},
+    {file = "pyasn1_modules-0.4.0.tar.gz", hash = "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6"},
+]
+
+[package.dependencies]
+pyasn1 = ">=0.4.6,<0.7.0"
 
 [[package]]
 name = "pytest"
@@ -367,9 +586,11 @@ files = [
 
 [package.dependencies]
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
+exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
 pluggy = ">=1.4,<2.0"
+tomli = {version = ">=1", markers = "python_version < \"3.11\""}
 
 [package.extras]
 testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
@@ -389,65 +610,40 @@ files = [
 six = ">=1.5"
 
 [[package]]
-name = "pyyaml"
-version = "6.0.1"
-description = "YAML parser and emitter for Python"
+name = "requests"
+version = "2.31.0"
+description = "Python HTTP for Humans."
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.7"
 files = [
-    {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"},
-    {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
-    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
-    {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
-    {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
-    {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
-    {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
-    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
-    {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
-    {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
-    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"},
-    {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
-    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
-    {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
-    {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
-    {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
-    {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
-    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
-    {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
-    {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
-    {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
+    {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
+    {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
 ]
 
+[package.dependencies]
+certifi = ">=2017.4.17"
+charset-normalizer = ">=2,<4"
+idna = ">=2.5,<4"
+urllib3 = ">=1.21.1,<3"
+
+[package.extras]
+socks = ["PySocks (>=1.5.6,!=1.5.7)"]
+use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
+
+[[package]]
+name = "rsa"
+version = "4.9"
+description = "Pure-Python RSA implementation"
+optional = false
+python-versions = ">=3.6,<4"
+files = [
+    {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"},
+    {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"},
+]
+
+[package.dependencies]
+pyasn1 = ">=0.1.3"
+
 [[package]]
 name = "s3transfer"
 version = "0.10.1"
@@ -476,6 +672,33 @@ files = [
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 
+[[package]]
+name = "tomli"
+version = "2.0.1"
+description = "A lil' TOML parser"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
+    {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
+]
+
+[[package]]
+name = "urllib3"
+version = "1.26.18"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+files = [
+    {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"},
+    {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},
+]
+
+[package.extras]
+brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
+secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
 [[package]]
 name = "urllib3"
 version = "2.2.1"
@@ -495,5 +718,5 @@ zstd = ["zstandard (>=0.18.0)"]
 
 [metadata]
 lock-version = "2.0"
-python-versions = "^3.12"
-content-hash = "b221af180d723448b4398e52b7b43ebac510f532d87dbe21de9b12668eb786ce"
+python-versions = "^3.8"
+content-hash = "aba102d48a694f410cb09cd8d862442328bf01203a1355072d3a35cf41dad874"

From ef4cde973ec81501a970f1be96808818f33a4243 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 15:53:21 +0530
Subject: [PATCH 26/59] obsrv-labs/issue-tracker#2: Cleanup

---
 .../{object_info.py => obsrv-labs/issue-tracker#2: Cleanup}       | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename object_store_connector/models/{object_info.py => obsrv-labs/issue-tracker#2: Cleanup} (100%)

diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/obsrv-labs/issue-tracker#2: Cleanup
similarity index 100%
rename from object_store_connector/models/object_info.py
rename to object_store_connector/models/obsrv-labs/issue-tracker#2: Cleanup

From d6b8e7691ba0682d756b62ea0228b7cda399df3d Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 15:54:59 +0530
Subject: [PATCH 27/59] obsrv-labs/issue-tracker#2: Added GCS Dependency

---
 pyproject.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index ea167c9..cbefdf0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,10 +6,10 @@ authors = ["Ravi Mula <ravismula@users.noreply.github.com>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.12"
+python = "^3.8"
 boto3 = "^1.34.82"
 coverage = "^7.5.0"
-
+google-cloud-storage = "^2.16.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.1.1"
@@ -17,4 +17,4 @@ coverage = "^7.4.3"
 
 [build-system]
 requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
\ No newline at end of file
+build-backend = "poetry.core.masonry.api"

From 8a80e4422f9706531e1e43d23ee5517f19c46ab0 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 15:58:47 +0530
Subject: [PATCH 28/59] Delete

---
 .../obsrv-labs/issue-tracker#2: Cleanup       | 60 -------------------
 1 file changed, 60 deletions(-)
 delete mode 100644 object_store_connector/models/obsrv-labs/issue-tracker#2: Cleanup

diff --git a/object_store_connector/models/obsrv-labs/issue-tracker#2: Cleanup b/object_store_connector/models/obsrv-labs/issue-tracker#2: Cleanup
deleted file mode 100644
index e9cdabb..0000000
--- a/object_store_connector/models/obsrv-labs/issue-tracker#2: Cleanup	
+++ /dev/null
@@ -1,60 +0,0 @@
-from dataclasses import dataclass, field
-from datetime import datetime
-from typing import List
-from uuid import uuid4
-
-
-@dataclass
-class Tag:
-    key: str
-    value: str
-
-    def to_dict(self):
-        return {
-            'key': self.key,
-            'value': self.value
-        }
-
-    def to_aws(self):
-        return {
-            'Key': self.key,
-            'Value': self.value
-        }
-    def to_gcs(self):
-        return {
-            'Key': self.key,
-            'Value': self.value
-        }
-
-@dataclass
-class ObjectInfo:
-    id: str = field(default_factory=lambda: str(uuid4()))
-    connector_id: str = None
-    dataset_id: str = None
-    location: str = None
-    format: str = None
-    file_size_kb: int = 0
-    in_time: datetime = field(default_factory=datetime.now)
-    start_processing_time: datetime = None
-    end_processing_time: datetime = None
-    download_time: float = 0.0
-    file_hash: str = None
-    num_of_retries: int = 0
-    tags: List[Tag] = field(default_factory=list)
-
-    def to_json(self):
-        return {
-            'id': self.id,
-            'connector_id': self.connector_id,
-            'dataset_id': self.dataset_id,
-            'location': self.location,
-            'format': self.format,
-            'file_size_kb': self.file_size_kb,
-            'in_time': self.in_time,
-            'download_time': self.download_time,
-            'start_processing_time': self.start_processing_time,
-            'end_processing_time': self.end_processing_time,
-            'file_hash': self.file_hash,
-            'num_of_retries': self.num_of_retries,
-            'tags': [tag.__dict__ for tag in self.tags]
-        }

From d58df5be856f5c43af77716d8fde4fece9a81092 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 16:00:58 +0530
Subject: [PATCH 29/59] obsrv-labs/issue-tracker#2: Cleanup

---
 object_store_connector/models/object_info.py | 59 ++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 object_store_connector/models/object_info.py

diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
new file mode 100644
index 0000000..92c9fce
--- /dev/null
+++ b/object_store_connector/models/object_info.py
@@ -0,0 +1,59 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List
+from uuid import uuid4
+
+@dataclass
+class Tag:
+    key: str
+    value: str
+
+    def to_dict(self):
+        return {
+            'key': self.key,
+            'value': self.value
+        }
+
+    def to_aws(self):
+        return {
+            'Key': self.key,
+            'Value': self.value
+        }
+    def to_gcs(self):
+        return {
+            'Key': self.key,
+            'Value': self.value
+        }
+
+@dataclass
+class ObjectInfo:
+    id: str = field(default_factory=lambda: str(uuid4()))
+    connector_id: str = None
+    dataset_id: str = None
+    location: str = None
+    format: str = None
+    file_size_kb: int = 0
+    in_time: datetime = field(default_factory=datetime.now)
+    start_processing_time: datetime = None
+    end_processing_time: datetime = None
+    download_time: float = 0.0
+    file_hash: str = None
+    num_of_retries: int = 0
+    tags: List[Tag] = field(default_factory=list)
+
+    def to_json(self):
+        return {
+            'id': self.id,
+            'connector_id': self.connector_id,
+            'dataset_id': self.dataset_id,
+            'location': self.location,
+            'format': self.format,
+            'file_size_kb': self.file_size_kb,
+            'in_time': self.in_time,
+            'download_time': self.download_time,
+            'start_processing_time': self.start_processing_time,
+            'end_processing_time': self.end_processing_time,
+            'file_hash': self.file_hash,
+            'num_of_retries': self.num_of_retries,
+            'tags': [tag.__dict__ for tag in self.tags]
+        }

From 9a22f4f9304bceab11aa860abfc007a2c7ad835e Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 16:01:32 +0530
Subject: [PATCH 30/59] obsrv-labs/issue-tracker#2: Jar File

---
 object_store_connector/models/lib | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 object_store_connector/models/lib

diff --git a/object_store_connector/models/lib b/object_store_connector/models/lib
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/object_store_connector/models/lib
@@ -0,0 +1 @@
+

From 95af58f71ed448c23ffe8c77fc01e050a0c84e4e Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 16:01:53 +0530
Subject: [PATCH 31/59] Delete

---
 object_store_connector/models/lib | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 object_store_connector/models/lib

diff --git a/object_store_connector/models/lib b/object_store_connector/models/lib
deleted file mode 100644
index 8b13789..0000000
--- a/object_store_connector/models/lib
+++ /dev/null
@@ -1 +0,0 @@
-

From 6fe32481b5a602f4ffaf863329ad9a1a41e0f246 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 16:06:19 +0530
Subject: [PATCH 32/59] obsrv-labs/issue-tracker#2: Model for gcs

---
 object_store_connector/models/object_info.py | 59 --------------------
 1 file changed, 59 deletions(-)
 delete mode 100644 object_store_connector/models/object_info.py

diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
deleted file mode 100644
index 92c9fce..0000000
--- a/object_store_connector/models/object_info.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from dataclasses import dataclass, field
-from datetime import datetime
-from typing import List
-from uuid import uuid4
-
-@dataclass
-class Tag:
-    key: str
-    value: str
-
-    def to_dict(self):
-        return {
-            'key': self.key,
-            'value': self.value
-        }
-
-    def to_aws(self):
-        return {
-            'Key': self.key,
-            'Value': self.value
-        }
-    def to_gcs(self):
-        return {
-            'Key': self.key,
-            'Value': self.value
-        }
-
-@dataclass
-class ObjectInfo:
-    id: str = field(default_factory=lambda: str(uuid4()))
-    connector_id: str = None
-    dataset_id: str = None
-    location: str = None
-    format: str = None
-    file_size_kb: int = 0
-    in_time: datetime = field(default_factory=datetime.now)
-    start_processing_time: datetime = None
-    end_processing_time: datetime = None
-    download_time: float = 0.0
-    file_hash: str = None
-    num_of_retries: int = 0
-    tags: List[Tag] = field(default_factory=list)
-
-    def to_json(self):
-        return {
-            'id': self.id,
-            'connector_id': self.connector_id,
-            'dataset_id': self.dataset_id,
-            'location': self.location,
-            'format': self.format,
-            'file_size_kb': self.file_size_kb,
-            'in_time': self.in_time,
-            'download_time': self.download_time,
-            'start_processing_time': self.start_processing_time,
-            'end_processing_time': self.end_processing_time,
-            'file_hash': self.file_hash,
-            'num_of_retries': self.num_of_retries,
-            'tags': [tag.__dict__ for tag in self.tags]
-        }

From 6952d73c8b24fd4213f3f8d6c069ec44de2627ef Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 16:06:51 +0530
Subject: [PATCH 33/59] obsrv-labs/issue-tracker#2: Cleanup

---
 object_store_connector/models/object_info.py | 61 ++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 object_store_connector/models/object_info.py

diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
new file mode 100644
index 0000000..ad3f252
--- /dev/null
+++ b/object_store_connector/models/object_info.py
@@ -0,0 +1,61 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List
+from uuid import uuid4
+
+
+@dataclass
+class Tag:
+    key: str
+    value: str
+
+    def to_dict(self):
+        return {
+            'key': self.key,
+            'value': self.value
+        }
+
+    def to_aws(self):
+        return {
+            'Key': self.key,
+            'Value': self.value
+        }
+    def to_gcs(self):
+        return {
+            'Key': self.key,
+            'Value': self.value
+        }
+
+
+@dataclass
+class ObjectInfo:
+    id: str = field(default_factory=lambda: str(uuid4()))
+    connector_id: str = None
+    dataset_id: str = None
+    location: str = None
+    format: str = None
+    file_size_kb: int = 0
+    in_time: datetime = field(default_factory=datetime.now)
+    start_processing_time: datetime = None
+    end_processing_time: datetime = None
+    download_time: float = 0.0
+    file_hash: str = None
+    num_of_retries: int = 0
+    tags: List[Tag] = field(default_factory=list)
+
+    def to_json(self):
+        return {
+            'id': self.id,
+            'connector_id': self.connector_id,
+            'dataset_id': self.dataset_id,
+            'location': self.location,
+            'format': self.format,
+            'file_size_kb': self.file_size_kb,
+            'in_time': self.in_time,
+            'download_time': self.download_time,
+            'start_processing_time': self.start_processing_time,
+            'end_processing_time': self.end_processing_time,
+            'file_hash': self.file_hash,
+            'num_of_retries': self.num_of_retries,
+            'tags': [tag.__dict__ for tag in self.tags]
+        }

From d72f7c78ea46951f8ac5216010af0a4a377f1b1b Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 16:42:56 +0530
Subject: [PATCH 34/59] obsrv-labs/issue-tracker#2: Model for gcs

---
 object_store_connector/models/object_info.py | 42 ++++++++------------
 1 file changed, 17 insertions(+), 25 deletions(-)

diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
index ad3f252..09b5107 100644
--- a/object_store_connector/models/object_info.py
+++ b/object_store_connector/models/object_info.py
@@ -10,21 +10,13 @@ class Tag:
     value: str
 
     def to_dict(self):
-        return {
-            'key': self.key,
-            'value': self.value
-        }
+        return {"key": self.key, "value": self.value}
 
     def to_aws(self):
-        return {
-            'Key': self.key,
-            'Value': self.value
-        }
+        return {"Key": self.key, "Value": self.value}
+
     def to_gcs(self):
-        return {
-            'Key': self.key,
-            'Value': self.value
-        }
+        return {"Key": self.key, "Value": self.value}
 
 
 @dataclass
@@ -45,17 +37,17 @@ class ObjectInfo:
 
     def to_json(self):
         return {
-            'id': self.id,
-            'connector_id': self.connector_id,
-            'dataset_id': self.dataset_id,
-            'location': self.location,
-            'format': self.format,
-            'file_size_kb': self.file_size_kb,
-            'in_time': self.in_time,
-            'download_time': self.download_time,
-            'start_processing_time': self.start_processing_time,
-            'end_processing_time': self.end_processing_time,
-            'file_hash': self.file_hash,
-            'num_of_retries': self.num_of_retries,
-            'tags': [tag.__dict__ for tag in self.tags]
+            "id": self.id,
+            "connector_id": self.connector_id,
+            "dataset_id": self.dataset_id,
+            "location": self.location,
+            "format": self.format,
+            "file_size_kb": self.file_size_kb,
+            "in_time": self.in_time,
+            "download_time": self.download_time,
+            "start_processing_time": self.start_processing_time,
+            "end_processing_time": self.end_processing_time,
+            "file_hash": self.file_hash,
+            "num_of_retries": self.num_of_retries,
+            "tags": [tag.__dict__ for tag in self.tags],
         }

From f5622f7e5615905e7d628c162cdb904c354e3baf Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 20 May 2024 16:47:56 +0530
Subject: [PATCH 35/59] obsrv-labs/issue-tracker#2: gcs connector

---
 object_store_connector/config/config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/object_store_connector/config/config.yaml b/object_store_connector/config/config.yaml
index 23d21df..7f7e2e6 100644
--- a/object_store_connector/config/config.yaml
+++ b/object_store_connector/config/config.yaml
@@ -13,9 +13,9 @@ kafka:
     compression: snappy
     max-request-size: 1000000 # 1MB {1M: 1000000, 10M: 10000000, 5M: 5000000}
 
-obsrv_encryption_key: random_thirty_two_encrypt_string
+obsrv_encryption_key: strong_encryption_key_to_encrypt
 
-connector_instance_id: s3.new-york-taxi-data.1
+connector_instance_id: gcs.new-york-taxi-data.1
 
 building-block: rv-test
-env: local
\ No newline at end of file
+env: local

From 20149accf8ecc270d74c60449bc49b7ac496c124 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Tue, 21 May 2024 11:39:44 +0530
Subject: [PATCH 36/59] obsrv-labs/issue-tracker#6:  fix:  updated the config,
 poetry dependencies and made changes in the connector file

---
 object_store_connector/config/config.yaml    |   2 +-
 object_store_connector/connector.py          |  12 +-
 object_store_connector/models/object_info.py |  15 +-
 object_store_connector/provider/azure.py     |  13 +-
 poetry.lock                                  | 337 ++++++++++++++++++-
 pyproject.toml                               |   3 +-
 6 files changed, 358 insertions(+), 24 deletions(-)

diff --git a/object_store_connector/config/config.yaml b/object_store_connector/config/config.yaml
index d09d89b..bcd1589 100644
--- a/object_store_connector/config/config.yaml
+++ b/object_store_connector/config/config.yaml
@@ -6,7 +6,7 @@ postgres:
   port: 5432
 
 kafka:
-  broker-servers: localhost:9092
+  bootstrap-servers: localhost:9092
   telemetry-topic: obsrv-connectors-telemetry
   connector-metrics-topic: obsrv-connectors-metrics
   producer:
diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index 3d2feb4..158e5b8 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -40,12 +40,12 @@ def process(
         connector_config: Dict[Any, Any],
         metrics_collector: MetricsCollector,
     ) -> Iterator[DataFrame]:
-        # if (
-        #     ctx.state.get_state("status", default_value=self.not_running_state)
-        #     == self.running_state
-        # ):
-        #     logger.info("Connector is already running. Skipping processing.")
-        #     return
+        if (
+            ctx.state.get_state("status", default_value=self.not_running_state)
+            == self.running_state
+        ):
+            logger.info("Connector is already running. Skipping processing.")
+            return
 
         ctx.state.put_state("status", self.running_state)
         ctx.state.save_state()
diff --git a/object_store_connector/models/object_info.py b/object_store_connector/models/object_info.py
index 357d4e0..f78286f 100644
--- a/object_store_connector/models/object_info.py
+++ b/object_store_connector/models/object_info.py
@@ -10,14 +10,15 @@ class Tag:
     value: str
 
     def to_dict(self):
-        return {"key": self.key, "value": self.value}
-
-    def to_azure(self):
         return {
-            'Key': self.key,
-            'Value': self.value
-        }
-
+            "key": self.key, 
+            "value": self.value}
+    
+    def to_aws(self):
+        return {
+            "Key": self.key, 
+            "Value": self.value}
+    
     
 @dataclass
 class ObjectInfo:
diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 6d02d2a..02c79fd 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -1,6 +1,6 @@
 from azure.core.exceptions import AzureError
 from typing import Dict, List, Any
-from azure.storage.blob import ContainerClient, BlobClient, BlobServiceClient
+from azure.storage.blob import ContainerClient, BlobClient
 from pyspark.sql import DataFrame, SparkSession
 from provider.blob_provider import BlobProvider
 from models.object_info import ObjectInfo,Tag
@@ -9,7 +9,7 @@
 from obsrv.connector import ConnectorContext, MetricsCollector
 from obsrv.common import ObsrvException
 from obsrv.models import ErrorData
-import json
+
 
 
 
@@ -67,7 +67,7 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
             {"key": "object_path", "value": object_path}
         ]
         
-        # file_format = self.connector_config.get("fileFormat", {}).get("type", "jsonl")
+        
         api_calls, errors, records_count = 0, 0, 0
 
         try:
@@ -75,14 +75,11 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
                 df = sc.read.format("json").load(object_path)
             elif file_format == "json":
                 df = sc.read.format("json").option("multiLine", False).load(object_path)
-            elif file_format == "json.gz":
-                df = sc.read.format("json").option("compression", "gzip").option("multiLine", True).load(object_path)
             elif file_format == "csv":
                 df = sc.read.format("csv").option("header", True).load(object_path)
             elif file_format == "parquet":
                 df = sc.read.parquet(object_path)
-            elif file_format == "csv.gz":
-                df = sc.read.format("csv").option("header", True).option("compression", "gzip").load(object_path)
+            
             else:
                 raise ObsrvException(ErrorData("UNSUPPORTED_FILE_FORMAT", f"unsupported file format: {file_format}"))
             records_count = df.count()
@@ -195,7 +192,7 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             
             return [Tag(key,value) for key,value in tags.items()]
 
-        #Need to check on the exception.message    
+          
         except AzureError as exception:
             errors += 1
             labels += [
diff --git a/poetry.lock b/poetry.lock
index 74a84fd..7da0315 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,5 +1,44 @@
 # This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
 
+[[package]]
+name = "azure-core"
+version = "1.30.1"
+description = "Microsoft Azure Core Library for Python"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "azure-core-1.30.1.tar.gz", hash = "sha256:26273a254131f84269e8ea4464f3560c731f29c0c1f69ac99010845f239c1a8f"},
+    {file = "azure_core-1.30.1-py3-none-any.whl", hash = "sha256:7c5ee397e48f281ec4dd773d67a0a47a0962ed6fa833036057f9ea067f688e74"},
+]
+
+[package.dependencies]
+requests = ">=2.21.0"
+six = ">=1.11.0"
+typing-extensions = ">=4.6.0"
+
+[package.extras]
+aio = ["aiohttp (>=3.0)"]
+
+[[package]]
+name = "azure-storage-blob"
+version = "12.20.0"
+description = "Microsoft Azure Blob Storage Client Library for Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "azure-storage-blob-12.20.0.tar.gz", hash = "sha256:eeb91256e41d4b5b9bad6a87fd0a8ade07dd58aa52344e2c8d2746e27a017d3b"},
+    {file = "azure_storage_blob-12.20.0-py3-none-any.whl", hash = "sha256:de6b3bf3a90e9341a6bcb96a2ebe981dffff993e9045818f6549afea827a52a9"},
+]
+
+[package.dependencies]
+azure-core = ">=1.28.0"
+cryptography = ">=2.1.4"
+isodate = ">=0.6.1"
+typing-extensions = ">=4.6.0"
+
+[package.extras]
+aio = ["azure-core[aio] (>=1.28.0)"]
+
 [[package]]
 name = "boto3"
 version = "1.34.105"
@@ -38,6 +77,180 @@ urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >
 [package.extras]
 crt = ["awscrt (==0.20.9)"]
 
+[[package]]
+name = "certifi"
+version = "2024.2.2"
+description = "Python package for providing Mozilla's CA Bundle."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"},
+    {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"},
+]
+
+[[package]]
+name = "cffi"
+version = "1.16.0"
+description = "Foreign Function Interface for Python calling C code."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"},
+    {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"},
+    {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"},
+    {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"},
+    {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"},
+    {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"},
+    {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"},
+    {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"},
+    {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"},
+    {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"},
+    {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"},
+    {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"},
+    {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"},
+    {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"},
+    {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"},
+    {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"},
+    {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"},
+    {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"},
+    {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"},
+    {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"},
+    {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"},
+    {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"},
+    {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"},
+    {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"},
+    {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"},
+    {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"},
+    {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"},
+    {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"},
+    {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"},
+    {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"},
+    {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"},
+    {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"},
+    {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"},
+    {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"},
+    {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"},
+    {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"},
+    {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"},
+    {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"},
+    {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"},
+    {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"},
+    {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"},
+    {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"},
+    {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"},
+    {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"},
+    {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"},
+    {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"},
+    {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"},
+    {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"},
+    {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"},
+    {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"},
+    {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"},
+    {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"},
+]
+
+[package.dependencies]
+pycparser = "*"
+
+[[package]]
+name = "charset-normalizer"
+version = "3.3.2"
+description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"},
+    {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"},
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -113,6 +326,60 @@ files = [
 [package.extras]
 toml = ["tomli"]
 
+[[package]]
+name = "cryptography"
+version = "42.0.7"
+description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"},
+    {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"},
+    {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"},
+    {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"},
+    {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"},
+    {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"},
+    {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"},
+    {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"},
+    {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"},
+    {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"},
+    {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"},
+    {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"},
+    {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"},
+    {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"},
+    {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"},
+    {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"},
+    {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"},
+    {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"},
+    {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"},
+    {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"},
+    {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"},
+    {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"},
+    {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"},
+    {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"},
+    {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"},
+    {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"},
+    {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"},
+    {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"},
+    {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"},
+    {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"},
+    {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"},
+    {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"},
+]
+
+[package.dependencies]
+cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""}
+
+[package.extras]
+docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"]
+docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"]
+nox = ["nox"]
+pep8test = ["check-sdist", "click", "mypy", "ruff"]
+sdist = ["build"]
+ssh = ["bcrypt (>=3.1.5)"]
+test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"]
+test-randomorder = ["pytest-randomly"]
+
 [[package]]
 name = "exceptiongroup"
 version = "1.2.1"
@@ -127,6 +394,17 @@ files = [
 [package.extras]
 test = ["pytest (>=6)"]
 
+[[package]]
+name = "idna"
+version = "3.7"
+description = "Internationalized Domain Names in Applications (IDNA)"
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"},
+    {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"},
+]
+
 [[package]]
 name = "iniconfig"
 version = "2.0.0"
@@ -138,6 +416,20 @@ files = [
     {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
 ]
 
+[[package]]
+name = "isodate"
+version = "0.6.1"
+description = "An ISO 8601 date/time/duration parser and formatter"
+optional = false
+python-versions = "*"
+files = [
+    {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"},
+    {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"},
+]
+
+[package.dependencies]
+six = "*"
+
 [[package]]
 name = "jmespath"
 version = "1.0.1"
@@ -175,6 +467,17 @@ files = [
 dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
+[[package]]
+name = "pycparser"
+version = "2.22"
+description = "C parser in Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},
+    {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
+]
+
 [[package]]
 name = "pytest"
 version = "8.2.0"
@@ -211,6 +514,27 @@ files = [
 [package.dependencies]
 six = ">=1.5"
 
+[[package]]
+name = "requests"
+version = "2.32.1"
+description = "Python HTTP for Humans."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "requests-2.32.1-py3-none-any.whl", hash = "sha256:21ac9465cdf8c1650fe1ecde8a71669a93d4e6f147550483a2967d08396a56a5"},
+    {file = "requests-2.32.1.tar.gz", hash = "sha256:eb97e87e64c79e64e5b8ac75cee9dd1f97f49e289b083ee6be96268930725685"},
+]
+
+[package.dependencies]
+certifi = ">=2017.4.17"
+charset-normalizer = ">=2,<4"
+idna = ">=2.5,<4"
+urllib3 = ">=1.21.1,<3"
+
+[package.extras]
+socks = ["PySocks (>=1.5.6,!=1.5.7)"]
+use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
+
 [[package]]
 name = "s3transfer"
 version = "0.10.1"
@@ -250,6 +574,17 @@ files = [
     {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
 ]
 
+[[package]]
+name = "typing-extensions"
+version = "4.11.0"
+description = "Backported and Experimental Type Hints for Python 3.8+"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"},
+    {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"},
+]
+
 [[package]]
 name = "urllib3"
 version = "2.2.1"
@@ -270,4 +605,4 @@ zstd = ["zstandard (>=0.18.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "435ba3dcfaa3224044b34dfbaa4e0b66931931b730fe0c4a029bb2dcf685b5e0"
+content-hash = "bfee156550f4d9533314ff0c47dd53e872e764a3640e9711a85be6efdda0d52e"
diff --git a/pyproject.toml b/pyproject.toml
index e2713b8..967864e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,9 +6,10 @@ authors = ["Ravi Mula <ravismula@users.noreply.github.com>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.12"
 boto3 = "^1.34.82"
 coverage = "^7.5.0"
+azure-storage-blob= "^12.20.0"
 
 
 [tool.poetry.group.dev.dependencies]

From 2ba2530e3c07c7b8695191cf1d8b213eb6ccf010 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Wed, 22 May 2024 18:35:42 +0530
Subject: [PATCH 37/59] obsrv-labs/issue-tracker#6: feat: added the test folder

---
 object_store_connector/config/config.yaml     |   2 +-
 object_store_connector/connector.py           |   9 +-
 object_store_connector/provider/azure.py      |   4 +-
 .../provider/blob_provider.py                 |   2 +-
 object_store_connector/provider/s3.py         |   4 +-
 tests/batch_setup.py                          |  52 +++++
 tests/config/config.template.yaml             |  21 ++
 tests/create_tables.py                        | 184 ++++++++++++++++++
 tests/test_azure.py                           | 100 ++++++++++
 9 files changed, 367 insertions(+), 11 deletions(-)
 create mode 100644 tests/batch_setup.py
 create mode 100644 tests/config/config.template.yaml
 create mode 100644 tests/create_tables.py
 create mode 100644 tests/test_azure.py

diff --git a/object_store_connector/config/config.yaml b/object_store_connector/config/config.yaml
index bcd1589..d09d89b 100644
--- a/object_store_connector/config/config.yaml
+++ b/object_store_connector/config/config.yaml
@@ -6,7 +6,7 @@ postgres:
   port: 5432
 
 kafka:
-  bootstrap-servers: localhost:9092
+  broker-servers: localhost:9092
   telemetry-topic: obsrv-connectors-telemetry
   connector-metrics-topic: obsrv-connectors-metrics
   producer:
diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index 158e5b8..9c7e2fd 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -3,20 +3,19 @@
 import time
 from typing import Any, Dict, Iterator
 
-from models.object_info import ObjectInfo
+from object_store_connector.models.object_info import ObjectInfo
 from obsrv.common import ObsrvException
 from obsrv.connector import ConnectorContext, MetricsCollector
 from obsrv.connector.batch import ISourceConnector
 from obsrv.models import ErrorData, ExecutionState, StatusCode
 from obsrv.utils import LoggerController
-from provider.s3 import S3
+from object_store_connector.provider.s3 import S3
 from pyspark.conf import SparkConf
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import lit
 
-from provider.s3 import S3
-from provider.azure import AzureBlobStorage
-from models.object_info import ObjectInfo
+from object_store_connector.provider.azure import AzureBlobStorage
+from object_store_connector.models.object_info import ObjectInfo
 logger = LoggerController(__name__)
 
 MAX_RETRY_COUNT = 10
diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 02c79fd..a80cafa 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -2,8 +2,8 @@
 from typing import Dict, List, Any
 from azure.storage.blob import ContainerClient, BlobClient
 from pyspark.sql import DataFrame, SparkSession
-from provider.blob_provider import BlobProvider
-from models.object_info import ObjectInfo,Tag
+from object_store_connector.provider.blob_provider import BlobProvider
+from object_store_connector.models.object_info import ObjectInfo,Tag
 from pyspark.conf import SparkConf
 from obsrv.job.batch import get_base_conf
 from obsrv.connector import ConnectorContext, MetricsCollector
diff --git a/object_store_connector/provider/blob_provider.py b/object_store_connector/provider/blob_provider.py
index c4e7a7e..1ac96fd 100644
--- a/object_store_connector/provider/blob_provider.py
+++ b/object_store_connector/provider/blob_provider.py
@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List
 
-from models.object_info import ObjectInfo, Tag
+from object_store_connector.models.object_info import ObjectInfo, Tag
 from obsrv.connector import MetricsCollector
 from pyspark.sql import DataFrame, SparkSession
 
diff --git a/object_store_connector/provider/s3.py b/object_store_connector/provider/s3.py
index 5b7793c..41a48a1 100644
--- a/object_store_connector/provider/s3.py
+++ b/object_store_connector/provider/s3.py
@@ -3,12 +3,12 @@
 
 import boto3
 from botocore.exceptions import BotoCoreError, ClientError
-from models.object_info import ObjectInfo, Tag
+from object_store_connector.models.object_info import ObjectInfo, Tag
 from obsrv.common import ObsrvException
 from obsrv.connector import ConnectorContext, MetricsCollector
 from obsrv.job.batch import get_base_conf
 from obsrv.models import ErrorData
-from provider.blob_provider import BlobProvider
+from object_store_connector.provider.blob_provider import BlobProvider
 from pyspark.conf import SparkConf
 from pyspark.sql import DataFrame, SparkSession
 
diff --git a/tests/batch_setup.py b/tests/batch_setup.py
new file mode 100644
index 0000000..fbb99c5
--- /dev/null
+++ b/tests/batch_setup.py
@@ -0,0 +1,52 @@
+import os
+
+import pytest
+import yaml
+from testcontainers.kafka import KafkaContainer
+from testcontainers.postgres import PostgresContainer
+
+from tests.create_tables import create_tables
+
+# import psycopg2
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_obsrv_database(request):
+    postgres = PostgresContainer("postgres:latest")
+    kafka = KafkaContainer("confluentinc/cp-kafka:latest")
+
+    postgres.start()
+    kafka.start()
+
+    with open(
+        os.path.join(os.path.dirname(__file__), "config/config.template.yaml")
+    ) as config_file:
+        config = yaml.safe_load(config_file)
+
+        # config["connector-instance-id"] = "test.new-york-taxi-data.1"
+
+        config["postgres"]["host"] = postgres.get_container_host_ip()
+        config["postgres"]["port"] = postgres.get_exposed_port(5432)
+        config["postgres"]["user"] = postgres.username
+        config["postgres"]["password"] = postgres.password
+        config["postgres"]["dbname"] = postgres.dbname
+        config["kafka"]["broker-servers"] = kafka.get_bootstrap_server()
+
+    with open(
+        os.path.join(os.path.dirname(__file__), "config/config.yaml"), "w"
+    ) as config_file:
+        yaml.dump(config, config_file)
+
+    create_tables(config)
+
+    # clean up
+    def remove_container():
+        print("Cleaning up the containers")
+        postgres.stop()
+        kafka.stop()
+        try:
+            os.remove(os.path.join(os.path.dirname(__file__), "config/config.yaml"))
+        except FileNotFoundError:
+            print("config file already removed")
+
+    request.addfinalizer(remove_container)
diff --git a/tests/config/config.template.yaml b/tests/config/config.template.yaml
new file mode 100644
index 0000000..a91d7ea
--- /dev/null
+++ b/tests/config/config.template.yaml
@@ -0,0 +1,21 @@
+postgres:
+  dbname: postgres
+  user: postgres
+  password: postgres
+  host: localhost
+  port: 5432
+
+kafka:
+  broker-servers: localhost:9092
+  telemetry-topic: azure.telemetry
+  connector-metrics-topic: azure.metrics
+  producer:
+    compression: gzip
+    max-request-size: 1000000 # 1MB {1M: 1000000, 10M: 10000000, 5M: 5000000}
+
+obsrv_encryption_key: random_32_byte_encryption_string
+
+connector_instance_id: azure.new-york-taxi-data.1
+
+building-block: py-sdk-test
+env: local
\ No newline at end of file
diff --git a/tests/create_tables.py b/tests/create_tables.py
new file mode 100644
index 0000000..feacf6a
--- /dev/null
+++ b/tests/create_tables.py
@@ -0,0 +1,184 @@
+import json
+import os
+
+import psycopg2
+import yaml
+
+from obsrv.utils import EncryptionUtil
+from testcontainers.azurite import AzuriteContainer
+from azure.storage.blob import *
+
+def init_azurite():
+    ## Init azurite
+    azurite= AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:latest")
+    azurite.start()
+
+    blob_port= azurite.get_exposed_port(10000)
+    conn_str=azurite.get_connection_string()
+    acc_name=azurite.account_name
+    acc_key=azurite.account_key
+
+    azure_conf = {
+        "AccountName":acc_name,
+        "AccountKey":acc_key,
+        "BlobEndpoint":f"http://{azurite.get_container_host_ip()}:{blob_port}/{acc_name}"
+    }
+
+    blob_service_client=BlobServiceClient.from_connection_string(conn_str)
+    
+
+    ## Create a bucket
+    container_name="test-container"
+
+    container_client = blob_service_client.create_container(container_name)
+    containers = blob_service_client.list_containers()
+    # for container in containers:
+    #     print("container-name",container['name'])
+
+    ## Upload objects
+    blob_client = blob_service_client.get_blob_client(container_name,blob="nyt_data_100")
+    bloblist = ["/home/vince/repogit/obsrv-python-sdk/tests/sample_data/nyt_data_100.json"]
+    for blob in bloblist:
+        with open(blob, "rb") as data:
+            blob_client.upload_blob(data)
+    
+    return azure_conf
+
+
+
+
+def create_tables(config):
+    enc = EncryptionUtil(config["obsrv_encryption_key"])
+
+    #implement
+    azure_conf = init_azurite()
+    
+
+    datasets = """
+        CREATE TABLE IF NOT EXISTS datasets (
+            id TEXT PRIMARY KEY,
+            dataset_id TEXT,
+            type TEXT NOT NULL,
+            name TEXT,
+            validation_config JSON,
+            extraction_config JSON,
+            dedup_config JSON,
+            data_schema JSON,
+            denorm_config JSON,
+            router_config JSON,
+            dataset_config JSON,
+            status TEXT,
+            tags TEXT[],
+            data_version INT,
+            created_by TEXT,
+            updated_by TEXT,
+            created_date TIMESTAMP NOT NULL DEFAULT now(),
+            updated_date TIMESTAMP NOT NULL DEFAULT now(),
+            published_date TIMESTAMP NOT NULL DEFAULT now()
+        );"""
+
+    connector_registry = """
+        CREATE TABLE IF NOT EXISTS connector_registry (
+            id TEXT PRIMARY KEY,
+            version TEXT NOT NULL,
+            type TEXT NOT NULL,
+            category TEXT NOT NULL,
+            name TEXT NOT NULL,
+            description TEXT,
+            technology TEXT NOT NULL,
+            licence TEXT NOT NULL,
+            owner TEXT NOT NULL,
+            iconURL TEXT,
+            status TEXT NOT NULL,
+            created_by text NOT NULL,
+            updated_by text NOT NULL,
+            created_date TIMESTAMP NOT NULL DEFAULT now(),
+            updated_date TIMESTAMP NOT NULL,
+            live_date TIMESTAMP NOT NULL DEFAULT now()
+        );"""
+
+    connector_instances = """
+        CREATE TABLE IF NOT EXISTS connector_instances (
+            id TEXT PRIMARY KEY,
+            dataset_id TEXT NOT NULL REFERENCES datasets (id),
+            connector_id TEXT NOT NULL REFERENCES connector_registry (id),
+            data_format TEXT NOT NULL DEFAULT 'jsonl',
+            connector_type TEXT NOT NULL,
+            connector_config json NOT NULL,
+            operations_config json NOT NULL,
+            status TEXT NOT NULL,
+            connector_state JSON,
+            connector_stats JSON,
+            created_by text NOT NULL,
+            updated_by text NOT NULL,
+            created_date TIMESTAMP NOT NULL DEFAULT now(),
+            updated_date TIMESTAMP NOT NULL,
+            published_date TIMESTAMP NOT NULL DEFAULT now()
+        );"""
+
+    indexes = """
+        CREATE INDEX IF NOT EXISTS connector_registry_category ON connector_registry(category);
+        CREATE INDEX IF NOT EXISTS connector_registry_type ON connector_registry(type);
+        CREATE INDEX IF NOT EXISTS connector_instances_connector_id ON connector_instances(connector_id);
+    """
+
+    ins_ds = """
+        INSERT INTO datasets (id, dataset_id, type, name, validation_config, extraction_config, dedup_config, data_schema, denorm_config, router_config, dataset_config, tags, data_version, status, created_by, updated_by, created_date, updated_date, published_date) VALUES
+        ('new-york-taxi-data', 'new-york-taxi-data', 'dataset', 'new-york-taxi-data', '{"validate": true, "mode": "Strict", "validation_mode": "Strict"}', '{"is_batch_event": false}', '{"drop_duplicates": true, "dedup_key": "tripID", "dedup_period": 604800}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"tripID":{"type":"string","suggestions":[{"message":"The Property tripID appears to be uuid format type.","advice":"Suggest to not to index the high cardinal columns","resolutionType":"DEDUP","severity":"LOW","path":"properties.tripID"}],"arrival_format":"text","data_type":"string"}},"additionalProperties":false}', '{}', '{"topic": "new-york-taxi-data"}', '{"data_key": "", "timestamp_key": "tpep_pickup_datetime", "exclude_fields": [], "entry_topic": "azure.ingest", "redis_db_host": "obsrv-dedup-redis-master.redis.svc.cluster.local", "redis_db_port": 6379, "index_data": true, "redis_db": 0}', '{}', '1', 'Live', 'SYSTEM', 'SYSTEM', '2024-03-27 06:48:35.993478', '2024-03-27 06:48:35.993478', '2024-03-27 06:48:35.993478');
+    """
+
+    # ins_ds = """
+    #     INSERT INTO datasets (id, dataset_id, type, name, dataset_config, status, created_by, updated_by) VALUES
+    #     ('new-york-taxi-data', 'new-york-taxi-data', 'dataset', 'new-york-taxi-data', '{"entry_topic": "azure.ingest"}', 'Live', 'SYSTEM', 'SYSTEM');
+    # """
+
+    ins_cr = """
+        INSERT INTO connector_registry (id, version, type, category, name, description, technology, licence, owner, iconURL, status, created_by, updated_by, updated_date) VALUES
+        ('azure.1', '1', 'source', 'object', 'test_reader', 'test_reader', 'Python', 'Apache 2.0', 'ravi@obsrv.ai', 'http://localhost', 'Live', 'SYSTEM', 'SYSTEM', now());
+    """
+
+    connector_config = json.dumps(
+        {
+        "source":{
+            "type":"azure_blob",
+            "credentials":{
+                "account_name":azure_conf['AccountName'],
+                "account_key":azure_conf['AccountKey']
+            },
+        "containername":"test-container",
+        "blob_endpoint":azure_conf['BlobEndpoint'],
+        "prefix":"/"
+        }})
+    enc_config = enc.encrypt(connector_config)
+
+    ins_ci = """
+        INSERT INTO connector_instances (id, dataset_id, connector_id, connector_type, connector_config, operations_config, status, connector_state, connector_stats, created_by, updated_by, created_date, updated_date, published_date) VALUES
+        ('azure.new-york-taxi-data.1', 'new-york-taxi-data', 'azure.1', 'source', %s, '{}', 'Live', '{}', '{}', 'SYSTEM', 'SYSTEM', now(), now(), now()
+        );
+    """
+
+    with open(
+        os.path.join(os.path.dirname(__file__), "config/config.yaml"), "r"
+    ) as config_file:
+        config = yaml.safe_load(config_file)
+        conn = psycopg2.connect(
+            host=config["postgres"]["host"],
+            port=config["postgres"]["port"],
+            user=config["postgres"]["user"],
+            password=config["postgres"]["password"],
+            dbname=config["postgres"]["dbname"],
+        )
+
+        cur = conn.cursor()
+
+        cur.execute(datasets)
+        cur.execute(connector_registry)
+        cur.execute(connector_instances)
+        cur.execute(indexes)
+        cur.execute(ins_ds)
+        cur.execute(ins_cr)
+        cur.execute(ins_ci, (json.dumps(enc_config),))
+        # cur.execute("select * from connector_instances")
+
+        conn.commit()
+        conn.close()
diff --git a/tests/test_azure.py b/tests/test_azure.py
new file mode 100644
index 0000000..b3f17fa
--- /dev/null
+++ b/tests/test_azure.py
@@ -0,0 +1,100 @@
+import os
+import unittest
+from typing import Any, Dict
+
+import pytest
+import yaml
+from kafka import KafkaConsumer, TopicPartition
+from pyspark.conf import SparkConf
+from pyspark.sql import DataFrame, SparkSession
+
+from obsrv.connector import ConnectorContext, MetricsCollector
+from obsrv.connector.batch import ISourceConnector, SourceConnector
+from obsrv.job.batch import get_base_conf
+from tests.batch_setup import setup_obsrv_database  # noqa
+
+from object_store_connector.connector import ObjectStoreConnector
+
+
+class TestSource(ISourceConnector):
+    def process(
+        self,
+        sc: SparkSession,
+        ctx: ConnectorContext,
+        connector_config: Dict[Any, Any],
+        metrics_collector: MetricsCollector,
+    ) -> DataFrame:
+        df = sc.read.format("json").load("tests/sample_data/nyt_data_100.json.gz")
+        yield df
+
+        df1 = sc.read.format("json").load("tests/sample_data/nyt_data_100.json")
+
+        yield df1
+
+    def get_spark_conf(self, connector_config) -> SparkConf:
+        conf = get_base_conf()
+        return conf
+
+
+@pytest.mark.usefixtures("setup_obsrv_database")
+class TestBatchConnector(unittest.TestCase):
+
+    def test_source_connector(self):
+
+        connector = ObjectStoreConnector()
+        config_file_path = os.path.join(os.path.dirname(__file__), "config/config.yaml")
+        # SourceConnector.process(connector=connector, config_file_path=config_file_path)
+         ## import the objectstorreconnector class
+
+        ## update the config file with your instances
+
+        ## call SourceConnector.process* - get the logic from __main__py.
+
+        # connector = TestSource()
+        # config_file_path = os.path.join(os.path.dirname(__file__), "config/config.yaml")
+
+        config = yaml.safe_load(open(config_file_path))
+
+        config['connector_instance_id'] = "azure.new-york-taxi-data.1"
+
+        with open(
+            config_file_path, "w"
+        ) as config_file:
+            yaml.dump(config, config_file)
+
+
+        self.assertEqual(os.path.exists(config_file_path), True)
+
+        test_raw_topic = "azure.ingest"
+        test_metrics_topic = "azure.metrics"
+
+        kafka_consumer = KafkaConsumer(
+            bootstrap_servers=config["kafka"]["broker-servers"],
+            group_id="test-group",
+            enable_auto_commit=True,
+        )
+
+        trt_consumer = TopicPartition(test_raw_topic, 0)
+        tmt_consumer = TopicPartition(test_metrics_topic, 0)
+
+        kafka_consumer.assign([trt_consumer, tmt_consumer])
+
+        # kafka_consumer.seek_to_beginning()
+
+        SourceConnector.process(connector=connector, config_file_path=config_file_path)
+
+        print("Done processing....0")
+
+        import time
+        time.sleep(300)
+
+        # metrics = []
+        # all_messages = kafka_consumer.poll(timeout_ms=10000)
+
+        # for topic_partition, messages in all_messages.items():
+        #     for message in messages:
+        #         if topic_partition.topic == test_metrics_topic:
+        #             metrics.append(message.value)
+
+        # assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
+        # assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 1}

From 944af8c347e510042281ed60777748ea8661f104 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 27 May 2024 12:04:17 +0530
Subject: [PATCH 38/59] obsrv-labs/issue-tracker#2: Added tests folder

---
 tests/config/config.template.yaml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 tests/config/config.template.yaml

diff --git a/tests/config/config.template.yaml b/tests/config/config.template.yaml
new file mode 100644
index 0000000..045c465
--- /dev/null
+++ b/tests/config/config.template.yaml
@@ -0,0 +1,21 @@
+postgres:
+  dbname: postgres
+  user: postgres
+  password: postgres
+  host: localhost
+  port: 5432
+
+kafka:
+  broker-servers: localhost:9092
+  telemetry-topic: s3.telemetry
+  connector-metrics-topic: s3.metrics
+  producer:
+    compression: gzip
+    max-request-size: 1000000 # 1MB {1M: 1000000, 10M: 10000000, 5M: 5000000}
+
+obsrv_encryption_key: strong_encryption_key_to_encrypt
+
+connector_instance_id: s3.new-york-taxi-data.1
+
+building-block: py-sdk-test
+env: local

From 0797f2e1898c7c18ab0eaeeb64c403c1569bc8cf Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 27 May 2024 12:05:45 +0530
Subject: [PATCH 39/59] obsrv-labs/issue-tracker#2: Added tests folder

---
 tests/batch_setup.py | 51 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 tests/batch_setup.py

diff --git a/tests/batch_setup.py b/tests/batch_setup.py
new file mode 100644
index 0000000..6d74538
--- /dev/null
+++ b/tests/batch_setup.py
@@ -0,0 +1,51 @@
+import os
+import pytest
+import yaml
+from testcontainers.kafka import KafkaContainer
+from testcontainers.postgres import PostgresContainer
+from tests.create_tables import create_tables
+
+# import psycopg2
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_obsrv_database(request):
+    postgres = PostgresContainer("postgres:latest")
+    kafka = KafkaContainer("confluentinc/cp-kafka:latest")
+
+    postgres.start()
+    kafka.start()
+
+    with open(
+        os.path.join(os.path.dirname(__file__), "config/config.template.yaml")
+    ) as config_file:
+        config = yaml.safe_load(config_file)
+
+        config["connector-instance-id"] = "s3.new-york-taxi-data.1"
+
+        config["postgres"]["host"] = postgres.get_container_host_ip()
+        config["postgres"]["port"] = postgres.get_exposed_port(5432)
+        config["postgres"]["user"] = postgres.username
+        config["postgres"]["password"] = postgres.password
+        config["postgres"]["dbname"] = postgres.dbname
+        config["kafka"]["broker-servers"] = kafka.get_bootstrap_server()
+
+    
+
+    with open(
+        os.path.join(os.path.dirname(__file__), "config/config.yaml"), "w"
+    ) as config_file:
+        yaml.dump(config, config_file)
+    
+    create_tables(config)
+
+    # clean up
+    def remove_container():
+        postgres.stop()
+        kafka.stop()
+        try:
+            os.remove(os.path.join(os.path.dirname(__file__), "config/config.yaml"))
+        except FileNotFoundError:
+            print("config file already removed")
+
+    request.addfinalizer(remove_container)

From add55cf8b441b536c04848dbc71d7ff165766d04 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 27 May 2024 12:07:21 +0530
Subject: [PATCH 40/59] obsrv-labs/issue-tracker#2: Added tests folder

---
 tests/create_tables.py | 202 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 202 insertions(+)
 create mode 100644 tests/create_tables.py

diff --git a/tests/create_tables.py b/tests/create_tables.py
new file mode 100644
index 0000000..828c652
--- /dev/null
+++ b/tests/create_tables.py
@@ -0,0 +1,202 @@
+import json
+import os
+import psycopg2
+import yaml
+from obsrv.utils import EncryptionUtil
+from testcontainers.minio import MinioContainer
+from minio import Minio
+from minio.error import S3Error
+
+def init_minio(connector_config):
+
+    config=json.loads(connector_config)
+    minio_conf = Minio(
+        endpoint=config['source']['credentials']['endpoint'],
+        access_key=config['source']['credentials']['access_key'],
+        secret_key=config['source']['credentials']['secret_key'],
+        region=config['source']['credentials']['region'],
+        secure=False
+
+    )
+    bucket_name = config['source']['bucket']
+
+    #Create Bucket
+    try:
+        if not minio_conf.bucket_exists(bucket_name):
+            minio_conf.make_bucket(bucket_name)
+            print(f"Bucket '{bucket_name}' created successfully.")
+        else:
+            print(f"Bucket '{bucket_name}' already exists.")
+    except S3Error as e:
+        print(f"Error: {e}")
+
+    object_name="data.json"
+    file_path = "/root/GitRepo/obsrv-python-sdk/tests/sample_data/nyt_data_100.json"
+    c_type="application/json"
+ 
+    #Upload Objects
+    try:
+        minio_conf.fput_object(
+                bucket_name,
+                object_name,
+                file_path,
+                c_type
+            )
+        print(f"Uploaded {object_name} in the bucket.")
+    except S3Error as e:
+        print(f"Error uploading {object_name} : {e}")
+    except FileNotFoundError:
+        print(f"File not found: {object_name}")
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+
+    return minio_conf
+
+
+def create_tables(config):
+    enc = EncryptionUtil(config["obsrv_encryption_key"])
+    minio_container= MinioContainer("minio/minio:latest")
+    minio_container.start()
+    
+    connector_config = json.dumps({
+            "source": {
+                "type": "s3",
+                "bucket": "bucket-new",
+                "prefix": "/",
+                "prefix_format": "v1/",
+                "credentials": {
+                    "access_key": minio_container.access_key,
+                    "secret_key": minio_container.secret_key,
+                    "endpoint": f"{minio_container.get_container_host_ip()}:{minio_container.get_exposed_port(9000)}",
+                    "region":"us-east-1"
+                }
+            }
+        })
+
+    # Initialize Minio client
+    minio_conf = init_minio(connector_config)
+
+    datasets = """
+        CREATE TABLE IF NOT EXISTS datasets (
+            id TEXT PRIMARY KEY,
+            dataset_id TEXT,
+            type TEXT NOT NULL,
+            name TEXT,
+            validation_config JSON,
+            extraction_config JSON,
+            dedup_config JSON,
+            data_schema JSON,
+            denorm_config JSON,
+            router_config JSON,
+            dataset_config JSON,
+            status TEXT,
+            tags TEXT[],
+            data_version INT,
+            created_by TEXT,
+            updated_by TEXT,
+            created_date TIMESTAMP NOT NULL DEFAULT now(),
+            updated_date TIMESTAMP NOT NULL DEFAULT now(),
+            published_date TIMESTAMP NOT NULL DEFAULT now()
+        );"""
+
+    connector_registry = """
+        CREATE TABLE IF NOT EXISTS connector_registry (
+            id TEXT PRIMARY KEY,
+            version TEXT NOT NULL,
+            type TEXT NOT NULL,
+            category TEXT NOT NULL,
+            name TEXT NOT NULL,
+            description TEXT,
+            technology TEXT NOT NULL,
+            licence TEXT NOT NULL,
+            owner TEXT NOT NULL,
+            iconURL TEXT,
+            status TEXT NOT NULL,
+            created_by text NOT NULL,
+            updated_by text NOT NULL,
+            created_date TIMESTAMP NOT NULL DEFAULT now(),
+            updated_date TIMESTAMP NOT NULL,
+            live_date TIMESTAMP NOT NULL DEFAULT now()
+        );"""
+
+    connector_instances = """
+        CREATE TABLE IF NOT EXISTS connector_instances (
+            id TEXT PRIMARY KEY,
+            dataset_id TEXT NOT NULL REFERENCES datasets (id),
+            connector_id TEXT NOT NULL REFERENCES connector_registry (id),
+            data_format TEXT NOT NULL DEFAULT 'jsonl',
+            connector_type TEXT NOT NULL,
+            connector_config json NOT NULL,
+            operations_config json NOT NULL,
+            status TEXT NOT NULL,
+            connector_state JSON,
+            connector_stats JSON,
+            created_by text NOT NULL,
+            updated_by text NOT NULL,
+            created_date TIMESTAMP NOT NULL DEFAULT now(),
+            updated_date TIMESTAMP NOT NULL,
+            published_date TIMESTAMP NOT NULL DEFAULT now()
+        );"""
+
+    indexes = """
+        CREATE INDEX IF NOT EXISTS connector_registry_category ON connector_registry(category);
+        CREATE INDEX IF NOT EXISTS connector_registry_type ON connector_registry(type);
+        CREATE INDEX IF NOT EXISTS connector_instances_connector_id ON connector_instances(connector_id);
+    """
+
+    ins_ds = """
+        INSERT INTO datasets (id, dataset_id, type, name, validation_config, extraction_config, dedup_config, data_schema, denorm_config, router_config, dataset_config, tags, data_version, status, created_by, updated_by, created_date, updated_date, published_date) VALUES
+        ('new-york-taxi-data', 'new-york-taxi-data', 'dataset', 'new-york-taxi-data', '{"validate": true, "mode": "Strict", "validation_mode": "Strict"}', '{"is_batch_event": false}', '{"drop_duplicates": true, "dedup_key": "tripID", "dedup_period": 604800}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"tripID":{"type":"string","suggestions":[{"message":"The Property tripID appears to be uuid format type.","advice":"Suggest to not to index the high cardinal columns","resolutionType":"DEDUP","severity":"LOW","path":"properties.tripID"}],"arrival_format":"text","data_type":"string"}},"additionalProperties":false}', '{}', '{"topic": "new-york-taxi-data"}', '{"data_key": "", "timestamp_key": "tpep_pickup_datetime", "exclude_fields": [], "entry_topic": "s3.ingest", "redis_db_host": "obsrv-dedup-redis-master.redis.svc.cluster.local", "redis_db_port": 6379, "index_data": true, "redis_db": 0}', '{}', '1', 'Live', 'SYSTEM', 'SYSTEM', '2024-03-27 06:48:35.993478', '2024-03-27 06:48:35.993478', '2024-03-27 06:48:35.993478');
+    """
+
+    # ins_ds = """
+    #     INSERT INTO datasets (id, dataset_id, type, name, dataset_config, status, created_by, updated_by) VALUES
+    #     ('new-york-taxi-data', 'new-york-taxi-data', 'dataset', 'new-york-taxi-data', '{"entry_topic": "test.ingest"}', 'Live', 'SYSTEM', 'SYSTEM');
+    # """
+
+    ins_cr = """
+        INSERT INTO connector_registry (id, version, type, category, name, description, technology, licence, owner, iconURL, status, created_by, updated_by, updated_date) VALUES
+        ('s3.1', '1', 'source', 'object', 'test_reader', 'test_reader', 'Python', 'Apache 2.0', 'ravi@obsrv.ai', 'http://localhost', 'Live', 'SYSTEM', 'SYSTEM', now());
+    """
+
+
+    enc_config = enc.encrypt(connector_config)
+
+    ins_ci = """
+        INSERT INTO connector_instances (id, dataset_id, connector_id, connector_type, connector_config, operations_config, status, connector_state, connector_stats, created_by, updated_by, created_date, updated_date, published_date) VALUES
+        ('s3.new-york-taxi-data.1', 'new-york-taxi-data', 's3.1', 'source', %s, '{}', 'Live', '{}', '{}', 'SYSTEM', 'SYSTEM', now(), now(), now()
+        );
+    """
+  
+
+    with open(
+        os.path.join(os.path.dirname(__file__), "config/config.yaml"), "r"
+    ) as config_file:
+        config = yaml.safe_load(config_file)
+        conn = psycopg2.connect(
+            host=config["postgres"]["host"],
+            port=config["postgres"]["port"],
+            user=config["postgres"]["user"],
+            password=config["postgres"]["password"],
+            dbname=config["postgres"]["dbname"],
+        )
+
+        cur = conn.cursor()
+        try : 
+            cur.execute(datasets)
+            cur.execute(connector_registry)
+            cur.execute(connector_instances)
+            cur.execute(indexes)
+            cur.execute(ins_ds)
+            cur.execute(ins_cr)
+            cur.execute(ins_ci, (json.dumps(enc_config),))
+
+            conn.commit()
+        except Exception as e:
+            print(f"Error occurred: {e}")
+            conn.rollback()
+        finally:
+            conn.close()
+            minio_container.stop()
+
+        return

From 189820b6150a2f882aa98e96a578a851001bf782 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 27 May 2024 12:07:56 +0530
Subject: [PATCH 41/59] obsrv-labs/issue-tracker#2: Added tests folder

---
 tests/test_s3.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 tests/test_s3.py

diff --git a/tests/test_s3.py b/tests/test_s3.py
new file mode 100644
index 0000000..4dc45f0
--- /dev/null
+++ b/tests/test_s3.py
@@ -0,0 +1,87 @@
+import os
+import unittest
+from typing import Any, Dict
+import sys
+sys.path.insert(0, '/root/GitRepo/object-store-connector/object_store_connector')
+
+# from connector import Info
+
+
+import pytest
+import yaml
+from kafka import KafkaConsumer, TopicPartition
+from pyspark.conf import SparkConf
+from pyspark.sql import DataFrame, SparkSession
+from connector import ObjectStoreConnector
+from obsrv.connector import ConnectorContext, MetricsCollector
+from obsrv.connector.batch import ISourceConnector, SourceConnector
+from obsrv.job.batch import get_base_conf
+from tests.batch_setup import setup_obsrv_database  # noqa
+
+
+
+class ObjectStoreConnector(ISourceConnector):
+    def process(
+        self,
+        sc: SparkSession,
+        ctx: ConnectorContext,
+        connector_config: Dict[Any, Any],
+        metrics_collector: MetricsCollector,
+    ) -> DataFrame:
+        df = sc.read.format("json").load("tests/sample_data/nyt_data_100.json.gz")
+        yield df
+
+        df1 = sc.read.format("json").load("tests/sample_data/nyt_data_100.json")
+
+        yield df1
+
+    def get_spark_conf(self, connector_config) -> SparkConf:
+        conf = get_base_conf()
+        return conf
+
+
+@pytest.mark.usefixtures("setup_obsrv_database")
+class TestBatchConnector(unittest.TestCase):
+    def test_source_connector(self):
+        connector = ObjectStoreConnector()
+
+        config_file_path = os.path.join(os.path.dirname(__file__), "config/config.yaml")
+
+        SourceConnector.process(connector=connector, config_file_path=config_file_path)
+
+        config = yaml.safe_load(open(config_file_path))
+
+        self.assertEqual(os.path.exists(config_file_path), True)
+
+        test_raw_topic = "s3.ingest"
+        test_metrics_topic = "s3.metrics"
+
+        kafka_consumer = KafkaConsumer(
+            bootstrap_servers=config["kafka"]["broker-servers"],
+            group_id="test-group",
+            enable_auto_commit=True,
+        )
+
+        trt_consumer = TopicPartition(test_raw_topic, 0)
+        tmt_consumer = TopicPartition(test_metrics_topic, 0)
+
+        kafka_consumer.assign([trt_consumer, tmt_consumer])
+
+        SourceConnector.process(connector=connector, config_file_path=config_file_path)
+
+        # Consuming messages for debugging
+        all_messages = kafka_consumer.poll(timeout_ms=10000)
+
+        metrics = []
+        for topic_partition, messages in all_messages.items():
+            for message in messages:
+                if topic_partition.topic == test_metrics_topic:
+                    metrics.append(message.value)
+
+        # Check number of messages
+        print(f"Number of messages in {test_metrics_topic}: {len(metrics)}")
+        for message in metrics:
+            print(message)
+
+        assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
+        assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 1}

From e4e71e476d6049ef5bb1798b00f2ca9f9cc660d2 Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 27 May 2024 12:09:08 +0530
Subject: [PATCH 42/59] obsrv-labs/issue-tracker#2: Added Sample Data

---
 tests/sample_data/nyt_data_100.json | 100 ++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 tests/sample_data/nyt_data_100.json

diff --git a/tests/sample_data/nyt_data_100.json b/tests/sample_data/nyt_data_100.json
new file mode 100644
index 0000000..2c88bf3
--- /dev/null
+++ b/tests/sample_data/nyt_data_100.json
@@ -0,0 +1,100 @@
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"7","PULocationID":"261","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"32.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"10.14","tolls_amount":"0","total_amount":"43.94"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Teagan.Frami@hotmail.com","mobile":"343-570-1197 x16129"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-22 03:16:20","tpep_pickup_datetime":"2023-09-18 02:50:01","tripID":"25be90b3-7cbb-49d4-9785-b6fe995b0025","trip_distance":"11.13"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794206,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":654}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"263","PULocationID":"229","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3","tolls_amount":"0","total_amount":"13.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Lorena.Daugherty@gmail.com","mobile":"559.624.3003 x61721"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-21 02:33:16","tpep_pickup_datetime":"2023-12-11 02:23:34","tripID":"86dba649-4810-47dc-856a-64dc6f583a09","trip_distance":"1.94"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794209,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":657}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"243","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"27","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3","tolls_amount":"0","total_amount":"31.3"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Marques_Carter68@gmail.com","mobile":"640.971.7796 x1582"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-02 03:12:12","tpep_pickup_datetime":"2023-06-23 02:47:05","tripID":"a1e9eae9-63d9-4a67-ba4d-1d3a0e69a909","trip_distance":"8.38"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794213,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":661}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"87","PULocationID":"229","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"20.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"4.35","tolls_amount":"0","total_amount":"26.15"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Rosina93@hotmail.com","mobile":"1-587-969-1603"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-14 02:53:19","tpep_pickup_datetime":"2023-03-11 02:30:46","tripID":"26c2008a-c9c3-412e-ada4-28546d661647","trip_distance":"5.70"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794216,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":664}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"181","PULocationID":"142","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"49.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"5.76","total_amount":"56.56"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Keon_Gorczany@hotmail.com","mobile":"(268) 788-3566 x92699"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-25 02:56:52","tpep_pickup_datetime":"2023-12-19 02:12:23","tripID":"d2176c39-080d-484a-ba20-a9f01188d8d3","trip_distance":"16.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794220,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":668}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"236","PULocationID":"170","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.15","tolls_amount":"0","total_amount":"12.95"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Adele_Langworth-Weimann82@hotmail.com","mobile":"1-941-205-3594 x0006"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-29 02:31:45","tpep_pickup_datetime":"2023-03-13 02:23:13","tripID":"81cd141b-1e9c-438d-b5ee-8c5259177063","trip_distance":"2.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794223,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":671}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"142","PULocationID":"236","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.7","tolls_amount":"0","total_amount":"13.5"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Roosevelt_Spinka2@gmail.com","mobile":"996.831.1399 x956"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-28 02:44:07","tpep_pickup_datetime":"2023-06-29 02:33:46","tripID":"b439d3ef-94cc-4018-951d-cd5a00c1f87c","trip_distance":"2.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794227,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":675}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"4","PULocationID":"68","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"12.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"13.8"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Raphael_Borer35@hotmail.com","mobile":"609-381-7064"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-02 03:14:08","tpep_pickup_datetime":"2023-04-16 02:58:03","tripID":"992381b3-e012-43a2-be16-e5e0146e3681","trip_distance":"2.80"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794230,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":678}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"232","PULocationID":"68","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"17.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"18.8"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Nayeli.Kub-Weber41@hotmail.com","mobile":"414.556.6205 x636"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-10 02:41:44","tpep_pickup_datetime":"2023-04-19 02:19:47","tripID":"83157245-9369-415c-89fa-e2d9ae674821","trip_distance":"4.53"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794233,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":681}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"229","PULocationID":"232","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"13.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"14.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Nadia.Dicki6@gmail.com","mobile":"(622) 541-8901 x824"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-07 02:59:33","tpep_pickup_datetime":"2023-06-14 02:47:05","tripID":"60301101-b6f9-4327-96dc-daeb27937a04","trip_distance":"3.56"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794237,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":685}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"229","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.36","tolls_amount":"0","total_amount":"14.16"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Harold.Greenfelder52@hotmail.com","mobile":"498.394.3088 x841"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-16 02:11:24","tpep_pickup_datetime":"2023-06-21 02:00:55","tripID":"e2476cbc-c609-499b-bdf9-90d42bc60a4a","trip_distance":"2.83"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794244,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":692}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"33","PULocationID":"229","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"20","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"4.26","tolls_amount":"0","total_amount":"25.56"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Edwina_Jacobs56@hotmail.com","mobile":"207-669-3585"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-28 02:31:55","tpep_pickup_datetime":"2024-01-09 02:14:01","tripID":"3bdb160a-23f3-4827-8619-b6443594e71a","trip_distance":"6.28"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794287,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":735}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"158","PULocationID":"114","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.56","tolls_amount":"0","total_amount":"9.36"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Jazmin68@yahoo.com","mobile":"(998) 720-5079 x4750"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-04 02:55:20","tpep_pickup_datetime":"2023-02-10 02:48:24","tripID":"3068ea37-fc73-445f-801d-b6a5434b29bb","trip_distance":"1.14"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794290,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":738}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"45","PULocationID":"158","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"10.8"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Lexie42@yahoo.com","mobile":"477.656.8319 x51739"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-06 03:08:37","tpep_pickup_datetime":"2023-01-16 02:57:04","tripID":"28b0589f-9769-4a83-9cc7-78eee8eb3209","trip_distance":"1.97"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794294,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":742}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"43","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"20.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"4.36","tolls_amount":"0","total_amount":"26.16"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Donald_Schowalter31@gmail.com","mobile":"221-276-6460 x6513"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-04 02:20:24","tpep_pickup_datetime":"2023-08-28 01:57:19","tripID":"a9fed048-4b77-47b0-a025-1085af772b64","trip_distance":"5.65"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794298,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":746}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"75","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"19","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"22.3"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Nola.Metz@yahoo.com","mobile":"332.759.9239"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-15 02:44:03","tpep_pickup_datetime":"2023-02-19 02:21:58","tripID":"e761c2c6-c757-47d8-96eb-cbc4e1cd7c90","trip_distance":"5.35"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794301,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":749}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"79","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.15","tolls_amount":"0","total_amount":"12.95"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Ibrahim_Daugherty@gmail.com","mobile":"800.910.7891"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-02 02:26:09","tpep_pickup_datetime":"2023-01-06 02:15:28","tripID":"e91e19cf-9d02-498f-b2c0-708d10106ad5","trip_distance":"2.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794304,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":752}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"162","PULocationID":"234","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"9.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Jacynthe.Bashirian79@gmail.com","mobile":"1-964-329-9021"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-29 02:43:11","tpep_pickup_datetime":"2024-02-06 02:34:21","tripID":"83ce3138-98dc-4c64-b28c-269a029886c8","trip_distance":"1.60"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794307,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":755}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"50","PULocationID":"50","RatecodeID":"5","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0","fare_amount":"0","improvement_surcharge":"0.3","mta_tax":"0","tip_amount":"0","tolls_amount":"0","total_amount":"0.3"},"passenger_count":"1","payment_type":"3","primary_passenger":{"email":"Rogelio34@hotmail.com","mobile":"898.781.9202 x14465"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-30 02:15:42","tpep_pickup_datetime":"2023-01-10 02:15:41","tripID":"c4d3ad69-2bf7-4e72-82ff-90ce63841704","trip_distance":".00"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794310,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":758}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"162","PULocationID":"48","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"9.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Elyse_Prohaska@yahoo.com","mobile":"(357) 962-5072 x6405"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-18 02:32:47","tpep_pickup_datetime":"2023-12-08 02:22:14","tripID":"169f7e3a-6c53-4ab7-bda9-5911b784f30c","trip_distance":"1.30"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794313,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":761}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"161","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"3.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"4.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Ima1@gmail.com","mobile":"1-773-959-5605 x73199"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-07 02:40:03","tpep_pickup_datetime":"2023-07-18 02:38:18","tripID":"2743672b-c4d1-4ce8-adc1-f90b78d3a44b","trip_distance":".30"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794317,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":765}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"181","PULocationID":"232","RatecodeID":"3","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"36.5","improvement_surcharge":"0.3","mta_tax":"0","tip_amount":"7.45","tolls_amount":"0","total_amount":"44.75"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Lynn.McClure66@yahoo.com","mobile":"351-615-5695 x4474"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-04 03:14:04","tpep_pickup_datetime":"2023-07-06 02:52:04","tripID":"64dc9673-c0af-4dc6-b18e-b8a7f499dee9","trip_distance":"5.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794320,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":768}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"52","PULocationID":"33","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"3.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"4.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Madge_Bergstrom68@yahoo.com","mobile":"1-680-654-8869"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-08 02:14:26","tpep_pickup_datetime":"2023-07-14 02:13:08","tripID":"bbfc9834-4e11-4e68-bd28-ec927655f78b","trip_distance":".38"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794323,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":771}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"40","PULocationID":"52","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"5.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"6.8"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Della21@hotmail.com","mobile":"755.562.8568 x399"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-22 02:21:30","tpep_pickup_datetime":"2023-07-09 02:16:47","tripID":"8cd91fbc-927b-4e56-8172-a0350db5ad63","trip_distance":"1.15"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794327,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":775}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"255","PULocationID":"65","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.56","tolls_amount":"0","total_amount":"15.36"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Bernadine.Friesen@yahoo.com","mobile":"555-470-0062"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-27 02:40:30","tpep_pickup_datetime":"2023-02-22 02:31:12","tripID":"1f8e276b-346d-4e00-b5d5-fcd688efbe56","trip_distance":"3.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794330,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":778}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"256","PULocationID":"255","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.16","tolls_amount":"0","total_amount":"6.96"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Linda96@gmail.com","mobile":"1-642-292-8585"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-10 02:45:24","tpep_pickup_datetime":"2023-03-28 02:41:58","tripID":"3e76db01-516a-4672-8a7f-d3ddfb7d285d","trip_distance":".81"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794333,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":781}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"255","PULocationID":"256","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"5.3"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Rosetta_Beer@hotmail.com","mobile":"798.313.5258 x6339"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-04 02:50:26","tpep_pickup_datetime":"2023-06-01 02:47:31","tripID":"6b41ee87-6fa1-4f83-9447-2e1e98667d5c","trip_distance":".62"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794336,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":784}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"90","PULocationID":"255","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"17","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.66","tolls_amount":"0","total_amount":"21.96"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Carmen57@gmail.com","mobile":"978.415.9386 x8923"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-08 03:09:43","tpep_pickup_datetime":"2023-06-26 02:50:53","tripID":"d501b0eb-c941-4328-9d0b-83001f652918","trip_distance":"4.64"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794340,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":788}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"162","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.26","tolls_amount":"0","total_amount":"13.56"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Oliver.Labadie10@yahoo.com","mobile":"(366) 454-2917 x477"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-21 02:29:12","tpep_pickup_datetime":"2023-05-11 02:18:46","tripID":"e08cf524-22a7-4e75-86aa-47497b76f378","trip_distance":"2.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794343,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":791}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"239","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"9.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Dora26@yahoo.com","mobile":"(478) 966-5035 x192"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-10-08 02:41:45","tpep_pickup_datetime":"2023-08-06 02:33:12","tripID":"e2621721-e337-43b7-9c2f-eb2769bffd0f","trip_distance":"1.85"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794385,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":833}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"162","PULocationID":"239","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.7","tolls_amount":"0","total_amount":"13.5"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Isabella_Wunsch89@gmail.com","mobile":"521.252.8903 x15758"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-06 02:53:08","tpep_pickup_datetime":"2023-08-28 02:43:48","tripID":"5f8ead46-df6d-427c-8f4e-8c51485f3bc4","trip_distance":"2.41"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794389,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":837}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"237","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.26","tolls_amount":"0","total_amount":"7.56"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Marcelina_Cormier21@hotmail.com","mobile":"557-359-4984 x5088"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-05 03:02:05","tpep_pickup_datetime":"2024-01-11 02:57:55","tripID":"d0e83758-cf79-44e9-a138-2c8c84e7e1ab","trip_distance":".89"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794393,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":841}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"229","PULocationID":"141","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"5.3"},"passenger_count":"5","payment_type":"2","primary_passenger":{"email":"Evie.Pouros@yahoo.com","mobile":"277-513-8399 x256"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-27 02:09:02","tpep_pickup_datetime":"2023-04-09 02:06:09","tripID":"93968446-dafa-476f-9d7e-570304c3dfd8","trip_distance":".68"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794396,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":844}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"224","PULocationID":"229","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"11.8"},"passenger_count":"5","payment_type":"2","primary_passenger":{"email":"Christ.Kuvalis@gmail.com","mobile":"(441) 378-5258 x6569"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-17 02:24:27","tpep_pickup_datetime":"2023-11-29 02:10:40","tripID":"50587962-2be2-4719-a09b-b8eb35b7be10","trip_distance":"2.33"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794399,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":847}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"116","PULocationID":"107","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"34.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"7.16","tolls_amount":"0","total_amount":"42.96"},"passenger_count":"5","payment_type":"1","primary_passenger":{"email":"Alexandre_Walker@yahoo.com","mobile":"458-362-0622 x630"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-26 03:12:19","tpep_pickup_datetime":"2024-01-14 02:27:14","tripID":"da36b5dd-0704-400c-adb5-888be2f6ebf8","trip_distance":"9.17"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794402,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":850}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"262","PULocationID":"237","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"10.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Rosalinda79@hotmail.com","mobile":"(285) 935-0429 x620"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-04 02:22:29","tpep_pickup_datetime":"2024-01-11 02:13:41","tripID":"02412500-2ffe-45e7-bee5-a6666dc1d839","trip_distance":"2.14"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794406,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":854}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"90","PULocationID":"263","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.46","tolls_amount":"0","total_amount":"20.76"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Gerda55@hotmail.com","mobile":"551.761.2078 x469"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-24 02:48:40","tpep_pickup_datetime":"2023-08-27 02:29:15","tripID":"b21960c7-f5a0-4bca-9402-abf7bd246004","trip_distance":"3.93"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794409,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":857}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"90","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.36","tolls_amount":"0","total_amount":"14.16"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Corene27@gmail.com","mobile":"1-390-967-3132 x173"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-14 03:03:44","tpep_pickup_datetime":"2023-12-21 02:50:12","tripID":"d8253210-ce80-48bf-ad78-f9e9b02a7659","trip_distance":"2.01"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794412,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":860}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"234","PULocationID":"137","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"7.8"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Alicia9@hotmail.com","mobile":"1-967-665-8696 x087"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-24 02:10:42","tpep_pickup_datetime":"2023-04-26 02:02:57","tripID":"cb63a45c-d619-4526-88fb-7f020417342f","trip_distance":"1.07"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794416,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":864}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"234","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.16","tolls_amount":"0","total_amount":"12.96"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Rickey.Heller29@hotmail.com","mobile":"(315) 523-5370 x8087"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-11 02:25:05","tpep_pickup_datetime":"2023-09-30 02:12:49","tripID":"8840c605-b20b-41fd-a06e-c724e1a0a64b","trip_distance":"1.81"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794420,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":868}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"49","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"17.3"},"passenger_count":"4","payment_type":"2","primary_passenger":{"email":"Aurore4@hotmail.com","mobile":"284-324-0491 x19914"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-18 02:25:05","tpep_pickup_datetime":"2023-02-09 02:27:35","tripID":"e67ffdc5-4302-48a8-8f44-cee84c2fe73b","trip_distance":"3.84"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794423,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":871}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"234","PULocationID":"236","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"14.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"15.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Jerrod_Osinski@hotmail.com","mobile":"1-689-477-9312 x99969"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-30 02:37:11","tpep_pickup_datetime":"2023-01-27 02:20:17","tripID":"95ef5fab-051a-4e15-9a4a-9cf49ca65231","trip_distance":"3.78"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794426,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":874}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"232","PULocationID":"234","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"12.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Brody.Strosin@yahoo.com","mobile":"300-552-9715 x358"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-06 02:51:00","tpep_pickup_datetime":"2023-05-19 02:38:39","tripID":"7fa48c82-4c15-4204-840b-a633b344cced","trip_distance":"2.39"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794430,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":878}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"87","PULocationID":"232","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"12","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.66","tolls_amount":"0","total_amount":"15.96"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Karlee_Bosco@hotmail.com","mobile":"891-778-9086 x5622"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-09 03:05:21","tpep_pickup_datetime":"2023-11-24 02:54:32","tripID":"3600dd4d-d3d4-48d4-8220-f02200161d3a","trip_distance":"3.00"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794433,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":881}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"236","PULocationID":"48","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.46","tolls_amount":"0","total_amount":"14.76"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Madelyn.Ullrich@yahoo.com","mobile":"725.333.9195"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-09 02:23:56","tpep_pickup_datetime":"2023-05-27 02:14:49","tripID":"c1049b7b-cf57-486b-a154-cdf1309b6841","trip_distance":"3.42"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794437,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":885}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"150","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"55.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"14.2","tolls_amount":"0","total_amount":"71"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Waylon14@hotmail.com","mobile":"564-322-3411 x656"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-22 03:11:46","tpep_pickup_datetime":"2023-12-07 02:33:40","tripID":"9d42a2fd-9910-4c79-86d4-262e3b7f5fae","trip_distance":"19.97"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794440,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":888}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"147","PULocationID":"41","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"19","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"22.3"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Manley_Hackett71@gmail.com","mobile":"1-871-508-0977 x05891"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-09 02:22:37","tpep_pickup_datetime":"2023-06-04 02:00:35","tripID":"b8946252-73ef-4e89-9af2-22ade786e381","trip_distance":"4.96"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794484,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":932}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"244","PULocationID":"75","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"17.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Marcus67@hotmail.com","mobile":"546-381-5618 x705"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-19 03:00:20","tpep_pickup_datetime":"2023-05-01 02:49:25","tripID":"ac08ee2e-a254-4a8a-85c3-1003e93b8df3","trip_distance":"5.42"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794488,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":936}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"65","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"19.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1","tolls_amount":"0","total_amount":"23.75"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Rebeca78@gmail.com","mobile":"857.384.4490 x9274"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-21 02:37:59","tpep_pickup_datetime":"2023-01-10 02:15:26","tripID":"a14db847-b6b2-4377-a8f8-be62cbc6c58b","trip_distance":"5.08"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794491,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":939}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"170","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.66","tolls_amount":"0","total_amount":"9.96"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Claudie_Hegmann@hotmail.com","mobile":"1-273-731-1442 x2409"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-17 02:46:37","tpep_pickup_datetime":"2023-09-20 02:39:34","tripID":"6e4e34f8-2747-4298-9d46-3e52e1ad4a49","trip_distance":"1.41"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794495,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":943}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"238","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.46","tolls_amount":"0","total_amount":"20.76"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Antonetta_Davis@hotmail.com","mobile":"371-641-0671 x207"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-12 03:05:15","tpep_pickup_datetime":"2023-04-30 02:49:45","tripID":"68b10553-f67d-4acd-ae8a-bd11b02f805d","trip_distance":"5.05"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794499,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":947}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"48","PULocationID":"48","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.45","tolls_amount":"0","total_amount":"8.75"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Clovis.Grady7@yahoo.com","mobile":"344-298-3292"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-16 02:08:57","tpep_pickup_datetime":"2023-02-12 02:02:52","tripID":"6b6ab88d-6ae4-407a-9d96-cafc40925f26","trip_distance":".90"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794502,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":950}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"90","PULocationID":"48","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.65","tolls_amount":"0","total_amount":"15.95"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Eve_Erdman51@hotmail.com","mobile":"221.581.9132 x7964"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-31 02:25:57","tpep_pickup_datetime":"2023-05-13 02:11:55","tripID":"ad35aadc-0dcb-43f6-8140-47fc8f10d3ad","trip_distance":"2.20"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794505,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":953}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"229","PULocationID":"90","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"13","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"16.3"},"passenger_count":"4","payment_type":"1","primary_passenger":{"email":"Ernestina_Corwin@gmail.com","mobile":"1-240-291-8146 x5567"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-24 02:43:12","tpep_pickup_datetime":"2023-01-12 02:27:06","tripID":"4eb66446-fd61-4410-b056-ffc4f512eb80","trip_distance":"2.90"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794509,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":957}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"234","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.6","tolls_amount":"0","total_amount":"11.4"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Oscar.Witting@hotmail.com","mobile":"281-827-1431 x92694"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-08 02:22:54","tpep_pickup_datetime":"2023-05-25 02:14:07","tripID":"21963119-b955-47c7-8dda-d8804bfeadf7","trip_distance":"1.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794513,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":961}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"263","PULocationID":"162","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"11.3"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Stone81@yahoo.com","mobile":"889-393-8174 x1107"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-27 02:34:52","tpep_pickup_datetime":"2023-07-23 02:28:20","tripID":"2b8b5deb-9b99-488f-9a66-bf07b19fd97f","trip_distance":"2.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794516,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":964}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"263","PULocationID":"237","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.45","tolls_amount":"0","total_amount":"8.75"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Winfield_Mayer42@yahoo.com","mobile":"(618) 838-8049"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-07 02:37:52","tpep_pickup_datetime":"2023-08-06 02:32:47","tripID":"e3ca466a-f2b9-4e65-84dd-66201b11a92f","trip_distance":"1.20"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794519,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":967}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"41","PULocationID":"237","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.35","tolls_amount":"0","total_amount":"14.65"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Jaeden.Von@gmail.com","mobile":"1-504-649-6832"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-25 02:58:27","tpep_pickup_datetime":"2023-03-23 02:48:45","tripID":"3ecddca9-4247-475a-bd93-c572db096fc4","trip_distance":"2.70"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794522,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":970}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"87","PULocationID":"66","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"11.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Rosella.Rolfson-Gibson@yahoo.com","mobile":"(617) 788-8313 x2277"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-19 02:15:18","tpep_pickup_datetime":"2023-03-25 02:05:08","tripID":"5c3d9658-0579-4365-82f3-f567c6fab6b6","trip_distance":"2.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794526,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":974}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"37","PULocationID":"87","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"26","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"5.45","tolls_amount":"0","total_amount":"32.75"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Pinkie17@yahoo.com","mobile":"206-799-5614"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-28 02:42:36","tpep_pickup_datetime":"2023-12-30 02:17:57","tripID":"f2213841-8cae-4c86-aa17-ab8150b0353b","trip_distance":"8.20"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794529,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":977}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"17","PULocationID":"37","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"7.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Carmine44@gmail.com","mobile":"369-639-0569 x643"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-29 02:50:36","tpep_pickup_datetime":"2024-01-16 02:43:48","tripID":"d5f74519-08c3-4ee1-9e2d-9b84876e157e","trip_distance":"1.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794532,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":980}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"107","PULocationID":"211","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"9.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Ardith.Tromp@hotmail.com","mobile":"612-805-1546 x7728"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-08 03:03:30","tpep_pickup_datetime":"2023-05-30 02:54:05","tripID":"4934ae36-14f0-46ea-8087-497b87aa6d5e","trip_distance":"1.90"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794536,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":984}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"140","PULocationID":"229","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.26","tolls_amount":"0","total_amount":"7.56"},"passenger_count":"5","payment_type":"1","primary_passenger":{"email":"Reinhold12@hotmail.com","mobile":"(478) 224-3991 x93298"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-15 02:16:51","tpep_pickup_datetime":"2023-01-01 02:13:30","tripID":"1d290a9e-1244-46ef-b22c-08336ea56864","trip_distance":"1.08"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794539,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":987}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"230","PULocationID":"161","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.32","tolls_amount":"0","total_amount":"11.62"},"passenger_count":"5","payment_type":"1","primary_passenger":{"email":"Eric29@yahoo.com","mobile":"774.435.3330 x4929"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-30 02:44:36","tpep_pickup_datetime":"2023-04-29 02:33:54","tripID":"4ef82abf-f75e-425b-a0d6-9b8d4521addb","trip_distance":"1.39"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794542,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":990}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"234","PULocationID":"230","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"10.8"},"passenger_count":"5","payment_type":"2","primary_passenger":{"email":"Rashad.Klocko@gmail.com","mobile":"1-902-813-8402 x18121"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-18 02:59:59","tpep_pickup_datetime":"2023-06-16 02:48:50","tripID":"cca51610-3623-4353-befe-465bcbcb6e5b","trip_distance":"1.91"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794545,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":993}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"107","PULocationID":"249","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.45","tolls_amount":"0","total_amount":"14.75"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Germaine.Johnson16@yahoo.com","mobile":"1-309-548-7894 x77988"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-05 02:55:40","tpep_pickup_datetime":"2023-09-28 02:41:47","tripID":"3f2e3f93-9088-4ba1-aad0-779c04192b96","trip_distance":"2.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794587,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1035}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"237","PULocationID":"151","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"11.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Taylor.Green@yahoo.com","mobile":"(571) 811-5436 x433"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-06 02:10:38","tpep_pickup_datetime":"2023-09-02 02:01:05","tripID":"1eb1e4c1-9c23-4dd4-8915-0986404ac3e8","trip_distance":"2.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794591,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1039}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"226","PULocationID":"230","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"17.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"18.8"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Keenan.Brown60@yahoo.com","mobile":"447-236-4948 x26679"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-05 02:44:31","tpep_pickup_datetime":"2023-02-10 02:23:48","tripID":"54287dc9-743e-40b3-a2e7-16c8139a405d","trip_distance":"4.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794594,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1042}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"48","PULocationID":"211","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"15.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.36","tolls_amount":"0","total_amount":"20.16"},"passenger_count":"6","payment_type":"1","primary_passenger":{"email":"Joana.Murray16@gmail.com","mobile":"1-205-355-1857 x6037"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-28 02:29:36","tpep_pickup_datetime":"2023-09-21 02:11:57","tripID":"cfc6db54-05f3-4ec3-9fac-35bc0642b5b0","trip_distance":"4.39"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794598,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1046}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"186","PULocationID":"48","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"7.3"},"passenger_count":"6","payment_type":"2","primary_passenger":{"email":"Pietro34@gmail.com","mobile":"(214) 940-1846"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-30 02:42:32","tpep_pickup_datetime":"2023-06-01 02:35:36","tripID":"40ddb784-9baa-49cc-b445-ba7c3c3a5350","trip_distance":".74"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794601,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1049}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"239","PULocationID":"186","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"15","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.44","tolls_amount":"0","total_amount":"18.74"},"passenger_count":"6","payment_type":"1","primary_passenger":{"email":"Hunter60@gmail.com","mobile":"1-605-974-7545 x2362"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-10-05 03:00:57","tpep_pickup_datetime":"2023-09-05 02:43:53","tripID":"1b84cc94-63cd-4bf7-a5a2-9adf220323de","trip_distance":"3.77"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794604,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1052}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"107","PULocationID":"231","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.36","tolls_amount":"0","total_amount":"14.16"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Berneice0@hotmail.com","mobile":"355.597.9064 x85306"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-02 02:15:56","tpep_pickup_datetime":"2023-09-05 02:04:15","tripID":"09cd6414-1451-4e47-903b-8e6020b85f0c","trip_distance":"2.59"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794607,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1055}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"237","PULocationID":"137","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"5","tolls_amount":"0","total_amount":"16.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Josephine_Thompson@hotmail.com","mobile":"(816) 899-1039 x9110"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-15 02:32:57","tpep_pickup_datetime":"2023-12-19 02:22:15","tripID":"0ea57d99-2441-4a96-aa30-d50601ae805e","trip_distance":"2.60"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794610,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1058}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"223","PULocationID":"170","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"29","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"7.21","tolls_amount":"5.76","total_amount":"43.27"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Blaze29@yahoo.com","mobile":"550-296-4027 x96854"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-22 03:14:31","tpep_pickup_datetime":"2023-08-11 02:47:39","tripID":"e7cd1818-3ac4-4c26-be7e-2811d3cf6eb9","trip_distance":"9.49"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794614,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1062}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"114","PULocationID":"79","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.7","tolls_amount":"0","total_amount":"10"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Gerda98@yahoo.com","mobile":"(771) 247-2009"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-10-09 02:17:42","tpep_pickup_datetime":"2023-04-14 02:11:46","tripID":"8f1d4aeb-d3a9-41f1-b869-dceba333ddb2","trip_distance":".90"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794617,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1065}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"114","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.85","tolls_amount":"0","total_amount":"11.15"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Lina78@hotmail.com","mobile":"(737) 801-0055 x389"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-10-05 02:28:10","tpep_pickup_datetime":"2023-02-02 02:18:49","tripID":"39948704-84d9-439f-a8f1-a041ec9d6882","trip_distance":"1.30"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794620,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1068}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"238","PULocationID":"79","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"23.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1","tolls_amount":"0","total_amount":"25.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Zachary.Kozey@hotmail.com","mobile":"1-367-977-9321 x10540"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-25 02:59:40","tpep_pickup_datetime":"2023-09-07 02:33:22","tripID":"edc121f9-b36f-4b24-afa9-b9fbadbfa297","trip_distance":"6.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794624,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1072}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"61","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"28","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"7.01","tolls_amount":"5.76","total_amount":"42.07"},"passenger_count":"4","payment_type":"1","primary_passenger":{"email":"Ralph.Greenfelder@hotmail.com","mobile":"646.847.7460 x03211"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-16 02:35:52","tpep_pickup_datetime":"2023-03-09 02:10:17","tripID":"7f53bade-5ce2-4666-8c48-ed3af53255f4","trip_distance":"8.88"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794627,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1075}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"49","PULocationID":"49","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"5.3"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Jermaine.Homenick@yahoo.com","mobile":"264.624.0022 x756"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-05 02:41:11","tpep_pickup_datetime":"2023-04-22 02:39:34","tripID":"64d7eb34-3d39-45c5-9090-acf4b28d8c6f","trip_distance":".61"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794630,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1078}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"48","PULocationID":"148","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"14","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.06","tolls_amount":"0","total_amount":"18.36"},"passenger_count":"5","payment_type":"1","primary_passenger":{"email":"Annetta_Heller@hotmail.com","mobile":"(754) 546-3157"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-17 03:15:03","tpep_pickup_datetime":"2023-01-23 02:57:58","tripID":"4d482dfe-2fcf-49e3-8fa0-daa7b7f05655","trip_distance":"3.55"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794634,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1082}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"164","PULocationID":"230","RatecodeID":"5","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0","fare_amount":"20","improvement_surcharge":"0.3","mta_tax":"0","tip_amount":"0","tolls_amount":"0","total_amount":"20.3"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Juston25@hotmail.com","mobile":"638-267-0522 x2725"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-24 02:24:26","tpep_pickup_datetime":"2023-07-22 02:14:37","tripID":"8bd8e0b7-c3f9-4080-af62-7e6f7d62033b","trip_distance":"1.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794637,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1085}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"209","PULocationID":"234","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1","tolls_amount":"0","total_amount":"18.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Beulah_Reichel@gmail.com","mobile":"268.992.6069 x11480"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-22 02:46:20","tpep_pickup_datetime":"2024-01-02 02:30:49","tripID":"6be6e503-53d3-449e-9b6e-3a8ab535d512","trip_distance":"4.70"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794640,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1088}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"141","PULocationID":"249","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"12","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.66","tolls_amount":"0","total_amount":"15.96"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Agnes_Hansen@gmail.com","mobile":"(549) 807-8566 x7952"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-25 03:10:18","tpep_pickup_datetime":"2023-10-13 02:58:01","tripID":"c6d635dc-daf6-4c45-8e27-7fe0ca048dcc","trip_distance":"3.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794643,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1091}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"13","PULocationID":"170","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"14.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"15.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Judah_Johnson44@hotmail.com","mobile":"363-606-8572"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-29 02:21:18","tpep_pickup_datetime":"2023-09-18 02:05:14","tripID":"1412a958-bb92-41fe-932e-a5299a3fff0f","trip_distance":"3.96"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794687,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1135}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"261","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"12","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"13.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Marisol.Beier-Miller@yahoo.com","mobile":"1-309-668-7163"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-28 02:38:04","tpep_pickup_datetime":"2023-11-07 02:23:14","tripID":"c0cdc45b-22e7-4ecc-a627-5c8b67d87e46","trip_distance":"2.59"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794690,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1138}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"42","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"28.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.98","tolls_amount":"0","total_amount":"32.78"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Thalia_Ortiz88@yahoo.com","mobile":"407-555-4002 x44648"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-24 03:03:01","tpep_pickup_datetime":"2023-07-25 02:30:57","tripID":"e0b47894-d40d-42c2-8336-c8b9c4ed3a33","trip_distance":"8.43"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794694,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1142}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"246","PULocationID":"68","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.66","tolls_amount":"0","total_amount":"9.96"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Hans.Smitham9@yahoo.com","mobile":"(339) 364-0841"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-19 02:16:50","tpep_pickup_datetime":"2023-10-09 02:10:41","tripID":"cd5a37af-4c4f-4424-9c93-b1244ad04fd7","trip_distance":"1.43"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794697,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1145}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"68","PULocationID":"246","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"5.3"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Lourdes38@hotmail.com","mobile":"1-751-666-0619 x542"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-02 02:16:50","tpep_pickup_datetime":"2023-12-20 02:17:22","tripID":"1f8d2fa0-a798-48b8-a9a9-e20e316d5531","trip_distance":".66"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794701,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1149}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"140","PULocationID":"48","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"11.8"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Riley_Yundt-Turcotte@gmail.com","mobile":"220.269.1280"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-02-03 02:35:05","tpep_pickup_datetime":"2023-08-14 02:23:08","tripID":"3ce204d3-9cba-46c1-85a1-f21292758378","trip_distance":"2.41"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794704,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1152}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"141","PULocationID":"141","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.74","tolls_amount":"0","total_amount":"7.54"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Edward14@yahoo.com","mobile":"320-494-1809 x531"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-30 02:39:22","tpep_pickup_datetime":"2024-01-14 02:36:30","tripID":"cdd42be7-9a59-4059-b8f1-dc3dddfdc0a2","trip_distance":".65"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794707,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1155}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"236","PULocationID":"143","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.25","tolls_amount":"0","total_amount":"13.55"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Blaise_Leffler7@hotmail.com","mobile":"275-547-8169 x12135"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-26 02:27:43","tpep_pickup_datetime":"2023-04-19 02:17:48","tripID":"986787ac-976d-4148-911e-a502b40ed9f2","trip_distance":"2.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794710,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1158}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"239","PULocationID":"236","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.76","tolls_amount":"0","total_amount":"10.56"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Anastacio0@gmail.com","mobile":"592.584.9471 x84515"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-11 02:25:16","tpep_pickup_datetime":"2023-07-16 02:17:47","tripID":"9b42bfac-1876-470a-86df-4f1dcff160cd","trip_distance":"1.68"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794714,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1162}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"75","PULocationID":"142","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"12.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Brooklyn.Feest@yahoo.com","mobile":"538.970.5053 x607"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-26 02:44:12","tpep_pickup_datetime":"2023-09-26 02:31:06","tripID":"7cfc8967-0cbd-45cb-a5a8-a9e7cb05f407","trip_distance":"2.73"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794717,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1165}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"145","PULocationID":"236","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"14.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"17.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Roberto.Lockman@hotmail.com","mobile":"409-514-9426 x587"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-18 03:04:58","tpep_pickup_datetime":"2023-11-02 02:50:32","tripID":"7255441b-919a-4cac-89b5-96f7ff21e0db","trip_distance":"4.12"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794722,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1170}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"163","PULocationID":"48","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.26","tolls_amount":"0","total_amount":"13.56"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Margarett.Sawayn@yahoo.com","mobile":"(866) 903-8125 x36548"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-15 02:17:06","tpep_pickup_datetime":"2023-06-22 02:03:24","tripID":"fd218127-8604-4a99-aa7b-51852fca4b2f","trip_distance":"1.24"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794725,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1173}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"239","PULocationID":"237","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"7.3"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Kaylah_Shields55@yahoo.com","mobile":"(924) 980-5772"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-19 02:26:50","tpep_pickup_datetime":"2023-11-17 02:22:43","tripID":"94df85f8-4b0a-45eb-a45a-ccc11dad11ab","trip_distance":"1.24"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794729,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1177}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"231","PULocationID":"158","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.76","tolls_amount":"0","total_amount":"10.56"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Alverta.Armstrong44@yahoo.com","mobile":"(878) 812-6613 x69201"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-20 02:21:36","tpep_pickup_datetime":"2023-01-20 02:13:30","tripID":"7b16e77f-898f-4793-879f-ccf0c85798d6","trip_distance":"1.37"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794732,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1180}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"107","PULocationID":"231","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.96","tolls_amount":"0","total_amount":"11.76"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Esperanza15@hotmail.com","mobile":"692-945-2420 x1931"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-15 02:31:22","tpep_pickup_datetime":"2023-01-02 02:22:45","tripID":"e11b4c6c-2c0d-46ee-ae04-0700d67c131f","trip_distance":"2.00"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794736,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1184}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"186","PULocationID":"107","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1","tolls_amount":"0","total_amount":"8.3"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Casper.Romaguera87@gmail.com","mobile":"491-889-2259"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-04 02:44:09","tpep_pickup_datetime":"2024-01-13 02:37:12","tripID":"5d52db41-4838-4a12-876b-056abd939f62","trip_distance":".92"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794740,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1188}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"164","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"6.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Eudora.Dickens@gmail.com","mobile":"(252) 559-0694 x683"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-22 02:51:52","tpep_pickup_datetime":"2023-01-16 02:48:04","tripID":"35cc2967-ce3d-4dac-a75f-74347e277630","trip_distance":".82"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524803950,"processingStartTime":1711524803850,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"4b1516a1-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524803850,"timespans":{"extractor":100}}}

From 2eb866d204487e1070f57edc0d4ff9e2e6fdcccc Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 27 May 2024 12:10:16 +0530
Subject: [PATCH 43/59] obsrv-labs/issue-tracker#2 : Added Sample Data

---
 tests/sample_data/nyt_data_100.json.gz | Bin 0 -> 10228 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 tests/sample_data/nyt_data_100.json.gz

diff --git a/tests/sample_data/nyt_data_100.json.gz b/tests/sample_data/nyt_data_100.json.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f6cd64609fc390927c770fd54f786ba0aa316681
GIT binary patch
literal 10228
zcmV<QCkxmgiwFP!000000OfsMb0f#G<@^2$hMqJLw>l~-v%dIbeObEtu)UVnj@=hV
zbyu~BI~ZUBBqh=g|M$%T8lxdqt%g`l$ZCh>utjo6aM*`=GV|m)|NJ0^+i;U^Ke+s0
zlYSHr+udt%8}3&kzx{*%`XK#~HvG<ie(>p^zutD?cD3F7_ow{9fR^9;m;dwljYRy$
zzlPh?ZDTsTiQn+Iw29l@yMOJ&E?vcR8&>O^{F%G$=KFM$Kgre2-LAV1yYCZ!X#V2&
zx4XdqS2p_i=fa!qU4B6y`2E{fZ{F^<KcqMOA|L%>*&x67&26~iZ|2=+xLv*d=@Y01
z_HWy+*Ec`CRqo$(8`eL)2~^|t2mku758j5Go5Zhmchw!=^+A5?!~V<PKD<rGH|<t$
z!tUYf-5<!`W_rWlFn_?uyY2RM(&`W2=QlLn_Dz1J+jh0ie_S~dR+AvKE`QCN+im`?
zH}R_fk^j36-}49kDZj|K>FrhAZQpMD{)&HC>2~#I{|Tf*A+<obka#KSQt9JIzg=~&
z@816WgPj1nkm^#a%l%LFcDH)F|7j692g(@XdJlpuNfCl=g$t-PV2G_be*9Hj-Q0#v
zxBsk-l=-!9+ne1FS8w<yoxlJ5+V?;2{ZF;y?~6b5=C13~&CULAcb9(n%Wj+hb+!5a
zyZs9U3!oKQj|OG_kmI|*y$!qDryrxW;=iyhKL#m(o#Pkg=WCPl*E{}*!rN7TxHh-^
zBRYQg@Wxd<Jb6E)-3>pG`RCc}znA=y)7AdnL_M@of=He01VBonP0)$H?RsN0IqCfV
zNB6MlZf}0}0`p(}&D*ef`gdX7zy9^F|Mkyj19FeOCxBGsCjhDD17s~6r8yOjXv%Oj
z&w%6SY5POiY*s*jxZd8DAgPsUG}y*ju+N7i$UP(#<MC3dDM+GFHh(Y>|MNa1d$fIr
zu@fmuE6B9Okc_S9r0T8ftaT<7APJ4e`5{RQhon<gkhK0eC2~GQ+S(E+PXeTC+{tiM
zQ*iwAdbMf32<!CV<R2cw^>+Idj-*ZFB{XQ!Ufcut=VFqCR9q=D%+O!T$ubE7%pSn*
z^1vL9m^4c2(6ycL&@v{YCDFG02x`~KzE>@&ghfok3t-YN9h17QOqxQOv}df6wV1@y
ziX{FDP)^p!ukThjf$Rt#!8D4PBcs<B_evRE#am%t_cUBeJe9YS97ygvK+4u?X@YRR
zjZCE}3Gb5=p=|+M9eQnik+*WKog0t@O9!N{1f(idNh!~UNLPnPXxxdsm4DyQ$Xoxq
zy$QS3w)tec*=--n(6q`5=ajHykfG@h9PpXY^g@|^OUbO%PC?TjTS~kn^*%JUBon0V
z1oPCE`71wvzOzXr^3;WnBqb_9lgmBK56z|XS6|6re-2D<&xT7?O<!HJFZR>WMB5O}
zH{tp7+v~Jmg{x1uTjsIemnS_0y5XN~gVLyrd!r1Q3UtB|oped&gzi!gg%ckYPU&Oc
z%T@`)j~6r9j%%huExOdpq@C0jO9LO**cT|AOJ}maj>)Rhzz6ofZO;TyRSQsR>}j=A
zxkiuF&gTI1s~tbPtIgHNyY%{XD%Vc5j?0r2Ol_$Gm;BG3&?+3D_RCdM?W?9VTxH+K
zr460NBqI~r4h326FoX0)^e%d75RBKQs%abLXUC;p`XJU<4q{6krrQ5f{?8cu6c9N%
z_W4U#zX{FX!}{jhnXy5v=O^A-%hXaq1ths`bbD?pFSVb7q!nsEwI%-N(ID1=4BGTb
zIN5qZsz(uwibBb*C!b8$Cnyrx7Z}9qr4w0@bxc;N(kRxPZxrj!gGoOpCcnGeZPL!-
z7#u&epk7@ll>uq#;+{}He;CV9GXii=Vd1o$r8({_AK?-db|x4SzSZ1dl1JfU>V=KQ
zBz~bNp^Qua0;O{46c%JHhb`bz;neVFJ{18}b6C~b6J-RZ(KE^jz6=|N#?{xM4RMvq
zEv5%4EO8@eQD596RLy_PFq&0#DJRRO6Uy(Iti9Cu2%avgC<DVcVwWWaEtgG-!nDab
z>QU+572v5FGry$3EPXT!vTio3_P;8BwWeS`c$#w*P21pnMf0y)e#4D7Wk7zYtbcXk
z@>o_PmrPfQNV#tpQC-61%m<G01H&KxJ|bIXJKJV`kCRDIZ-Gl^%e>Zx;1UMq_rv(7
zh-~}|0CMp>7V3CxQRH(@)g9IW5}E_~n#81T2<EGm-*(q`4{QD;U#B;(M{1=uvN4`t
zpkiWq&+x2SW75!KqPbLZ3YmBmA<&auNd%i*bW(7Q=N8lTod}^*B2jOYbzO_8FCkOC
z05aX;k!k8mCAtDK_kGNnm~8DH^CVm<IaVyyGjRFQ&GqfWrui~-{IHH}Nel|h#Yuv6
z7xx~NtB}Ql66!Ge@s}E=aOnpSle`4?J}xmB<t_CBdWeF8&RY@}MesTSH+|${375Ko
z`GM&c-(q5Ai;2Z??AdU+*Lv3Um#5)U{|XX2t%=>PW7=GOxxL=pj7@*IBs8S3k;d4I
zd*f=BmwMQHpuD6>Skkb(%$v?ft!0~}Zjq$uBRWB<&EkicgqDg@i*0bRr25e>!DRi?
zmn49dTM}7N7aXv99$ZrG;0I2<A2G9@k9@k97JM8Y(ryfz+{!7#9i|p!HE`ABvRQ@B
z!wM7i37S0OO1=MJ;JW!fIvqces?|n7+qHsPnJ2UqMGG=Gm8|rANx1$Jb05F>Hj`@G
zOe~ODIi>xXk<3;Oez2j_u*rPZ&Iry=)?xR$`DVKb*I`p8wE2ii0|-a(>?LLf*?=>x
zFKKdFLc_9z+fnlGgOWSQ<dZ{@67g?~Xt-ojpO|s`J^~}M=Lh=dpv1<#04f*HWT}?P
zmQd-Zgava_+3K44b5QD|2I#ROPtGd%`7}#qH_g6kI#(9_eCUmQb)l7F!id~!5~!-O
z>4)j8I!a)F0ihs;k^`xIeD9UoVA>v|Fd;BP9gt`}(_7PFP+*m`sVyMXzCf>O<>L1x
zRJSiNr5TW!Qsm88HqRn505j*p<lF1;dbJDQlu-$STT5Z26xJC=C0gYwb|zeS6%M+A
z=`E%nRP4%KD)T-nH6a-fLB!U1L80wLP|=DOb=S2RbdRN-iE8KtP`P+Et1HDn+WZ`r
zd#IkNY|7fQsZXY}wox<E*{{OtcD>rT5w8MZBCCaBq&Ud8UtZdp18F7XUc#>6@}Op&
z=ra!oCUmJE6-+`v=(4~-@smhx=!H{VCosgWHLW9$rDIHb0ZMD-;wi1Jq_oe|<^3DE
zh@4n5<(M$9ur`@2nV;+)Zt|!9A~7sC8%O2R>6~wjg~p<);jxC1w6`MgmwHlVz@6q{
zIP0HUiTJklgi4^x!g?8{@KJYyv{5E@ou#hCW7SLWSXexd)s?y*Rp>PFd<u32=Yb_R
zv@_~uKU=SEf`+lFELztn?;7n7_g)oOCCp-BFRvqa36owTa1`!oZZ9AAmaPn(;s=jR
z6cm9AxiVU4=X-5Zrj}g+lcuq1eoQW&#hS{QY-wS#pVOWPlTf+GJh4#Xuh17vBJ#^@
z?31p(2;ut?fh<~KwdUT^`|KqGvZl9GKk4ykiok*t@>o_M_m)BSF>2$5iiruVl)d1}
z*$b15RFHaQT3tXTHgfh0X$y-du%^;ZL>CmY)0>hRVR@#@5*R%#p=SW|o3wt|gyx&=
zZpH7ty-kWn<Ugc@H6R@SI4|y{tlMM4VP#@2)#M)24g*;<Lxp{3z@<ZKW3L3bz(o^G
zE4<GCs(K}ROWt;+Q%pTH=Ci_6Ena0{>Qn|)7|81LU6;@~kU6USc}~UrY#q0o=JVCf
zI^1kGV<TCoNO<tVA~VZ->nf%`5(jn)FtF28EVv}|UU(msEq2CO(+dlMizfDxUnGMV
z7>>P_sFf@k7rewgRxO^%+RAyXDRh@~zIklzmINDh;>DLAz4&^LTK3bhxlLD}@4|I>
zGlophB$T`s#v&71?WkrT3yjV|mUj@qPV7qd&;)$qDH08(kf0Z!vty+-*|D;ZOn(FS
zmsYx%N_>`HV)COF&u48dpDnF0@qEgkN*CFa$FU<dvp{C2-SN_P7p^|*`?S0Lhr=?F
zHB4kt8bQV|FdcbcbNS_lRf)8hG>J^dUwF@Z?Gt30L@jWc6hT{lWdRwPzUzdiHlat?
zxuW_9USd^(i=WBbN}s&as^qZR{CymlVB=0)Vq#X-&bGw-Cj8HKXGY9q^M$lRBamuV
zml=<Kur8Rs#MJu_R+proOnww)y>2T-XF^~St9lV2^+NWY>Ah`{d}$u5U%+nNF5cT2
zD$Sist>v`K4=tp(e9nI4i?mLgtFQQ7ck9<lkI3*q<P#1qKSFD(xl}r;)B6ApI@w9H
zpP$adn8Q9oW~;OW&A%r)A9dbYw!-JdiEmpV?|Rq4x!TLc8$3g$!*kJfeLto>liUaE
zZSnkxmC}!@*Pqj1e!L40VWUR<ez>uGbz$=1#{NcHAwtcOv>NK5I(mfsGBgh)HrLHZ
zS6{tMCL(o$2#`r^up-D9MAvs@l%x=KN$+D`g2XP~>lrG|o=e!AG+&<)n&;V-jC!vZ
zswPf>^1HxK#;*C>b~9EnF<(1^5#IS4`z^b7kUeGEvp+vM$Y_<n#77q;mJ%q?T)~a)
zxl;C4v^D}$<YZg#xmJdfzt2kydlr8httwBW_r_Jf>6O<D+>?~XJ-^1Pa<WQ(w%hQx
zdiC|{_BO4>w{KRr*JBmac@heU(Wx?%MPn-UvHML4`O~hX?a>?P(H*Ar*61!s(I+mI
z<ZLHG%av18Qcl@uK$gr4sL}Hybn#cxs`g6S%4ZbJ8Px^V(sgwzmj#-Y%YL!FW0-B6
zA7!%+;MD~gZs&~h7xy0P^7amGCK1!wDQvo<;SXHuN5&#xymGNe(Z@V0_OTb<8z+(`
z<*n}Wdkff9jhtUwu=pcsRe2;`;<D2Q2FtN|ewK25z3Lg2SKoA>uOBuW7>O;>HVaXO
zR#mPda6Az)m+}M?u7f=uxhJC^8T-_Conb<o<?1E@6WZj2_JpA|<j|D^_sr}*&vx-A
z(yI1Ex@cTr&N%$3J}5D#+KS+~uZVm`!Td7C6j$L|jSAS6mkmGf4RD#Ng5^QGjx=a^
zCOo+8Ao!7wc9)$7mGT4Fl9x<lvr8T`TG2(N`shhpX$r9PFVJW@xp*2&bu?CCu~wc>
z@>AVtp6Vn!QY#C9oUD~!uhy%uNq%%s!cVv`n*Z0>9E53P#V|InJ`Q^l2HGSl%|Ku^
zJPAeU&}Tspii91vy~cXsj11YmucsJo46>{!n16%m7JnD5D(|8X?vzha3Fj-8)dpNA
z>hSEWGQ5x0>HZ&#ue7tH%_W1<Yv!+#`!}5>vZ`i>DTkv<a3>gW9f1sLSa`lO;G&RJ
zpJ{bZK~M+WT((Y&fSR<?Q5h;VmoG5)af`o;R+V?rMRCCykqMRR0&L`oQy<shjM)74
zI&4-CVH}trDx+RqpyT3X0Wa=dmBkMYdI8ka0@>_5vPZFkC-Kq2K?r$F3@WG9=FYcG
zwigCeFs;tBpAwa&mnfRBcq*$aC$g?!CUQpcx;<0P9G9*?&wp#PO6veLo+V_=s|%}{
zS=t)C&yZ59F6n4>H*rpKz^@yu4c_rsMHoCvVX;kuz_r4qt{2hylwx#U3+18D{MPFZ
zaOsdVFuz;D0jFmg`n*JYIpT|lqrQ00n#W#;JAQN@oErnC27l14E6T0~XrLwy(=eIM
zI@x?AGO1hl?{%3^5=#^WC6i!U%}iEzPH@i|xz&WEtyRtx<K5j0NPgVnFQirFg>=ai
zZ}J3l#x2P?=00vtne3<eo7|*y_0^i6C_6fib=kcHsWIL$k(CZ>GBM3iHkFwg%F<60
zTsY2(-UTBlPl5j<q|CR`SzI52>vAS$Y<t}XeyODv)6ecF;ue1ztzO_Xdb()N2h956
z&uP)oc2*<Vk79VE?fBRSO}@QUf(xb2GueE0Tt#MasDRT$SvNQqF!NQ9R0SRbXb(vQ
zm(RSqsD<ks2u)U|K3eKb(Rp3l`BAy}nQY~4v@hs=rt>i8LnT(nBKU?1zyxTt`Z+kY
zY3z*BypPsx+TAw)_iht!%UcrzihprH<0;=qlRQL!JlncJ?E`BGFZHAgm);8u-b=TA
z=fY*82L2g~E)oiAJ1bhPf-n+186#bU@+NbB7cRH>`)E~pAFWg=O=xm5J7bV^ZP~OZ
zRZW;Neea8Od%H?*boOHnHkP=DmeMjNwXf+g)xN`IZb~O~nD!VYq%Rdb#w7ntooxkB
zzW1^&JL9#)rQM=3zPF^xd(8PoKMohqWNoFopy(%ZHbg>o$;1Xu<FeoyKO>p_Yf4?x
zBhFhNTIF6{Xl?S8C3}hJstaj5+;1s=36la@fAkVDPsDhtcQQ$r1oVjx7t*aUNtl+R
z$G+2jD2vx|{#v>C3u#q(Azc)&PmA$pY%$NH`Z05w$bOn`@=R9aSV)P6R~Jh8d?5`N
z_x!sPYs7d*&-FZ$J!M_u2O|;nIMf~cD0{C3bS?;z#19}BO_8Dju+gfB#iK7fe~0Op
zeu9Zu=_4WwsGMA2{u|AG4kr_*Xz^z3EdRdRx0X0M`pJd`+^q?)N?qK0Tf<@XFv$UX
zDW_^B9Cw&m@AIgyR9Sgvvm0IlcQ}l0bx$Iq(w**1@}b0GhlYG!Q0k?FlCUyxuV^EC
zzPZl~f}fKqEY3(_KS^=Nz-a!M_<JrBS<FXlXd7vD#jq#)HPc`Si<841JDh5Q<6`+f
zD5JL$yVi4Q^x3J8scftHaqJywl@LjWQg9G=2(z0zJHPZycci*Z_p%wDIiu(LzX2+r
zBg6Y-+df=<%Gg+qmCNkaVU1=2iZv5hy)S;a&w@Uos~i?2_GzUgW~)+VPoLaX22H|6
z0)f2&ZYWW<Sh94+#?aiL^inMylv-Bak`x4ZYSN){K2TOh)1G=sGNLHZ&*&+?yvvs)
z0Y@VCtk;dv&^RUK#l5u}Ysjk|c9w`k>H8%mff_;(lpj)T-v#WU1|w}JobPnLgeDOZ
zca$CazK@~9)|cXfU~mqc7m|x-uey%C{^pRZER%3*nKYwSvH;8Ji_7gM++K%G^T*xl
z_NKcwBm7ly;qqCdlT&GpH8tqOVdFu1$eI1+dCy=}c~tAYQ>=&3r)Z?erz3*;2%=>^
zDYU|<RUdmxr7>({W}m`(=@$=9JAZ3=PD#O;0!l{nNuE<NzYRZx%_e{=LsFA((ECQ}
z8W-n$*L9Et=+jzCJh~Y?+Ew<g_Dc64d>gGGGEM}gjp$`B$y(Abltg#X==s?zES|lZ
zI`(=<gZ>S*>XrWLryq9VtcP9uRb|5u)aG`(89(kKe0AZp%2U{9x0QQ=>tR_5`jRH?
zoZn@V(D1?Utnfl7dnco410+G(cfw=L?#>->XW6D?N_NiB*x6<39?ar_N%LD2oHe_>
z-fHVqV9vM~`R_Mx!safmn=jHTrp+#`y6dsHXzRGoB-MDMn8-S-bxj1O66%=EKCO+V
zLEa8-IMjR>fe<nEoiw7;$p}i0gmb+VDv_mRF}Na=<!|BzFuHguYwD=%Z=M!iZMmhI
z6G1wo{O;@R-RdS>eQ_6Wwi|0l-9#8sj*I{V+AG&9thaJ<kRDvu<q7#DhanKT!Kdq;
zV?pGHkU9`X^-hqLIS0_?hSM96s0<X<KG2x?!KoI{Y;7g8Em?1!)7!baex5c1Qlln6
z_$*%Arq|)>?_vEq?ZoHnlw<e6j~G2?11?1k8Y6QUW>xN<AB3`bxHH*&s$)nWJ~@<*
zmLOSgL<XuL<i4||p8rJaAtbbjMUG1^0@1~DTwBL+flAFMosZ-G*8b~-&XcAP`q7)P
zUNJ&fx3~OMjr++eu!4WG+~*-?6+5}gtlHzQ@?aJ+(Sg$N!H-8n!5N8dn{t9sFGyk%
zL3$E_AHvXef$MAOwB!YhMZ8)(xwVz#mI_N0IHNERDo>^9RApYiTb{86{do5fHlg|7
zcdPH$>4Bi!d^&Hq=3qnKn~QtYAZv67M<pyBtqD$nr;<<d_n=cc<%H|jbSCGQ1QD?J
zBE+cs07-Ygq|n30&#s&GxOA5hbd@e6rOE9XjSQ-_K&LgHzsAbogy_Ay-HLy%hpYrY
zc4^|he|6!}3e>j7!~l<3U+xko4WHdl%grY~d7aljB}S#r_n1=k!Xr1K33@|qAPQ8<
z`ofnO3E|=?t*ac{LdpB*j8${LU43yzOKMJsk&kzYD<+IOi+pI6dUZis2}80b6~XVX
zOBj$``6i7k&9LAEd8r=R{UmP_wlWF|iF-`jN8wuNMe40j&Pt@B)pNe|o`1uMi|4Vf
zc1T+S<oRZ_bf&8>8a?|eWJpgNacws@cWJ%7`X+q;{c1bHVWnxbcMU;Jj!@PBslkK`
zrpRkPFHu8|5I!2t_QtB9eJdbj1NJWai;%OTC4R&J`!04R6E4s%vEHP`PiKAQbk^FR
z!*cIKai;C&S$w#D6?>h;<eygE^>!U@v>w};c$Lioyzrhu!cAq>)t5UKOLM8Gq<MM!
zDUZ&xk6=lCa58GoPhE~H2`vieG(V0isR&GFeP24oqz30l<>CYQs4{@>5VrRemFGM#
z!8%x~=Aa^-My6?qX549}ADeHwufyH<*Xz}$TrEMMb=kL)+c(I91E;AbuGS4BS+qms
z<uAMa4EdPw<kqBT5^JN2qJu8uGjVqbs6>j%B!!`CWBJxZ%`a?UeEc50MEu@#LgtLO
zCcklR$#cr)m+8l}UN_%%*Kg7~?M8N)j84p@2?bJimeqiD;5;;sq^H$DAO{gVx+k2r
zi#B+ZqKK}yg0!{5Var%eiO49lu^ovonZMCoeEuF)=I@oIwDx=x*~)?JY0%VjGTQIL
z!<rw9FT!rOx+&vQkrB!=GNs0gdvAQrK-Lb;qm~0pomi8=;9vPT71MZr`a*9+uRRH(
z$b>th6~5Q5(^dhc(#`a6t1~Ys7td%b1Ne#(gM+sAcX6EwMfb|0<+Gj9#^BSmetR8A
zk4<E~<QL|2V{OgBNoJ+`C{>?qG38za&-~Rt(fk+;;7v>dJwuVC*1|hNvA_R_kT6jR
zkC0u?<^?24xpcc9qB^^uqRsQ9<mW%zgX`m6{+rlbefxH`c^yVhz&y8PlKWq)DThZt
z|K4Ojkv)ul04~)Db@DL!k!G;$dN=wB$;iy89T5oXjTE5^iODg6h7_burJL#I@LbLd
z%f*w|+U&h&#R$zf_))ckA9JF!h@RE5@|%a<ZB`WgadmT@c4cTP0AZCD#@TGe#Sdd;
z%{0RTWV&X))5&rh`Z%%#lIyauh}H&kG6D(j+ay$Id&VePlXL-_FuxlYgLCP=Kcwq?
ze@b=+=Zr>BXDXU%)~flJ@Nm1`H2?2z9q#%vG7S>*Yu}L0`x2_!VjhyA_s*5-mv1!p
z2b~8$E~F>s>mgcj3>;yk?gar!;S6cfgYH6ywntZ5m)J(lUo>IyTvpd{+47Ahoi7&i
zx3)5P4lMr|+SPU)%*Y(}Lx=j+h1Xsfay3q}X0IyP%XjqTJ?7D+eJ|E~R3~L<okS9f
z@~lVZklKh+w5pHLGD?gtn;Fa=?Zsbv@g&w%lGw7f$oX!*s_!x37_v7>W96(dLZ7d<
zyH!@;mG8V<HZ13_s7#Kit?^yYjMN<$k&}i$dJxd&@PtRX?$MeywaJK12Fvu7lyI1O
z5vcF^|FkZ4rR->!KRlXYx%e4uW#V2j58>d^btVa3Ehta)7%`(3BhS+m+}x&J^WF6-
zt^Ig_v;+U@!eyH~n-9Gt)%jN*W;x29WHh4o3lgP=xS@AOBi{DC)JACA`#kt@BFMZY
z@vT;gAauTL$2C7V)#5p<skC1&Iz69}!(uguZTxBQ)Qz1Ho*(Z*Tt6I)p4EuM2<22e
zDFrHPFYdjQP~!r0G^f;bNmEnV?B%tuvpNfe-Up{K5WVCV6_ql~Xj5b|+wx;+OjKTF
z)92#$l3oCuizl>|>3a%`;`QH6IKrvBk{P{a|F+`ib#v?Gn9PUBG9J01(^zp3{cf^K
zZV=Q>sACTy@QmEYQi9Mc4Mqv047ZmCyl5jyL9uf#>D1{m813A#UJS;?)7aXmy%HD?
zZ~V_#T|NhnpM^|f^@!6mGe}LsyPO^bHA@i`>X-zClfLWj7#k#q?dH3MCHBgNBt_!d
zm`h`(tTKtb)(?q^t8`F8qn=v|OLFlcdsG>+SE8-<cjUZvYLNSklh<$3>+2QikyVIs
zjY4W9nF^z=JV@PR3+0p%!F~&(_BB-D6Wy!Sk)N}k+eT~i)HNywqILXCxWMJnIYni+
z^%ofLEIw9`YGd`v9psembv`Wq)^)Mq;xmoK?Pk>lH?jmV=o?SEaTaU9sE;Cae?piZ
z9_6n3NnCqu0e!HYuwJ%;d@>>=)eGDAsFHFqy5eEw{H{F=#>JCVS4&ch{sZS5=TwK!
z?jti!!=s<m(D@<qXZX`9ZMHv--ETQAZ(cEdG+JNWgMRjUl74b&)_6iLG!2th4NnZU
zGAVW1f{560p^RK8Lr9jfiQGX()silGgUo;OOp=yv*h9X~u%~>yb;kRU878f#4q8VB
zI?n;-N9%OY=t?^^;-p31HOvkhiB&D6889Z@v<pFIu4>S{KK58w#&s6EC=w(sNcTYm
zk5WY0nV?#mf+{U08=8NhLreGSAz$g$Q`$?O^O_Q>OQku{MawpRMjNfq+BRH$89IJk
zM-sdoG-y?09CD#Vgt|!RK{4PjY4Vg(4Jd{zUVk5$I)U+BXL6EW5M-SfE-+51?|ZF%
zm)gE0<e~Y4F1deQI*Zk{J_MywIqBW}Zxj&(S!8|^GGW%ct#8u$ht-B?Jds4{R~Ony
zhLwkldo97&U=xHq<Xx&0&YWTJAkasJ>`#VTtvAVN(Pw)QQUOE&pFJ&8OdfoG5{n(C
zpI`L4cn<<y=|N!i&!MT*ltsvl&OI;TL7<PiyT^KToK<A^=bInHI&QB%UNg?ZFGExj
zaoM9eqTAhYEuyE|%<Mzp2W_5w1kk?2ke@jbTGI+@xs0}*C6QE4^wPd9UyJAl=I=8X
z??a#~eF%yppfd&^_*#6P80e+v%nka6-}66VlgubAjW@3@T)skfu5MK+VTg#<lYOQ>
zs(9#0ZhJx<z{Mn-j!sZ;t#GnU{A^$}(I`r)gh(~NtrjD4>GxZ@I=8)~OFyTrmaIdh
zYSf8zcHWA~s0`V-=kD!WexA$YA8&1A(KZrtS~Q`patP7K%V#_xj~0(3BYZNJCGI9|
zV8Yu)BtcFo!H;DtFsN2qn=C(xWpvJMq)xDSM*{r<js)j)2>Cr8aecmB$I$%wtL<h~
zW|z_7Xz!*~)nE=XyAy^#(hp>p<ddnaGo1|5aknZ>4mOA=S^_Vk!Zvx;MsLdvrkh>a
z1B<sJ(3MsMt}y#KqvRDUYo$50R&vdnvsUgcdsbX7!=~Tv;uw$RN3c;cy9IIWx(6~f
ze(JPO62W{fq9^^-?Q!D?Pb^}&d5mBKx2h^vOY$9em(p;x1c%8u7i6hg+QG<SUUV*=
z%T{_3*uu&4AqVqJ<DW{SkW+V<^TyJCyyI_k6Rtkqe!sd|b=Tsr{JGM|wIyO>jBoPX
z^Wt8bsw-%}?>6_Tw9}4v{&;SAv=;fvEhO?&XIs%xPL7Uk5Urw|N*fY(*tTFxdOY<4
zIs;gIq`R(+bXU4y7jnk=Pql~hY4{wy8hKWuxw;8yea!3{i3Kv!zPhm932l9yEbq`Z
z5)RiO6FsIHwx0^$$7Dn$xyf{l#OzFft;$(?QS_lF?|a*WEMszhX9E7;#bXjH!}NX*
z$>}?;8G|l=a|fU2NPoUs?Or#ZY&W~@L%CFX3k^DKkb65WmCyIQJ6Q6VJ(o2n<j=x!
zF6*9T1KHO17`e3!UF1q>WG`GaQD~23b?RHMN{-F8Q7?eXrAGtl%KUb~QrrQZu~b%q
za__=_QnmEMk<v`o1gn39tH1D9Why%|m36vNL=DNB-1c1k_I?E5ru+yF2tK&U%1e4=
zL@6N=DAfz>jTYov$qi<Q+-h=r34PGUg#sw?1#X^U@wx4~Hn+XxM0r5!@8q-#PHi#e
zoD}w>-FoXslYtC^aMlQm*@*zElG<_TDG^S{f~MgHMP*ZwN1$vw)AtrdK=Oh#KZuSD
zlj|70k`heoOVah1I4gn0C$;O!q;`d+B?+C+!UwAFN=|Gme@*nrBqqOG-QL|ae@l07
zNR4K8X(==^OpNsx_l5xL5-tx$cG{lc+-!$sXSOAHv@fA#x~MScZALDWk*llYUmPOz
zKDGcMXe_Z<dI7x;EIz1RR|d6%E@2bSck)#mac~kaXU&2BB<yaso92^Udb8S$ZAw6(
z0b#jl8lx&Sc<}C6gh}kARR|3A@@JZY9AiM%ph(K}f-JX}t>X5wl|fpE5=~ojhJ}~F
z<l@8Gb!9j^6l0+E`363&mcqIdIqd#ib4J0>{nZ!Y4gXNYm)q;j$a#s=zCo`VE|wJ;
zU2q)YfS<PT$+c1scCzx(9M-n6SGw;+fM~e6^j^5GYekC8X)Uy<i>4tj!ebX7&#r6Z
z*|F$Va6aFV>N%`xP9w5@K2ZLNL3s6TcfDd<u8kd4$a+n$E_6OtmQ_1TH>3iQ9a_d;
zKKPQyWePrWQm5W@lAslR-#S5vL4*#y(5)lp+?sI2w%{e|<>EPPWk@@yl9T#56|%k#
zmek18R6mM_9JrZ`n?t)>r8SP+MjPU4L<|!vE5@+z%<j+)lht2V5gcqPL0{sdO4)^0
zI@k6hMwbM&K?{#<5;pj@Gs>E_s3M@o&yPsAbQPY#N)=vVHF8ep$=|J&`e(Pdt8~SG
zNw3prrL+=XU07v=v*5D|&-sd>56I(5N#1=-!SiUkrRj+b&v#MVo|!9k*-NDNQnbE}
z!X^o}^&pchi)0T1-+AG=^nz)yu3(NtkDSkpvU2EiY7X>_VY8p`-^*^b3C-WP>ybPl
zWlV$CHD(V>MF2HbQx4-*dE)J*+NWEZCuZu-%Fdw6Is(kbur>;>RT5~l3#pI%noDNt
zFVbK7r7tfHRch}_8PMlECIL9po`mMylYF$!f`Ts|!tJdZF>d})YoBW-a4VN%^fc<4
zJ1=vb{n!)Se#{Zb_#}DOV|2ZYkv~~xw3HdGmnfWeq*L#r2_?6mm$<?Nzx3IUp}N^m
z(NyG|5tr4_JgsPgn)A}@v-RpGG+&1{#MLOFMX4GKE}O^ZoCB*X#<F>Sq>p|-lVe$P
z6bRuHzn@-Xw!m`2M6Ru5xz7yzTng3M1g7)Y>vAZxpWTT|$)$UW7^?FWaivh`GdknJ
zS=cPhOJ{d!yAHY0ydJ5T9__0OM<$C;;o_dEL_cmQm_&mP{g)lq4+?iwPvo&dnUIX_
uh3<3gY-+6tu64pCBQ^FEA(#RxnQYMfs9gFI)9C#Y^Zx@lnC<CPX8{25O>@Qo

literal 0
HcmV?d00001


From 7e69c0ada7bceb7d1c1448819b7b05e973c6c6ff Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 27 May 2024 12:57:38 +0530
Subject: [PATCH 44/59] obsrv-labs/issue-tracker#2: Updated test case

---
 tests/test_s3.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/tests/test_s3.py b/tests/test_s3.py
index 4dc45f0..db91c39 100644
--- a/tests/test_s3.py
+++ b/tests/test_s3.py
@@ -47,8 +47,6 @@ def test_source_connector(self):
 
         config_file_path = os.path.join(os.path.dirname(__file__), "config/config.yaml")
 
-        SourceConnector.process(connector=connector, config_file_path=config_file_path)
-
         config = yaml.safe_load(open(config_file_path))
 
         self.assertEqual(os.path.exists(config_file_path), True)
@@ -66,17 +64,17 @@ def test_source_connector(self):
         tmt_consumer = TopicPartition(test_metrics_topic, 0)
 
         kafka_consumer.assign([trt_consumer, tmt_consumer])
-
+        
+        # kafka_consumer.seek_to_beginning()
+        
         SourceConnector.process(connector=connector, config_file_path=config_file_path)
 
-        # Consuming messages for debugging
-        all_messages = kafka_consumer.poll(timeout_ms=10000)
-
         metrics = []
-        for topic_partition, messages in all_messages.items():
-            for message in messages:
-                if topic_partition.topic == test_metrics_topic:
-                    metrics.append(message.value)
+        # all_messages = kafka_consumer.poll(timeout_ms=10000)
+        #for topic_partition, messages in all_messages.items():
+        #    for message in messages:
+        #        if topic_partition.topic == test_metrics_topic:
+        #            metrics.append(message.value)
 
         # Check number of messages
         print(f"Number of messages in {test_metrics_topic}: {len(metrics)}")

From 13282a7d6667aa441bfe99f0e11d4d4e0b86160f Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Mon, 27 May 2024 18:23:15 +0530
Subject: [PATCH 45/59] obsrv-labs/issue-tracker#6:  feat: added test case for
 azure-connector

---
 object_store_connector/provider/azure.py |  18 ++--
 tests/batch_setup.py                     |   7 +-
 tests/create_tables.py                   |  34 ++++----
 tests/sample_data/nyt_data_100.json      | 100 +++++++++++++++++++++++
 tests/sample_data/nyt_data_100.json.gz   | Bin 0 -> 10228 bytes
 tests/test_azure.py                      |  40 +++++----
 6 files changed, 154 insertions(+), 45 deletions(-)
 create mode 100644 tests/sample_data/nyt_data_100.json
 create mode 100644 tests/sample_data/nyt_data_100.json.gz

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index a80cafa..a821bed 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -17,12 +17,15 @@
 class AzureBlobStorage(BlobProvider):
     def __init__(self, connector_config: str)-> None:
         super().__init__()
-        self.config=connector_config
-        self.account_name = self.config["source"]["credentials"]["account_name"]
-        self.account_key = self.config["source"]["credentials"]["account_key"]
-        self.container_name = self.config["source"]["containername"]
-        self.blob_endpoint = self.config["source"]["blob_endpoint"]
-        self.prefix = self.config["source"]["prefix"]
+        self.connector_config=connector_config
+        self.account_name = connector_config["source"]["credentials"]["account_name"]
+        self.account_key = connector_config["source"]["credentials"]["account_key"]
+        self.container_name = connector_config["source"]["containername"]
+        self.blob_endpoint = connector_config["source"]["blob_endpoint"]
+        self.prefix = (connector_config["source"]["prefix"]
+                       if "prefix" in connector_config["source"]
+                       else "/")
+        # self.obj_prefix= f"wasb//:{self.container_name}@"
         
 
         self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
@@ -50,6 +53,7 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
             blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
             
             object_info = ObjectInfo(
+                    # location=f"{self.obj_prefix}{obj['name']}",
                     location=blob_location,
                     format=obj["name"].split(".")[-1],
                     file_size_kb=obj["size"] // 1024,
@@ -112,7 +116,7 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
 
 
     def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> list:
-        self.container_name = self.config['source']['containername']
+        self.container_name = self.connector_config['source']['containername']
         
         summaries = []
         continuation_token = None
diff --git a/tests/batch_setup.py b/tests/batch_setup.py
index fbb99c5..3609ba9 100644
--- a/tests/batch_setup.py
+++ b/tests/batch_setup.py
@@ -23,7 +23,7 @@ def setup_obsrv_database(request):
     ) as config_file:
         config = yaml.safe_load(config_file)
 
-        # config["connector-instance-id"] = "test.new-york-taxi-data.1"
+        config["connector-instance-id"] = "azure.new-york-taxi-data.1"
 
         config["postgres"]["host"] = postgres.get_container_host_ip()
         config["postgres"]["port"] = postgres.get_exposed_port(5432)
@@ -32,16 +32,17 @@ def setup_obsrv_database(request):
         config["postgres"]["dbname"] = postgres.dbname
         config["kafka"]["broker-servers"] = kafka.get_bootstrap_server()
 
+    
+
     with open(
         os.path.join(os.path.dirname(__file__), "config/config.yaml"), "w"
     ) as config_file:
         yaml.dump(config, config_file)
-
+    
     create_tables(config)
 
     # clean up
     def remove_container():
-        print("Cleaning up the containers")
         postgres.stop()
         kafka.stop()
         try:
diff --git a/tests/create_tables.py b/tests/create_tables.py
index feacf6a..83bd31e 100644
--- a/tests/create_tables.py
+++ b/tests/create_tables.py
@@ -10,35 +10,33 @@
 
 def init_azurite():
     ## Init azurite
-    azurite= AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:latest")
+    azurite= AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:latest").with_bind_ports(10000, 10000)
     azurite.start()
 
     blob_port= azurite.get_exposed_port(10000)
+    # blob_port=10000
     conn_str=azurite.get_connection_string()
     acc_name=azurite.account_name
     acc_key=azurite.account_key
 
     azure_conf = {
-        "AccountName":acc_name,
-        "AccountKey":acc_key,
-        "BlobEndpoint":f"http://{azurite.get_container_host_ip()}:{blob_port}/{acc_name}"
+        "accountName":acc_name,
+        "accountKey":acc_key,
+        "blobEndpoint":f"http://{azurite.get_container_host_ip()}:{blob_port}/{acc_name}"
     }
 
     blob_service_client=BlobServiceClient.from_connection_string(conn_str)
     
 
-    ## Create a bucket
+    
     container_name="test-container"
 
-    container_client = blob_service_client.create_container(container_name)
-    containers = blob_service_client.list_containers()
-    # for container in containers:
-    #     print("container-name",container['name'])
-
-    ## Upload objects
-    blob_client = blob_service_client.get_blob_client(container_name,blob="nyt_data_100")
-    bloblist = ["/home/vince/repogit/obsrv-python-sdk/tests/sample_data/nyt_data_100.json"]
+    blob_service_client.create_container(container_name)
+    
+    bloblist = ["/home/vince/repogit/object-store-connector/tests/sample_data/nyt_data_100.json","/home/vince/repogit/object-store-connector/tests/sample_data/nyt_data_100.json.gz"]
     for blob in bloblist:
+        blob_name=blob.split("/")[-1]
+        blob_client = blob_service_client.get_blob_client(container_name,blob=blob_name)
         with open(blob, "rb") as data:
             blob_client.upload_blob(data)
     
@@ -50,10 +48,8 @@ def init_azurite():
 def create_tables(config):
     enc = EncryptionUtil(config["obsrv_encryption_key"])
 
-    #implement
     azure_conf = init_azurite()
-    
-
+   
     datasets = """
         CREATE TABLE IF NOT EXISTS datasets (
             id TEXT PRIMARY KEY,
@@ -142,11 +138,11 @@ def create_tables(config):
         "source":{
             "type":"azure_blob",
             "credentials":{
-                "account_name":azure_conf['AccountName'],
-                "account_key":azure_conf['AccountKey']
+                "account_name":azure_conf['accountName'],
+                "account_key":azure_conf['accountKey']
             },
         "containername":"test-container",
-        "blob_endpoint":azure_conf['BlobEndpoint'],
+        "blob_endpoint":azure_conf['blobEndpoint'],
         "prefix":"/"
         }})
     enc_config = enc.encrypt(connector_config)
diff --git a/tests/sample_data/nyt_data_100.json b/tests/sample_data/nyt_data_100.json
new file mode 100644
index 0000000..2c88bf3
--- /dev/null
+++ b/tests/sample_data/nyt_data_100.json
@@ -0,0 +1,100 @@
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"7","PULocationID":"261","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"32.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"10.14","tolls_amount":"0","total_amount":"43.94"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Teagan.Frami@hotmail.com","mobile":"343-570-1197 x16129"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-22 03:16:20","tpep_pickup_datetime":"2023-09-18 02:50:01","tripID":"25be90b3-7cbb-49d4-9785-b6fe995b0025","trip_distance":"11.13"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794206,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":654}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"263","PULocationID":"229","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3","tolls_amount":"0","total_amount":"13.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Lorena.Daugherty@gmail.com","mobile":"559.624.3003 x61721"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-21 02:33:16","tpep_pickup_datetime":"2023-12-11 02:23:34","tripID":"86dba649-4810-47dc-856a-64dc6f583a09","trip_distance":"1.94"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794209,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":657}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"243","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"27","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3","tolls_amount":"0","total_amount":"31.3"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Marques_Carter68@gmail.com","mobile":"640.971.7796 x1582"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-02 03:12:12","tpep_pickup_datetime":"2023-06-23 02:47:05","tripID":"a1e9eae9-63d9-4a67-ba4d-1d3a0e69a909","trip_distance":"8.38"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794213,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":661}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"87","PULocationID":"229","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"20.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"4.35","tolls_amount":"0","total_amount":"26.15"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Rosina93@hotmail.com","mobile":"1-587-969-1603"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-14 02:53:19","tpep_pickup_datetime":"2023-03-11 02:30:46","tripID":"26c2008a-c9c3-412e-ada4-28546d661647","trip_distance":"5.70"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794216,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":664}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"181","PULocationID":"142","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"49.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"5.76","total_amount":"56.56"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Keon_Gorczany@hotmail.com","mobile":"(268) 788-3566 x92699"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-25 02:56:52","tpep_pickup_datetime":"2023-12-19 02:12:23","tripID":"d2176c39-080d-484a-ba20-a9f01188d8d3","trip_distance":"16.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794220,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":668}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"236","PULocationID":"170","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.15","tolls_amount":"0","total_amount":"12.95"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Adele_Langworth-Weimann82@hotmail.com","mobile":"1-941-205-3594 x0006"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-29 02:31:45","tpep_pickup_datetime":"2023-03-13 02:23:13","tripID":"81cd141b-1e9c-438d-b5ee-8c5259177063","trip_distance":"2.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794223,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":671}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"142","PULocationID":"236","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.7","tolls_amount":"0","total_amount":"13.5"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Roosevelt_Spinka2@gmail.com","mobile":"996.831.1399 x956"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-28 02:44:07","tpep_pickup_datetime":"2023-06-29 02:33:46","tripID":"b439d3ef-94cc-4018-951d-cd5a00c1f87c","trip_distance":"2.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794227,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":675}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"4","PULocationID":"68","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"12.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"13.8"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Raphael_Borer35@hotmail.com","mobile":"609-381-7064"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-02 03:14:08","tpep_pickup_datetime":"2023-04-16 02:58:03","tripID":"992381b3-e012-43a2-be16-e5e0146e3681","trip_distance":"2.80"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794230,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":678}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"232","PULocationID":"68","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"17.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"18.8"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Nayeli.Kub-Weber41@hotmail.com","mobile":"414.556.6205 x636"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-10 02:41:44","tpep_pickup_datetime":"2023-04-19 02:19:47","tripID":"83157245-9369-415c-89fa-e2d9ae674821","trip_distance":"4.53"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794233,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":681}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"229","PULocationID":"232","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"13.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"14.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Nadia.Dicki6@gmail.com","mobile":"(622) 541-8901 x824"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-07 02:59:33","tpep_pickup_datetime":"2023-06-14 02:47:05","tripID":"60301101-b6f9-4327-96dc-daeb27937a04","trip_distance":"3.56"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794237,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":685}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"229","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.36","tolls_amount":"0","total_amount":"14.16"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Harold.Greenfelder52@hotmail.com","mobile":"498.394.3088 x841"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-16 02:11:24","tpep_pickup_datetime":"2023-06-21 02:00:55","tripID":"e2476cbc-c609-499b-bdf9-90d42bc60a4a","trip_distance":"2.83"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794244,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":692}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"33","PULocationID":"229","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"20","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"4.26","tolls_amount":"0","total_amount":"25.56"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Edwina_Jacobs56@hotmail.com","mobile":"207-669-3585"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-28 02:31:55","tpep_pickup_datetime":"2024-01-09 02:14:01","tripID":"3bdb160a-23f3-4827-8619-b6443594e71a","trip_distance":"6.28"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794287,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":735}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"158","PULocationID":"114","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.56","tolls_amount":"0","total_amount":"9.36"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Jazmin68@yahoo.com","mobile":"(998) 720-5079 x4750"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-04 02:55:20","tpep_pickup_datetime":"2023-02-10 02:48:24","tripID":"3068ea37-fc73-445f-801d-b6a5434b29bb","trip_distance":"1.14"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794290,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":738}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"45","PULocationID":"158","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"10.8"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Lexie42@yahoo.com","mobile":"477.656.8319 x51739"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-06 03:08:37","tpep_pickup_datetime":"2023-01-16 02:57:04","tripID":"28b0589f-9769-4a83-9cc7-78eee8eb3209","trip_distance":"1.97"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794294,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":742}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"43","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"20.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"4.36","tolls_amount":"0","total_amount":"26.16"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Donald_Schowalter31@gmail.com","mobile":"221-276-6460 x6513"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-04 02:20:24","tpep_pickup_datetime":"2023-08-28 01:57:19","tripID":"a9fed048-4b77-47b0-a025-1085af772b64","trip_distance":"5.65"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794298,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":746}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"75","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"19","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"22.3"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Nola.Metz@yahoo.com","mobile":"332.759.9239"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-15 02:44:03","tpep_pickup_datetime":"2023-02-19 02:21:58","tripID":"e761c2c6-c757-47d8-96eb-cbc4e1cd7c90","trip_distance":"5.35"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794301,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":749}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"79","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.15","tolls_amount":"0","total_amount":"12.95"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Ibrahim_Daugherty@gmail.com","mobile":"800.910.7891"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-02 02:26:09","tpep_pickup_datetime":"2023-01-06 02:15:28","tripID":"e91e19cf-9d02-498f-b2c0-708d10106ad5","trip_distance":"2.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794304,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":752}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"162","PULocationID":"234","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"9.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Jacynthe.Bashirian79@gmail.com","mobile":"1-964-329-9021"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-29 02:43:11","tpep_pickup_datetime":"2024-02-06 02:34:21","tripID":"83ce3138-98dc-4c64-b28c-269a029886c8","trip_distance":"1.60"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794307,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":755}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"50","PULocationID":"50","RatecodeID":"5","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0","fare_amount":"0","improvement_surcharge":"0.3","mta_tax":"0","tip_amount":"0","tolls_amount":"0","total_amount":"0.3"},"passenger_count":"1","payment_type":"3","primary_passenger":{"email":"Rogelio34@hotmail.com","mobile":"898.781.9202 x14465"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-30 02:15:42","tpep_pickup_datetime":"2023-01-10 02:15:41","tripID":"c4d3ad69-2bf7-4e72-82ff-90ce63841704","trip_distance":".00"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794310,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":758}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"162","PULocationID":"48","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"9.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Elyse_Prohaska@yahoo.com","mobile":"(357) 962-5072 x6405"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-18 02:32:47","tpep_pickup_datetime":"2023-12-08 02:22:14","tripID":"169f7e3a-6c53-4ab7-bda9-5911b784f30c","trip_distance":"1.30"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794313,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":761}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"161","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"3.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"4.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Ima1@gmail.com","mobile":"1-773-959-5605 x73199"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-07 02:40:03","tpep_pickup_datetime":"2023-07-18 02:38:18","tripID":"2743672b-c4d1-4ce8-adc1-f90b78d3a44b","trip_distance":".30"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794317,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":765}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"181","PULocationID":"232","RatecodeID":"3","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"36.5","improvement_surcharge":"0.3","mta_tax":"0","tip_amount":"7.45","tolls_amount":"0","total_amount":"44.75"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Lynn.McClure66@yahoo.com","mobile":"351-615-5695 x4474"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-04 03:14:04","tpep_pickup_datetime":"2023-07-06 02:52:04","tripID":"64dc9673-c0af-4dc6-b18e-b8a7f499dee9","trip_distance":"5.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794320,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":768}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"52","PULocationID":"33","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"3.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"4.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Madge_Bergstrom68@yahoo.com","mobile":"1-680-654-8869"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-08 02:14:26","tpep_pickup_datetime":"2023-07-14 02:13:08","tripID":"bbfc9834-4e11-4e68-bd28-ec927655f78b","trip_distance":".38"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794323,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":771}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"40","PULocationID":"52","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"5.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"6.8"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Della21@hotmail.com","mobile":"755.562.8568 x399"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-22 02:21:30","tpep_pickup_datetime":"2023-07-09 02:16:47","tripID":"8cd91fbc-927b-4e56-8172-a0350db5ad63","trip_distance":"1.15"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794327,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":775}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"255","PULocationID":"65","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.56","tolls_amount":"0","total_amount":"15.36"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Bernadine.Friesen@yahoo.com","mobile":"555-470-0062"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-27 02:40:30","tpep_pickup_datetime":"2023-02-22 02:31:12","tripID":"1f8e276b-346d-4e00-b5d5-fcd688efbe56","trip_distance":"3.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794330,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":778}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"256","PULocationID":"255","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.16","tolls_amount":"0","total_amount":"6.96"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Linda96@gmail.com","mobile":"1-642-292-8585"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-10 02:45:24","tpep_pickup_datetime":"2023-03-28 02:41:58","tripID":"3e76db01-516a-4672-8a7f-d3ddfb7d285d","trip_distance":".81"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794333,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":781}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"255","PULocationID":"256","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"5.3"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Rosetta_Beer@hotmail.com","mobile":"798.313.5258 x6339"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-04 02:50:26","tpep_pickup_datetime":"2023-06-01 02:47:31","tripID":"6b41ee87-6fa1-4f83-9447-2e1e98667d5c","trip_distance":".62"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794336,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":784}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"90","PULocationID":"255","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"17","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.66","tolls_amount":"0","total_amount":"21.96"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Carmen57@gmail.com","mobile":"978.415.9386 x8923"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-08 03:09:43","tpep_pickup_datetime":"2023-06-26 02:50:53","tripID":"d501b0eb-c941-4328-9d0b-83001f652918","trip_distance":"4.64"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794340,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":788}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"162","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.26","tolls_amount":"0","total_amount":"13.56"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Oliver.Labadie10@yahoo.com","mobile":"(366) 454-2917 x477"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-21 02:29:12","tpep_pickup_datetime":"2023-05-11 02:18:46","tripID":"e08cf524-22a7-4e75-86aa-47497b76f378","trip_distance":"2.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794343,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":791}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"239","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"9.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Dora26@yahoo.com","mobile":"(478) 966-5035 x192"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-10-08 02:41:45","tpep_pickup_datetime":"2023-08-06 02:33:12","tripID":"e2621721-e337-43b7-9c2f-eb2769bffd0f","trip_distance":"1.85"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794385,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":833}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"162","PULocationID":"239","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.7","tolls_amount":"0","total_amount":"13.5"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Isabella_Wunsch89@gmail.com","mobile":"521.252.8903 x15758"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-06 02:53:08","tpep_pickup_datetime":"2023-08-28 02:43:48","tripID":"5f8ead46-df6d-427c-8f4e-8c51485f3bc4","trip_distance":"2.41"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794389,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":837}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"237","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.26","tolls_amount":"0","total_amount":"7.56"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Marcelina_Cormier21@hotmail.com","mobile":"557-359-4984 x5088"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-05 03:02:05","tpep_pickup_datetime":"2024-01-11 02:57:55","tripID":"d0e83758-cf79-44e9-a138-2c8c84e7e1ab","trip_distance":".89"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794393,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":841}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"229","PULocationID":"141","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"5.3"},"passenger_count":"5","payment_type":"2","primary_passenger":{"email":"Evie.Pouros@yahoo.com","mobile":"277-513-8399 x256"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-27 02:09:02","tpep_pickup_datetime":"2023-04-09 02:06:09","tripID":"93968446-dafa-476f-9d7e-570304c3dfd8","trip_distance":".68"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794396,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":844}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"224","PULocationID":"229","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"11.8"},"passenger_count":"5","payment_type":"2","primary_passenger":{"email":"Christ.Kuvalis@gmail.com","mobile":"(441) 378-5258 x6569"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-17 02:24:27","tpep_pickup_datetime":"2023-11-29 02:10:40","tripID":"50587962-2be2-4719-a09b-b8eb35b7be10","trip_distance":"2.33"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794399,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":847}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"116","PULocationID":"107","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"34.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"7.16","tolls_amount":"0","total_amount":"42.96"},"passenger_count":"5","payment_type":"1","primary_passenger":{"email":"Alexandre_Walker@yahoo.com","mobile":"458-362-0622 x630"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-26 03:12:19","tpep_pickup_datetime":"2024-01-14 02:27:14","tripID":"da36b5dd-0704-400c-adb5-888be2f6ebf8","trip_distance":"9.17"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794402,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":850}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"262","PULocationID":"237","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"10.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Rosalinda79@hotmail.com","mobile":"(285) 935-0429 x620"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-04 02:22:29","tpep_pickup_datetime":"2024-01-11 02:13:41","tripID":"02412500-2ffe-45e7-bee5-a6666dc1d839","trip_distance":"2.14"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794406,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":854}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"90","PULocationID":"263","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.46","tolls_amount":"0","total_amount":"20.76"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Gerda55@hotmail.com","mobile":"551.761.2078 x469"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-24 02:48:40","tpep_pickup_datetime":"2023-08-27 02:29:15","tripID":"b21960c7-f5a0-4bca-9402-abf7bd246004","trip_distance":"3.93"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794409,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":857}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"90","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.36","tolls_amount":"0","total_amount":"14.16"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Corene27@gmail.com","mobile":"1-390-967-3132 x173"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-14 03:03:44","tpep_pickup_datetime":"2023-12-21 02:50:12","tripID":"d8253210-ce80-48bf-ad78-f9e9b02a7659","trip_distance":"2.01"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794412,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":860}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"234","PULocationID":"137","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"7.8"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Alicia9@hotmail.com","mobile":"1-967-665-8696 x087"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-24 02:10:42","tpep_pickup_datetime":"2023-04-26 02:02:57","tripID":"cb63a45c-d619-4526-88fb-7f020417342f","trip_distance":"1.07"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794416,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":864}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"234","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.16","tolls_amount":"0","total_amount":"12.96"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Rickey.Heller29@hotmail.com","mobile":"(315) 523-5370 x8087"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-11 02:25:05","tpep_pickup_datetime":"2023-09-30 02:12:49","tripID":"8840c605-b20b-41fd-a06e-c724e1a0a64b","trip_distance":"1.81"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794420,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":868}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"49","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"17.3"},"passenger_count":"4","payment_type":"2","primary_passenger":{"email":"Aurore4@hotmail.com","mobile":"284-324-0491 x19914"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-18 02:25:05","tpep_pickup_datetime":"2023-02-09 02:27:35","tripID":"e67ffdc5-4302-48a8-8f44-cee84c2fe73b","trip_distance":"3.84"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794423,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":871}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"234","PULocationID":"236","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"14.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"15.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Jerrod_Osinski@hotmail.com","mobile":"1-689-477-9312 x99969"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-30 02:37:11","tpep_pickup_datetime":"2023-01-27 02:20:17","tripID":"95ef5fab-051a-4e15-9a4a-9cf49ca65231","trip_distance":"3.78"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794426,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":874}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"232","PULocationID":"234","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"12.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Brody.Strosin@yahoo.com","mobile":"300-552-9715 x358"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-06 02:51:00","tpep_pickup_datetime":"2023-05-19 02:38:39","tripID":"7fa48c82-4c15-4204-840b-a633b344cced","trip_distance":"2.39"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794430,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":878}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"87","PULocationID":"232","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"12","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.66","tolls_amount":"0","total_amount":"15.96"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Karlee_Bosco@hotmail.com","mobile":"891-778-9086 x5622"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-09 03:05:21","tpep_pickup_datetime":"2023-11-24 02:54:32","tripID":"3600dd4d-d3d4-48d4-8220-f02200161d3a","trip_distance":"3.00"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794433,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":881}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"236","PULocationID":"48","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.46","tolls_amount":"0","total_amount":"14.76"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Madelyn.Ullrich@yahoo.com","mobile":"725.333.9195"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-09 02:23:56","tpep_pickup_datetime":"2023-05-27 02:14:49","tripID":"c1049b7b-cf57-486b-a154-cdf1309b6841","trip_distance":"3.42"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794437,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":885}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"150","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"55.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"14.2","tolls_amount":"0","total_amount":"71"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Waylon14@hotmail.com","mobile":"564-322-3411 x656"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-22 03:11:46","tpep_pickup_datetime":"2023-12-07 02:33:40","tripID":"9d42a2fd-9910-4c79-86d4-262e3b7f5fae","trip_distance":"19.97"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794440,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":888}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"147","PULocationID":"41","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"19","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"22.3"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Manley_Hackett71@gmail.com","mobile":"1-871-508-0977 x05891"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-09 02:22:37","tpep_pickup_datetime":"2023-06-04 02:00:35","tripID":"b8946252-73ef-4e89-9af2-22ade786e381","trip_distance":"4.96"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794484,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":932}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"244","PULocationID":"75","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"17.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Marcus67@hotmail.com","mobile":"546-381-5618 x705"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-19 03:00:20","tpep_pickup_datetime":"2023-05-01 02:49:25","tripID":"ac08ee2e-a254-4a8a-85c3-1003e93b8df3","trip_distance":"5.42"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794488,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":936}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"65","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"19.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1","tolls_amount":"0","total_amount":"23.75"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Rebeca78@gmail.com","mobile":"857.384.4490 x9274"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-21 02:37:59","tpep_pickup_datetime":"2023-01-10 02:15:26","tripID":"a14db847-b6b2-4377-a8f8-be62cbc6c58b","trip_distance":"5.08"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794491,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":939}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"170","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.66","tolls_amount":"0","total_amount":"9.96"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Claudie_Hegmann@hotmail.com","mobile":"1-273-731-1442 x2409"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-17 02:46:37","tpep_pickup_datetime":"2023-09-20 02:39:34","tripID":"6e4e34f8-2747-4298-9d46-3e52e1ad4a49","trip_distance":"1.41"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794495,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":943}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"238","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.46","tolls_amount":"0","total_amount":"20.76"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Antonetta_Davis@hotmail.com","mobile":"371-641-0671 x207"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-12 03:05:15","tpep_pickup_datetime":"2023-04-30 02:49:45","tripID":"68b10553-f67d-4acd-ae8a-bd11b02f805d","trip_distance":"5.05"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794499,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":947}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"48","PULocationID":"48","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.45","tolls_amount":"0","total_amount":"8.75"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Clovis.Grady7@yahoo.com","mobile":"344-298-3292"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-16 02:08:57","tpep_pickup_datetime":"2023-02-12 02:02:52","tripID":"6b6ab88d-6ae4-407a-9d96-cafc40925f26","trip_distance":".90"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794502,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":950}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"90","PULocationID":"48","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.65","tolls_amount":"0","total_amount":"15.95"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Eve_Erdman51@hotmail.com","mobile":"221.581.9132 x7964"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-31 02:25:57","tpep_pickup_datetime":"2023-05-13 02:11:55","tripID":"ad35aadc-0dcb-43f6-8140-47fc8f10d3ad","trip_distance":"2.20"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794505,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":953}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"229","PULocationID":"90","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"13","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"16.3"},"passenger_count":"4","payment_type":"1","primary_passenger":{"email":"Ernestina_Corwin@gmail.com","mobile":"1-240-291-8146 x5567"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-24 02:43:12","tpep_pickup_datetime":"2023-01-12 02:27:06","tripID":"4eb66446-fd61-4410-b056-ffc4f512eb80","trip_distance":"2.90"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794509,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":957}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"234","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.6","tolls_amount":"0","total_amount":"11.4"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Oscar.Witting@hotmail.com","mobile":"281-827-1431 x92694"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-08 02:22:54","tpep_pickup_datetime":"2023-05-25 02:14:07","tripID":"21963119-b955-47c7-8dda-d8804bfeadf7","trip_distance":"1.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794513,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":961}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"263","PULocationID":"162","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"11.3"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Stone81@yahoo.com","mobile":"889-393-8174 x1107"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-27 02:34:52","tpep_pickup_datetime":"2023-07-23 02:28:20","tripID":"2b8b5deb-9b99-488f-9a66-bf07b19fd97f","trip_distance":"2.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794516,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":964}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"263","PULocationID":"237","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.45","tolls_amount":"0","total_amount":"8.75"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Winfield_Mayer42@yahoo.com","mobile":"(618) 838-8049"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-07 02:37:52","tpep_pickup_datetime":"2023-08-06 02:32:47","tripID":"e3ca466a-f2b9-4e65-84dd-66201b11a92f","trip_distance":"1.20"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794519,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":967}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"41","PULocationID":"237","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.35","tolls_amount":"0","total_amount":"14.65"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Jaeden.Von@gmail.com","mobile":"1-504-649-6832"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-25 02:58:27","tpep_pickup_datetime":"2023-03-23 02:48:45","tripID":"3ecddca9-4247-475a-bd93-c572db096fc4","trip_distance":"2.70"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794522,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":970}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"87","PULocationID":"66","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"11.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Rosella.Rolfson-Gibson@yahoo.com","mobile":"(617) 788-8313 x2277"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-19 02:15:18","tpep_pickup_datetime":"2023-03-25 02:05:08","tripID":"5c3d9658-0579-4365-82f3-f567c6fab6b6","trip_distance":"2.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794526,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":974}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"37","PULocationID":"87","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"26","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"5.45","tolls_amount":"0","total_amount":"32.75"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Pinkie17@yahoo.com","mobile":"206-799-5614"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-28 02:42:36","tpep_pickup_datetime":"2023-12-30 02:17:57","tripID":"f2213841-8cae-4c86-aa17-ab8150b0353b","trip_distance":"8.20"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794529,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":977}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"17","PULocationID":"37","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"7.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Carmine44@gmail.com","mobile":"369-639-0569 x643"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-29 02:50:36","tpep_pickup_datetime":"2024-01-16 02:43:48","tripID":"d5f74519-08c3-4ee1-9e2d-9b84876e157e","trip_distance":"1.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794532,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":980}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"107","PULocationID":"211","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"9.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Ardith.Tromp@hotmail.com","mobile":"612-805-1546 x7728"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-08 03:03:30","tpep_pickup_datetime":"2023-05-30 02:54:05","tripID":"4934ae36-14f0-46ea-8087-497b87aa6d5e","trip_distance":"1.90"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794536,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":984}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"140","PULocationID":"229","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.26","tolls_amount":"0","total_amount":"7.56"},"passenger_count":"5","payment_type":"1","primary_passenger":{"email":"Reinhold12@hotmail.com","mobile":"(478) 224-3991 x93298"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-15 02:16:51","tpep_pickup_datetime":"2023-01-01 02:13:30","tripID":"1d290a9e-1244-46ef-b22c-08336ea56864","trip_distance":"1.08"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794539,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":987}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"230","PULocationID":"161","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.32","tolls_amount":"0","total_amount":"11.62"},"passenger_count":"5","payment_type":"1","primary_passenger":{"email":"Eric29@yahoo.com","mobile":"774.435.3330 x4929"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-30 02:44:36","tpep_pickup_datetime":"2023-04-29 02:33:54","tripID":"4ef82abf-f75e-425b-a0d6-9b8d4521addb","trip_distance":"1.39"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794542,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":990}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"234","PULocationID":"230","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"9.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"10.8"},"passenger_count":"5","payment_type":"2","primary_passenger":{"email":"Rashad.Klocko@gmail.com","mobile":"1-902-813-8402 x18121"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-18 02:59:59","tpep_pickup_datetime":"2023-06-16 02:48:50","tripID":"cca51610-3623-4353-befe-465bcbcb6e5b","trip_distance":"1.91"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794545,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":993}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"107","PULocationID":"249","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.45","tolls_amount":"0","total_amount":"14.75"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Germaine.Johnson16@yahoo.com","mobile":"1-309-548-7894 x77988"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-05 02:55:40","tpep_pickup_datetime":"2023-09-28 02:41:47","tripID":"3f2e3f93-9088-4ba1-aad0-779c04192b96","trip_distance":"2.10"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794587,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1035}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"237","PULocationID":"151","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"11.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Taylor.Green@yahoo.com","mobile":"(571) 811-5436 x433"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-02-06 02:10:38","tpep_pickup_datetime":"2023-09-02 02:01:05","tripID":"1eb1e4c1-9c23-4dd4-8915-0986404ac3e8","trip_distance":"2.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794591,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1039}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"226","PULocationID":"230","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"17.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"18.8"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Keenan.Brown60@yahoo.com","mobile":"447-236-4948 x26679"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-05 02:44:31","tpep_pickup_datetime":"2023-02-10 02:23:48","tripID":"54287dc9-743e-40b3-a2e7-16c8139a405d","trip_distance":"4.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794594,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1042}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"48","PULocationID":"211","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"15.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.36","tolls_amount":"0","total_amount":"20.16"},"passenger_count":"6","payment_type":"1","primary_passenger":{"email":"Joana.Murray16@gmail.com","mobile":"1-205-355-1857 x6037"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-28 02:29:36","tpep_pickup_datetime":"2023-09-21 02:11:57","tripID":"cfc6db54-05f3-4ec3-9fac-35bc0642b5b0","trip_distance":"4.39"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794598,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1046}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"186","PULocationID":"48","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"7.3"},"passenger_count":"6","payment_type":"2","primary_passenger":{"email":"Pietro34@gmail.com","mobile":"(214) 940-1846"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-30 02:42:32","tpep_pickup_datetime":"2023-06-01 02:35:36","tripID":"40ddb784-9baa-49cc-b445-ba7c3c3a5350","trip_distance":".74"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794601,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1049}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"239","PULocationID":"186","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"15","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.44","tolls_amount":"0","total_amount":"18.74"},"passenger_count":"6","payment_type":"1","primary_passenger":{"email":"Hunter60@gmail.com","mobile":"1-605-974-7545 x2362"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-10-05 03:00:57","tpep_pickup_datetime":"2023-09-05 02:43:53","tripID":"1b84cc94-63cd-4bf7-a5a2-9adf220323de","trip_distance":"3.77"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794604,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1052}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"107","PULocationID":"231","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.36","tolls_amount":"0","total_amount":"14.16"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Berneice0@hotmail.com","mobile":"355.597.9064 x85306"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-12-02 02:15:56","tpep_pickup_datetime":"2023-09-05 02:04:15","tripID":"09cd6414-1451-4e47-903b-8e6020b85f0c","trip_distance":"2.59"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794607,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1055}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"237","PULocationID":"137","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"5","tolls_amount":"0","total_amount":"16.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Josephine_Thompson@hotmail.com","mobile":"(816) 899-1039 x9110"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-15 02:32:57","tpep_pickup_datetime":"2023-12-19 02:22:15","tripID":"0ea57d99-2441-4a96-aa30-d50601ae805e","trip_distance":"2.60"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794610,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1058}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"223","PULocationID":"170","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"29","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"7.21","tolls_amount":"5.76","total_amount":"43.27"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Blaze29@yahoo.com","mobile":"550-296-4027 x96854"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-22 03:14:31","tpep_pickup_datetime":"2023-08-11 02:47:39","tripID":"e7cd1818-3ac4-4c26-be7e-2811d3cf6eb9","trip_distance":"9.49"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794614,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1062}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"114","PULocationID":"79","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.7","tolls_amount":"0","total_amount":"10"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Gerda98@yahoo.com","mobile":"(771) 247-2009"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-10-09 02:17:42","tpep_pickup_datetime":"2023-04-14 02:11:46","tripID":"8f1d4aeb-d3a9-41f1-b869-dceba333ddb2","trip_distance":".90"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794617,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1065}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"114","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.85","tolls_amount":"0","total_amount":"11.15"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Lina78@hotmail.com","mobile":"(737) 801-0055 x389"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-10-05 02:28:10","tpep_pickup_datetime":"2023-02-02 02:18:49","tripID":"39948704-84d9-439f-a8f1-a041ec9d6882","trip_distance":"1.30"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794620,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1068}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"238","PULocationID":"79","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"23.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1","tolls_amount":"0","total_amount":"25.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Zachary.Kozey@hotmail.com","mobile":"1-367-977-9321 x10540"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-25 02:59:40","tpep_pickup_datetime":"2023-09-07 02:33:22","tripID":"edc121f9-b36f-4b24-afa9-b9fbadbfa297","trip_distance":"6.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794624,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1072}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"61","PULocationID":"162","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"28","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"7.01","tolls_amount":"5.76","total_amount":"42.07"},"passenger_count":"4","payment_type":"1","primary_passenger":{"email":"Ralph.Greenfelder@hotmail.com","mobile":"646.847.7460 x03211"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-16 02:35:52","tpep_pickup_datetime":"2023-03-09 02:10:17","tripID":"7f53bade-5ce2-4666-8c48-ed3af53255f4","trip_distance":"8.88"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794627,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1075}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"49","PULocationID":"49","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"5.3"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Jermaine.Homenick@yahoo.com","mobile":"264.624.0022 x756"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-05 02:41:11","tpep_pickup_datetime":"2023-04-22 02:39:34","tripID":"64d7eb34-3d39-45c5-9090-acf4b28d8c6f","trip_distance":".61"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794630,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1078}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"48","PULocationID":"148","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"14","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"3.06","tolls_amount":"0","total_amount":"18.36"},"passenger_count":"5","payment_type":"1","primary_passenger":{"email":"Annetta_Heller@hotmail.com","mobile":"(754) 546-3157"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-17 03:15:03","tpep_pickup_datetime":"2023-01-23 02:57:58","tripID":"4d482dfe-2fcf-49e3-8fa0-daa7b7f05655","trip_distance":"3.55"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794634,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1082}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"164","PULocationID":"230","RatecodeID":"5","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0","fare_amount":"20","improvement_surcharge":"0.3","mta_tax":"0","tip_amount":"0","tolls_amount":"0","total_amount":"20.3"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Juston25@hotmail.com","mobile":"638-267-0522 x2725"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-24 02:24:26","tpep_pickup_datetime":"2023-07-22 02:14:37","tripID":"8bd8e0b7-c3f9-4080-af62-7e6f7d62033b","trip_distance":"1.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794637,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1085}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"209","PULocationID":"234","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"16.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1","tolls_amount":"0","total_amount":"18.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Beulah_Reichel@gmail.com","mobile":"268.992.6069 x11480"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-22 02:46:20","tpep_pickup_datetime":"2024-01-02 02:30:49","tripID":"6be6e503-53d3-449e-9b6e-3a8ab535d512","trip_distance":"4.70"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794640,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1088}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"141","PULocationID":"249","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"12","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.66","tolls_amount":"0","total_amount":"15.96"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Agnes_Hansen@gmail.com","mobile":"(549) 807-8566 x7952"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-09-25 03:10:18","tpep_pickup_datetime":"2023-10-13 02:58:01","tripID":"c6d635dc-daf6-4c45-8e27-7fe0ca048dcc","trip_distance":"3.40"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794643,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1091}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"13","PULocationID":"170","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"14.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"15.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Judah_Johnson44@hotmail.com","mobile":"363-606-8572"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-29 02:21:18","tpep_pickup_datetime":"2023-09-18 02:05:14","tripID":"1412a958-bb92-41fe-932e-a5299a3fff0f","trip_distance":"3.96"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794687,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1135}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"79","PULocationID":"261","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"12","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"13.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Marisol.Beier-Miller@yahoo.com","mobile":"1-309-668-7163"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-28 02:38:04","tpep_pickup_datetime":"2023-11-07 02:23:14","tripID":"c0cdc45b-22e7-4ecc-a627-5c8b67d87e46","trip_distance":"2.59"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794690,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1138}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"42","PULocationID":"79","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"28.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.98","tolls_amount":"0","total_amount":"32.78"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Thalia_Ortiz88@yahoo.com","mobile":"407-555-4002 x44648"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-24 03:03:01","tpep_pickup_datetime":"2023-07-25 02:30:57","tripID":"e0b47894-d40d-42c2-8336-c8b9c4ed3a33","trip_distance":"8.43"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794694,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1142}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"246","PULocationID":"68","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.66","tolls_amount":"0","total_amount":"9.96"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Hans.Smitham9@yahoo.com","mobile":"(339) 364-0841"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-19 02:16:50","tpep_pickup_datetime":"2023-10-09 02:10:41","tripID":"cd5a37af-4c4f-4424-9c93-b1244ad04fd7","trip_distance":"1.43"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794697,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1145}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"68","PULocationID":"246","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"5.3"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Lourdes38@hotmail.com","mobile":"1-751-666-0619 x542"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-01-02 02:16:50","tpep_pickup_datetime":"2023-12-20 02:17:22","tripID":"1f8d2fa0-a798-48b8-a9a9-e20e316d5531","trip_distance":".66"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794701,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1149}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"140","PULocationID":"48","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"11.8"},"passenger_count":"2","payment_type":"2","primary_passenger":{"email":"Riley_Yundt-Turcotte@gmail.com","mobile":"220.269.1280"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-02-03 02:35:05","tpep_pickup_datetime":"2023-08-14 02:23:08","tripID":"3ce204d3-9cba-46c1-85a1-f21292758378","trip_distance":"2.41"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794704,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1152}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"141","PULocationID":"141","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"4.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.74","tolls_amount":"0","total_amount":"7.54"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Edward14@yahoo.com","mobile":"320-494-1809 x531"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-30 02:39:22","tpep_pickup_datetime":"2024-01-14 02:36:30","tripID":"cdd42be7-9a59-4059-b8f1-dc3dddfdc0a2","trip_distance":".65"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794707,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1155}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"236","PULocationID":"143","RatecodeID":"1","VendorID":"1","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.25","tolls_amount":"0","total_amount":"13.55"},"passenger_count":"2","payment_type":"1","primary_passenger":{"email":"Blaise_Leffler7@hotmail.com","mobile":"275-547-8169 x12135"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-26 02:27:43","tpep_pickup_datetime":"2023-04-19 02:17:48","tripID":"986787ac-976d-4148-911e-a502b40ed9f2","trip_distance":"2.50"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794710,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1158}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"239","PULocationID":"236","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.76","tolls_amount":"0","total_amount":"10.56"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Anastacio0@gmail.com","mobile":"592.584.9471 x84515"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-03-11 02:25:16","tpep_pickup_datetime":"2023-07-16 02:17:47","tripID":"9b42bfac-1876-470a-86df-4f1dcff160cd","trip_distance":"1.68"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794714,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1162}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"75","PULocationID":"142","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"11.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"12.8"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Brooklyn.Feest@yahoo.com","mobile":"538.970.5053 x607"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-04-26 02:44:12","tpep_pickup_datetime":"2023-09-26 02:31:06","tripID":"7cfc8967-0cbd-45cb-a5a8-a9e7cb05f407","trip_distance":"2.73"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794717,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1165}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"145","PULocationID":"236","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"14.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2","tolls_amount":"0","total_amount":"17.8"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Roberto.Lockman@hotmail.com","mobile":"409-514-9426 x587"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-06-18 03:04:58","tpep_pickup_datetime":"2023-11-02 02:50:32","tripID":"7255441b-919a-4cac-89b5-96f7ff21e0db","trip_distance":"4.12"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794722,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1170}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"163","PULocationID":"48","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"10","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"2.26","tolls_amount":"0","total_amount":"13.56"},"passenger_count":"3","payment_type":"1","primary_passenger":{"email":"Margarett.Sawayn@yahoo.com","mobile":"(866) 903-8125 x36548"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-15 02:17:06","tpep_pickup_datetime":"2023-06-22 02:03:24","tripID":"fd218127-8604-4a99-aa7b-51852fca4b2f","trip_distance":"1.24"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794725,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1173}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"239","PULocationID":"237","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"7.3"},"passenger_count":"3","payment_type":"2","primary_passenger":{"email":"Kaylah_Shields55@yahoo.com","mobile":"(924) 980-5772"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-08-19 02:26:50","tpep_pickup_datetime":"2023-11-17 02:22:43","tripID":"94df85f8-4b0a-45eb-a45a-ccc11dad11ab","trip_distance":"1.24"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794729,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1177}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"231","PULocationID":"158","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"7.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.76","tolls_amount":"0","total_amount":"10.56"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Alverta.Armstrong44@yahoo.com","mobile":"(878) 812-6613 x69201"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2024-01-20 02:21:36","tpep_pickup_datetime":"2023-01-20 02:13:30","tripID":"7b16e77f-898f-4793-879f-ccf0c85798d6","trip_distance":"1.37"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794732,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1180}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"107","PULocationID":"231","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"8.5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1.96","tolls_amount":"0","total_amount":"11.76"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Esperanza15@hotmail.com","mobile":"692-945-2420 x1931"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-11-15 02:31:22","tpep_pickup_datetime":"2023-01-02 02:22:45","tripID":"e11b4c6c-2c0d-46ee-ae04-0700d67c131f","trip_distance":"2.00"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794736,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1184}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"186","PULocationID":"107","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"6","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"1","tolls_amount":"0","total_amount":"8.3"},"passenger_count":"1","payment_type":"1","primary_passenger":{"email":"Casper.Romaguera87@gmail.com","mobile":"491-889-2259"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-07-04 02:44:09","tpep_pickup_datetime":"2024-01-13 02:37:12","tripID":"5d52db41-4838-4a12-876b-056abd939f62","trip_distance":".92"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524794740,"processingStartTime":1711524793552,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"44f1bd01-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524793552,"timespans":{"extractor":1188}}}
+{"dataset":"new-york-taxi-data","event":{"DOLocationID":"170","PULocationID":"164","RatecodeID":"1","VendorID":"2","fare_details":{"congestion_surcharge":"","extra":"0.5","fare_amount":"5","improvement_surcharge":"0.3","mta_tax":"0.5","tip_amount":"0","tolls_amount":"0","total_amount":"6.3"},"passenger_count":"1","payment_type":"2","primary_passenger":{"email":"Eudora.Dickens@gmail.com","mobile":"(252) 559-0694 x683"},"store_and_fwd_flag":"N","tpep_dropoff_datetime":"2023-05-22 02:51:52","tpep_pickup_datetime":"2023-01-16 02:48:04","tripID":"35cc2967-ce3d-4dac-a75f-74347e277630","trip_distance":".82"},"obsrv_meta":{"flags":{"extractor":"success"},"prevProcessingTime":1711524803950,"processingStartTime":1711524803850,"source":{"meta":{"connector_type":"api","entry_source":"api","id":"","version":"1.0"},"trace_id":"4b1516a1-ec0c-11ee-be1c-fbcf66548089"},"syncts":1711524803850,"timespans":{"extractor":100}}}
diff --git a/tests/sample_data/nyt_data_100.json.gz b/tests/sample_data/nyt_data_100.json.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f6cd64609fc390927c770fd54f786ba0aa316681
GIT binary patch
literal 10228
zcmV<QCkxmgiwFP!000000OfsMb0f#G<@^2$hMqJLw>l~-v%dIbeObEtu)UVnj@=hV
zbyu~BI~ZUBBqh=g|M$%T8lxdqt%g`l$ZCh>utjo6aM*`=GV|m)|NJ0^+i;U^Ke+s0
zlYSHr+udt%8}3&kzx{*%`XK#~HvG<ie(>p^zutD?cD3F7_ow{9fR^9;m;dwljYRy$
zzlPh?ZDTsTiQn+Iw29l@yMOJ&E?vcR8&>O^{F%G$=KFM$Kgre2-LAV1yYCZ!X#V2&
zx4XdqS2p_i=fa!qU4B6y`2E{fZ{F^<KcqMOA|L%>*&x67&26~iZ|2=+xLv*d=@Y01
z_HWy+*Ec`CRqo$(8`eL)2~^|t2mku758j5Go5Zhmchw!=^+A5?!~V<PKD<rGH|<t$
z!tUYf-5<!`W_rWlFn_?uyY2RM(&`W2=QlLn_Dz1J+jh0ie_S~dR+AvKE`QCN+im`?
zH}R_fk^j36-}49kDZj|K>FrhAZQpMD{)&HC>2~#I{|Tf*A+<obka#KSQt9JIzg=~&
z@816WgPj1nkm^#a%l%LFcDH)F|7j692g(@XdJlpuNfCl=g$t-PV2G_be*9Hj-Q0#v
zxBsk-l=-!9+ne1FS8w<yoxlJ5+V?;2{ZF;y?~6b5=C13~&CULAcb9(n%Wj+hb+!5a
zyZs9U3!oKQj|OG_kmI|*y$!qDryrxW;=iyhKL#m(o#Pkg=WCPl*E{}*!rN7TxHh-^
zBRYQg@Wxd<Jb6E)-3>pG`RCc}znA=y)7AdnL_M@of=He01VBonP0)$H?RsN0IqCfV
zNB6MlZf}0}0`p(}&D*ef`gdX7zy9^F|Mkyj19FeOCxBGsCjhDD17s~6r8yOjXv%Oj
z&w%6SY5POiY*s*jxZd8DAgPsUG}y*ju+N7i$UP(#<MC3dDM+GFHh(Y>|MNa1d$fIr
zu@fmuE6B9Okc_S9r0T8ftaT<7APJ4e`5{RQhon<gkhK0eC2~GQ+S(E+PXeTC+{tiM
zQ*iwAdbMf32<!CV<R2cw^>+Idj-*ZFB{XQ!Ufcut=VFqCR9q=D%+O!T$ubE7%pSn*
z^1vL9m^4c2(6ycL&@v{YCDFG02x`~KzE>@&ghfok3t-YN9h17QOqxQOv}df6wV1@y
ziX{FDP)^p!ukThjf$Rt#!8D4PBcs<B_evRE#am%t_cUBeJe9YS97ygvK+4u?X@YRR
zjZCE}3Gb5=p=|+M9eQnik+*WKog0t@O9!N{1f(idNh!~UNLPnPXxxdsm4DyQ$Xoxq
zy$QS3w)tec*=--n(6q`5=ajHykfG@h9PpXY^g@|^OUbO%PC?TjTS~kn^*%JUBon0V
z1oPCE`71wvzOzXr^3;WnBqb_9lgmBK56z|XS6|6re-2D<&xT7?O<!HJFZR>WMB5O}
zH{tp7+v~Jmg{x1uTjsIemnS_0y5XN~gVLyrd!r1Q3UtB|oped&gzi!gg%ckYPU&Oc
z%T@`)j~6r9j%%huExOdpq@C0jO9LO**cT|AOJ}maj>)Rhzz6ofZO;TyRSQsR>}j=A
zxkiuF&gTI1s~tbPtIgHNyY%{XD%Vc5j?0r2Ol_$Gm;BG3&?+3D_RCdM?W?9VTxH+K
zr460NBqI~r4h326FoX0)^e%d75RBKQs%abLXUC;p`XJU<4q{6krrQ5f{?8cu6c9N%
z_W4U#zX{FX!}{jhnXy5v=O^A-%hXaq1ths`bbD?pFSVb7q!nsEwI%-N(ID1=4BGTb
zIN5qZsz(uwibBb*C!b8$Cnyrx7Z}9qr4w0@bxc;N(kRxPZxrj!gGoOpCcnGeZPL!-
z7#u&epk7@ll>uq#;+{}He;CV9GXii=Vd1o$r8({_AK?-db|x4SzSZ1dl1JfU>V=KQ
zBz~bNp^Qua0;O{46c%JHhb`bz;neVFJ{18}b6C~b6J-RZ(KE^jz6=|N#?{xM4RMvq
zEv5%4EO8@eQD596RLy_PFq&0#DJRRO6Uy(Iti9Cu2%avgC<DVcVwWWaEtgG-!nDab
z>QU+572v5FGry$3EPXT!vTio3_P;8BwWeS`c$#w*P21pnMf0y)e#4D7Wk7zYtbcXk
z@>o_PmrPfQNV#tpQC-61%m<G01H&KxJ|bIXJKJV`kCRDIZ-Gl^%e>Zx;1UMq_rv(7
zh-~}|0CMp>7V3CxQRH(@)g9IW5}E_~n#81T2<EGm-*(q`4{QD;U#B;(M{1=uvN4`t
zpkiWq&+x2SW75!KqPbLZ3YmBmA<&auNd%i*bW(7Q=N8lTod}^*B2jOYbzO_8FCkOC
z05aX;k!k8mCAtDK_kGNnm~8DH^CVm<IaVyyGjRFQ&GqfWrui~-{IHH}Nel|h#Yuv6
z7xx~NtB}Ql66!Ge@s}E=aOnpSle`4?J}xmB<t_CBdWeF8&RY@}MesTSH+|${375Ko
z`GM&c-(q5Ai;2Z??AdU+*Lv3Um#5)U{|XX2t%=>PW7=GOxxL=pj7@*IBs8S3k;d4I
zd*f=BmwMQHpuD6>Skkb(%$v?ft!0~}Zjq$uBRWB<&EkicgqDg@i*0bRr25e>!DRi?
zmn49dTM}7N7aXv99$ZrG;0I2<A2G9@k9@k97JM8Y(ryfz+{!7#9i|p!HE`ABvRQ@B
z!wM7i37S0OO1=MJ;JW!fIvqces?|n7+qHsPnJ2UqMGG=Gm8|rANx1$Jb05F>Hj`@G
zOe~ODIi>xXk<3;Oez2j_u*rPZ&Iry=)?xR$`DVKb*I`p8wE2ii0|-a(>?LLf*?=>x
zFKKdFLc_9z+fnlGgOWSQ<dZ{@67g?~Xt-ojpO|s`J^~}M=Lh=dpv1<#04f*HWT}?P
zmQd-Zgava_+3K44b5QD|2I#ROPtGd%`7}#qH_g6kI#(9_eCUmQb)l7F!id~!5~!-O
z>4)j8I!a)F0ihs;k^`xIeD9UoVA>v|Fd;BP9gt`}(_7PFP+*m`sVyMXzCf>O<>L1x
zRJSiNr5TW!Qsm88HqRn505j*p<lF1;dbJDQlu-$STT5Z26xJC=C0gYwb|zeS6%M+A
z=`E%nRP4%KD)T-nH6a-fLB!U1L80wLP|=DOb=S2RbdRN-iE8KtP`P+Et1HDn+WZ`r
zd#IkNY|7fQsZXY}wox<E*{{OtcD>rT5w8MZBCCaBq&Ud8UtZdp18F7XUc#>6@}Op&
z=ra!oCUmJE6-+`v=(4~-@smhx=!H{VCosgWHLW9$rDIHb0ZMD-;wi1Jq_oe|<^3DE
zh@4n5<(M$9ur`@2nV;+)Zt|!9A~7sC8%O2R>6~wjg~p<);jxC1w6`MgmwHlVz@6q{
zIP0HUiTJklgi4^x!g?8{@KJYyv{5E@ou#hCW7SLWSXexd)s?y*Rp>PFd<u32=Yb_R
zv@_~uKU=SEf`+lFELztn?;7n7_g)oOCCp-BFRvqa36owTa1`!oZZ9AAmaPn(;s=jR
z6cm9AxiVU4=X-5Zrj}g+lcuq1eoQW&#hS{QY-wS#pVOWPlTf+GJh4#Xuh17vBJ#^@
z?31p(2;ut?fh<~KwdUT^`|KqGvZl9GKk4ykiok*t@>o_M_m)BSF>2$5iiruVl)d1}
z*$b15RFHaQT3tXTHgfh0X$y-du%^;ZL>CmY)0>hRVR@#@5*R%#p=SW|o3wt|gyx&=
zZpH7ty-kWn<Ugc@H6R@SI4|y{tlMM4VP#@2)#M)24g*;<Lxp{3z@<ZKW3L3bz(o^G
zE4<GCs(K}ROWt;+Q%pTH=Ci_6Ena0{>Qn|)7|81LU6;@~kU6USc}~UrY#q0o=JVCf
zI^1kGV<TCoNO<tVA~VZ->nf%`5(jn)FtF28EVv}|UU(msEq2CO(+dlMizfDxUnGMV
z7>>P_sFf@k7rewgRxO^%+RAyXDRh@~zIklzmINDh;>DLAz4&^LTK3bhxlLD}@4|I>
zGlophB$T`s#v&71?WkrT3yjV|mUj@qPV7qd&;)$qDH08(kf0Z!vty+-*|D;ZOn(FS
zmsYx%N_>`HV)COF&u48dpDnF0@qEgkN*CFa$FU<dvp{C2-SN_P7p^|*`?S0Lhr=?F
zHB4kt8bQV|FdcbcbNS_lRf)8hG>J^dUwF@Z?Gt30L@jWc6hT{lWdRwPzUzdiHlat?
zxuW_9USd^(i=WBbN}s&as^qZR{CymlVB=0)Vq#X-&bGw-Cj8HKXGY9q^M$lRBamuV
zml=<Kur8Rs#MJu_R+proOnww)y>2T-XF^~St9lV2^+NWY>Ah`{d}$u5U%+nNF5cT2
zD$Sist>v`K4=tp(e9nI4i?mLgtFQQ7ck9<lkI3*q<P#1qKSFD(xl}r;)B6ApI@w9H
zpP$adn8Q9oW~;OW&A%r)A9dbYw!-JdiEmpV?|Rq4x!TLc8$3g$!*kJfeLto>liUaE
zZSnkxmC}!@*Pqj1e!L40VWUR<ez>uGbz$=1#{NcHAwtcOv>NK5I(mfsGBgh)HrLHZ
zS6{tMCL(o$2#`r^up-D9MAvs@l%x=KN$+D`g2XP~>lrG|o=e!AG+&<)n&;V-jC!vZ
zswPf>^1HxK#;*C>b~9EnF<(1^5#IS4`z^b7kUeGEvp+vM$Y_<n#77q;mJ%q?T)~a)
zxl;C4v^D}$<YZg#xmJdfzt2kydlr8httwBW_r_Jf>6O<D+>?~XJ-^1Pa<WQ(w%hQx
zdiC|{_BO4>w{KRr*JBmac@heU(Wx?%MPn-UvHML4`O~hX?a>?P(H*Ar*61!s(I+mI
z<ZLHG%av18Qcl@uK$gr4sL}Hybn#cxs`g6S%4ZbJ8Px^V(sgwzmj#-Y%YL!FW0-B6
zA7!%+;MD~gZs&~h7xy0P^7amGCK1!wDQvo<;SXHuN5&#xymGNe(Z@V0_OTb<8z+(`
z<*n}Wdkff9jhtUwu=pcsRe2;`;<D2Q2FtN|ewK25z3Lg2SKoA>uOBuW7>O;>HVaXO
zR#mPda6Az)m+}M?u7f=uxhJC^8T-_Conb<o<?1E@6WZj2_JpA|<j|D^_sr}*&vx-A
z(yI1Ex@cTr&N%$3J}5D#+KS+~uZVm`!Td7C6j$L|jSAS6mkmGf4RD#Ng5^QGjx=a^
zCOo+8Ao!7wc9)$7mGT4Fl9x<lvr8T`TG2(N`shhpX$r9PFVJW@xp*2&bu?CCu~wc>
z@>AVtp6Vn!QY#C9oUD~!uhy%uNq%%s!cVv`n*Z0>9E53P#V|InJ`Q^l2HGSl%|Ku^
zJPAeU&}Tspii91vy~cXsj11YmucsJo46>{!n16%m7JnD5D(|8X?vzha3Fj-8)dpNA
z>hSEWGQ5x0>HZ&#ue7tH%_W1<Yv!+#`!}5>vZ`i>DTkv<a3>gW9f1sLSa`lO;G&RJ
zpJ{bZK~M+WT((Y&fSR<?Q5h;VmoG5)af`o;R+V?rMRCCykqMRR0&L`oQy<shjM)74
zI&4-CVH}trDx+RqpyT3X0Wa=dmBkMYdI8ka0@>_5vPZFkC-Kq2K?r$F3@WG9=FYcG
zwigCeFs;tBpAwa&mnfRBcq*$aC$g?!CUQpcx;<0P9G9*?&wp#PO6veLo+V_=s|%}{
zS=t)C&yZ59F6n4>H*rpKz^@yu4c_rsMHoCvVX;kuz_r4qt{2hylwx#U3+18D{MPFZ
zaOsdVFuz;D0jFmg`n*JYIpT|lqrQ00n#W#;JAQN@oErnC27l14E6T0~XrLwy(=eIM
zI@x?AGO1hl?{%3^5=#^WC6i!U%}iEzPH@i|xz&WEtyRtx<K5j0NPgVnFQirFg>=ai
zZ}J3l#x2P?=00vtne3<eo7|*y_0^i6C_6fib=kcHsWIL$k(CZ>GBM3iHkFwg%F<60
zTsY2(-UTBlPl5j<q|CR`SzI52>vAS$Y<t}XeyODv)6ecF;ue1ztzO_Xdb()N2h956
z&uP)oc2*<Vk79VE?fBRSO}@QUf(xb2GueE0Tt#MasDRT$SvNQqF!NQ9R0SRbXb(vQ
zm(RSqsD<ks2u)U|K3eKb(Rp3l`BAy}nQY~4v@hs=rt>i8LnT(nBKU?1zyxTt`Z+kY
zY3z*BypPsx+TAw)_iht!%UcrzihprH<0;=qlRQL!JlncJ?E`BGFZHAgm);8u-b=TA
z=fY*82L2g~E)oiAJ1bhPf-n+186#bU@+NbB7cRH>`)E~pAFWg=O=xm5J7bV^ZP~OZ
zRZW;Neea8Od%H?*boOHnHkP=DmeMjNwXf+g)xN`IZb~O~nD!VYq%Rdb#w7ntooxkB
zzW1^&JL9#)rQM=3zPF^xd(8PoKMohqWNoFopy(%ZHbg>o$;1Xu<FeoyKO>p_Yf4?x
zBhFhNTIF6{Xl?S8C3}hJstaj5+;1s=36la@fAkVDPsDhtcQQ$r1oVjx7t*aUNtl+R
z$G+2jD2vx|{#v>C3u#q(Azc)&PmA$pY%$NH`Z05w$bOn`@=R9aSV)P6R~Jh8d?5`N
z_x!sPYs7d*&-FZ$J!M_u2O|;nIMf~cD0{C3bS?;z#19}BO_8Dju+gfB#iK7fe~0Op
zeu9Zu=_4WwsGMA2{u|AG4kr_*Xz^z3EdRdRx0X0M`pJd`+^q?)N?qK0Tf<@XFv$UX
zDW_^B9Cw&m@AIgyR9Sgvvm0IlcQ}l0bx$Iq(w**1@}b0GhlYG!Q0k?FlCUyxuV^EC
zzPZl~f}fKqEY3(_KS^=Nz-a!M_<JrBS<FXlXd7vD#jq#)HPc`Si<841JDh5Q<6`+f
zD5JL$yVi4Q^x3J8scftHaqJywl@LjWQg9G=2(z0zJHPZycci*Z_p%wDIiu(LzX2+r
zBg6Y-+df=<%Gg+qmCNkaVU1=2iZv5hy)S;a&w@Uos~i?2_GzUgW~)+VPoLaX22H|6
z0)f2&ZYWW<Sh94+#?aiL^inMylv-Bak`x4ZYSN){K2TOh)1G=sGNLHZ&*&+?yvvs)
z0Y@VCtk;dv&^RUK#l5u}Ysjk|c9w`k>H8%mff_;(lpj)T-v#WU1|w}JobPnLgeDOZ
zca$CazK@~9)|cXfU~mqc7m|x-uey%C{^pRZER%3*nKYwSvH;8Ji_7gM++K%G^T*xl
z_NKcwBm7ly;qqCdlT&GpH8tqOVdFu1$eI1+dCy=}c~tAYQ>=&3r)Z?erz3*;2%=>^
zDYU|<RUdmxr7>({W}m`(=@$=9JAZ3=PD#O;0!l{nNuE<NzYRZx%_e{=LsFA((ECQ}
z8W-n$*L9Et=+jzCJh~Y?+Ew<g_Dc64d>gGGGEM}gjp$`B$y(Abltg#X==s?zES|lZ
zI`(=<gZ>S*>XrWLryq9VtcP9uRb|5u)aG`(89(kKe0AZp%2U{9x0QQ=>tR_5`jRH?
zoZn@V(D1?Utnfl7dnco410+G(cfw=L?#>->XW6D?N_NiB*x6<39?ar_N%LD2oHe_>
z-fHVqV9vM~`R_Mx!safmn=jHTrp+#`y6dsHXzRGoB-MDMn8-S-bxj1O66%=EKCO+V
zLEa8-IMjR>fe<nEoiw7;$p}i0gmb+VDv_mRF}Na=<!|BzFuHguYwD=%Z=M!iZMmhI
z6G1wo{O;@R-RdS>eQ_6Wwi|0l-9#8sj*I{V+AG&9thaJ<kRDvu<q7#DhanKT!Kdq;
zV?pGHkU9`X^-hqLIS0_?hSM96s0<X<KG2x?!KoI{Y;7g8Em?1!)7!baex5c1Qlln6
z_$*%Arq|)>?_vEq?ZoHnlw<e6j~G2?11?1k8Y6QUW>xN<AB3`bxHH*&s$)nWJ~@<*
zmLOSgL<XuL<i4||p8rJaAtbbjMUG1^0@1~DTwBL+flAFMosZ-G*8b~-&XcAP`q7)P
zUNJ&fx3~OMjr++eu!4WG+~*-?6+5}gtlHzQ@?aJ+(Sg$N!H-8n!5N8dn{t9sFGyk%
zL3$E_AHvXef$MAOwB!YhMZ8)(xwVz#mI_N0IHNERDo>^9RApYiTb{86{do5fHlg|7
zcdPH$>4Bi!d^&Hq=3qnKn~QtYAZv67M<pyBtqD$nr;<<d_n=cc<%H|jbSCGQ1QD?J
zBE+cs07-Ygq|n30&#s&GxOA5hbd@e6rOE9XjSQ-_K&LgHzsAbogy_Ay-HLy%hpYrY
zc4^|he|6!}3e>j7!~l<3U+xko4WHdl%grY~d7aljB}S#r_n1=k!Xr1K33@|qAPQ8<
z`ofnO3E|=?t*ac{LdpB*j8${LU43yzOKMJsk&kzYD<+IOi+pI6dUZis2}80b6~XVX
zOBj$``6i7k&9LAEd8r=R{UmP_wlWF|iF-`jN8wuNMe40j&Pt@B)pNe|o`1uMi|4Vf
zc1T+S<oRZ_bf&8>8a?|eWJpgNacws@cWJ%7`X+q;{c1bHVWnxbcMU;Jj!@PBslkK`
zrpRkPFHu8|5I!2t_QtB9eJdbj1NJWai;%OTC4R&J`!04R6E4s%vEHP`PiKAQbk^FR
z!*cIKai;C&S$w#D6?>h;<eygE^>!U@v>w};c$Lioyzrhu!cAq>)t5UKOLM8Gq<MM!
zDUZ&xk6=lCa58GoPhE~H2`vieG(V0isR&GFeP24oqz30l<>CYQs4{@>5VrRemFGM#
z!8%x~=Aa^-My6?qX549}ADeHwufyH<*Xz}$TrEMMb=kL)+c(I91E;AbuGS4BS+qms
z<uAMa4EdPw<kqBT5^JN2qJu8uGjVqbs6>j%B!!`CWBJxZ%`a?UeEc50MEu@#LgtLO
zCcklR$#cr)m+8l}UN_%%*Kg7~?M8N)j84p@2?bJimeqiD;5;;sq^H$DAO{gVx+k2r
zi#B+ZqKK}yg0!{5Var%eiO49lu^ovonZMCoeEuF)=I@oIwDx=x*~)?JY0%VjGTQIL
z!<rw9FT!rOx+&vQkrB!=GNs0gdvAQrK-Lb;qm~0pomi8=;9vPT71MZr`a*9+uRRH(
z$b>th6~5Q5(^dhc(#`a6t1~Ys7td%b1Ne#(gM+sAcX6EwMfb|0<+Gj9#^BSmetR8A
zk4<E~<QL|2V{OgBNoJ+`C{>?qG38za&-~Rt(fk+;;7v>dJwuVC*1|hNvA_R_kT6jR
zkC0u?<^?24xpcc9qB^^uqRsQ9<mW%zgX`m6{+rlbefxH`c^yVhz&y8PlKWq)DThZt
z|K4Ojkv)ul04~)Db@DL!k!G;$dN=wB$;iy89T5oXjTE5^iODg6h7_burJL#I@LbLd
z%f*w|+U&h&#R$zf_))ckA9JF!h@RE5@|%a<ZB`WgadmT@c4cTP0AZCD#@TGe#Sdd;
z%{0RTWV&X))5&rh`Z%%#lIyauh}H&kG6D(j+ay$Id&VePlXL-_FuxlYgLCP=Kcwq?
ze@b=+=Zr>BXDXU%)~flJ@Nm1`H2?2z9q#%vG7S>*Yu}L0`x2_!VjhyA_s*5-mv1!p
z2b~8$E~F>s>mgcj3>;yk?gar!;S6cfgYH6ywntZ5m)J(lUo>IyTvpd{+47Ahoi7&i
zx3)5P4lMr|+SPU)%*Y(}Lx=j+h1Xsfay3q}X0IyP%XjqTJ?7D+eJ|E~R3~L<okS9f
z@~lVZklKh+w5pHLGD?gtn;Fa=?Zsbv@g&w%lGw7f$oX!*s_!x37_v7>W96(dLZ7d<
zyH!@;mG8V<HZ13_s7#Kit?^yYjMN<$k&}i$dJxd&@PtRX?$MeywaJK12Fvu7lyI1O
z5vcF^|FkZ4rR->!KRlXYx%e4uW#V2j58>d^btVa3Ehta)7%`(3BhS+m+}x&J^WF6-
zt^Ig_v;+U@!eyH~n-9Gt)%jN*W;x29WHh4o3lgP=xS@AOBi{DC)JACA`#kt@BFMZY
z@vT;gAauTL$2C7V)#5p<skC1&Iz69}!(uguZTxBQ)Qz1Ho*(Z*Tt6I)p4EuM2<22e
zDFrHPFYdjQP~!r0G^f;bNmEnV?B%tuvpNfe-Up{K5WVCV6_ql~Xj5b|+wx;+OjKTF
z)92#$l3oCuizl>|>3a%`;`QH6IKrvBk{P{a|F+`ib#v?Gn9PUBG9J01(^zp3{cf^K
zZV=Q>sACTy@QmEYQi9Mc4Mqv047ZmCyl5jyL9uf#>D1{m813A#UJS;?)7aXmy%HD?
zZ~V_#T|NhnpM^|f^@!6mGe}LsyPO^bHA@i`>X-zClfLWj7#k#q?dH3MCHBgNBt_!d
zm`h`(tTKtb)(?q^t8`F8qn=v|OLFlcdsG>+SE8-<cjUZvYLNSklh<$3>+2QikyVIs
zjY4W9nF^z=JV@PR3+0p%!F~&(_BB-D6Wy!Sk)N}k+eT~i)HNywqILXCxWMJnIYni+
z^%ofLEIw9`YGd`v9psembv`Wq)^)Mq;xmoK?Pk>lH?jmV=o?SEaTaU9sE;Cae?piZ
z9_6n3NnCqu0e!HYuwJ%;d@>>=)eGDAsFHFqy5eEw{H{F=#>JCVS4&ch{sZS5=TwK!
z?jti!!=s<m(D@<qXZX`9ZMHv--ETQAZ(cEdG+JNWgMRjUl74b&)_6iLG!2th4NnZU
zGAVW1f{560p^RK8Lr9jfiQGX()silGgUo;OOp=yv*h9X~u%~>yb;kRU878f#4q8VB
zI?n;-N9%OY=t?^^;-p31HOvkhiB&D6889Z@v<pFIu4>S{KK58w#&s6EC=w(sNcTYm
zk5WY0nV?#mf+{U08=8NhLreGSAz$g$Q`$?O^O_Q>OQku{MawpRMjNfq+BRH$89IJk
zM-sdoG-y?09CD#Vgt|!RK{4PjY4Vg(4Jd{zUVk5$I)U+BXL6EW5M-SfE-+51?|ZF%
zm)gE0<e~Y4F1deQI*Zk{J_MywIqBW}Zxj&(S!8|^GGW%ct#8u$ht-B?Jds4{R~Ony
zhLwkldo97&U=xHq<Xx&0&YWTJAkasJ>`#VTtvAVN(Pw)QQUOE&pFJ&8OdfoG5{n(C
zpI`L4cn<<y=|N!i&!MT*ltsvl&OI;TL7<PiyT^KToK<A^=bInHI&QB%UNg?ZFGExj
zaoM9eqTAhYEuyE|%<Mzp2W_5w1kk?2ke@jbTGI+@xs0}*C6QE4^wPd9UyJAl=I=8X
z??a#~eF%yppfd&^_*#6P80e+v%nka6-}66VlgubAjW@3@T)skfu5MK+VTg#<lYOQ>
zs(9#0ZhJx<z{Mn-j!sZ;t#GnU{A^$}(I`r)gh(~NtrjD4>GxZ@I=8)~OFyTrmaIdh
zYSf8zcHWA~s0`V-=kD!WexA$YA8&1A(KZrtS~Q`patP7K%V#_xj~0(3BYZNJCGI9|
zV8Yu)BtcFo!H;DtFsN2qn=C(xWpvJMq)xDSM*{r<js)j)2>Cr8aecmB$I$%wtL<h~
zW|z_7Xz!*~)nE=XyAy^#(hp>p<ddnaGo1|5aknZ>4mOA=S^_Vk!Zvx;MsLdvrkh>a
z1B<sJ(3MsMt}y#KqvRDUYo$50R&vdnvsUgcdsbX7!=~Tv;uw$RN3c;cy9IIWx(6~f
ze(JPO62W{fq9^^-?Q!D?Pb^}&d5mBKx2h^vOY$9em(p;x1c%8u7i6hg+QG<SUUV*=
z%T{_3*uu&4AqVqJ<DW{SkW+V<^TyJCyyI_k6Rtkqe!sd|b=Tsr{JGM|wIyO>jBoPX
z^Wt8bsw-%}?>6_Tw9}4v{&;SAv=;fvEhO?&XIs%xPL7Uk5Urw|N*fY(*tTFxdOY<4
zIs;gIq`R(+bXU4y7jnk=Pql~hY4{wy8hKWuxw;8yea!3{i3Kv!zPhm932l9yEbq`Z
z5)RiO6FsIHwx0^$$7Dn$xyf{l#OzFft;$(?QS_lF?|a*WEMszhX9E7;#bXjH!}NX*
z$>}?;8G|l=a|fU2NPoUs?Or#ZY&W~@L%CFX3k^DKkb65WmCyIQJ6Q6VJ(o2n<j=x!
zF6*9T1KHO17`e3!UF1q>WG`GaQD~23b?RHMN{-F8Q7?eXrAGtl%KUb~QrrQZu~b%q
za__=_QnmEMk<v`o1gn39tH1D9Why%|m36vNL=DNB-1c1k_I?E5ru+yF2tK&U%1e4=
zL@6N=DAfz>jTYov$qi<Q+-h=r34PGUg#sw?1#X^U@wx4~Hn+XxM0r5!@8q-#PHi#e
zoD}w>-FoXslYtC^aMlQm*@*zElG<_TDG^S{f~MgHMP*ZwN1$vw)AtrdK=Oh#KZuSD
zlj|70k`heoOVah1I4gn0C$;O!q;`d+B?+C+!UwAFN=|Gme@*nrBqqOG-QL|ae@l07
zNR4K8X(==^OpNsx_l5xL5-tx$cG{lc+-!$sXSOAHv@fA#x~MScZALDWk*llYUmPOz
zKDGcMXe_Z<dI7x;EIz1RR|d6%E@2bSck)#mac~kaXU&2BB<yaso92^Udb8S$ZAw6(
z0b#jl8lx&Sc<}C6gh}kARR|3A@@JZY9AiM%ph(K}f-JX}t>X5wl|fpE5=~ojhJ}~F
z<l@8Gb!9j^6l0+E`363&mcqIdIqd#ib4J0>{nZ!Y4gXNYm)q;j$a#s=zCo`VE|wJ;
zU2q)YfS<PT$+c1scCzx(9M-n6SGw;+fM~e6^j^5GYekC8X)Uy<i>4tj!ebX7&#r6Z
z*|F$Va6aFV>N%`xP9w5@K2ZLNL3s6TcfDd<u8kd4$a+n$E_6OtmQ_1TH>3iQ9a_d;
zKKPQyWePrWQm5W@lAslR-#S5vL4*#y(5)lp+?sI2w%{e|<>EPPWk@@yl9T#56|%k#
zmek18R6mM_9JrZ`n?t)>r8SP+MjPU4L<|!vE5@+z%<j+)lht2V5gcqPL0{sdO4)^0
zI@k6hMwbM&K?{#<5;pj@Gs>E_s3M@o&yPsAbQPY#N)=vVHF8ep$=|J&`e(Pdt8~SG
zNw3prrL+=XU07v=v*5D|&-sd>56I(5N#1=-!SiUkrRj+b&v#MVo|!9k*-NDNQnbE}
z!X^o}^&pchi)0T1-+AG=^nz)yu3(NtkDSkpvU2EiY7X>_VY8p`-^*^b3C-WP>ybPl
zWlV$CHD(V>MF2HbQx4-*dE)J*+NWEZCuZu-%Fdw6Is(kbur>;>RT5~l3#pI%noDNt
zFVbK7r7tfHRch}_8PMlECIL9po`mMylYF$!f`Ts|!tJdZF>d})YoBW-a4VN%^fc<4
zJ1=vb{n!)Se#{Zb_#}DOV|2ZYkv~~xw3HdGmnfWeq*L#r2_?6mm$<?Nzx3IUp}N^m
z(NyG|5tr4_JgsPgn)A}@v-RpGG+&1{#MLOFMX4GKE}O^ZoCB*X#<F>Sq>p|-lVe$P
z6bRuHzn@-Xw!m`2M6Ru5xz7yzTng3M1g7)Y>vAZxpWTT|$)$UW7^?FWaivh`GdknJ
zS=cPhOJ{d!yAHY0ydJ5T9__0OM<$C;;o_dEL_cmQm_&mP{g)lq4+?iwPvo&dnUIX_
uh3<3gY-+6tu64pCBQ^FEA(#RxnQYMfs9gFI)9C#Y^Zx@lnC<CPX8{25O>@Qo

literal 0
HcmV?d00001

diff --git a/tests/test_azure.py b/tests/test_azure.py
index b3f17fa..9395b1a 100644
--- a/tests/test_azure.py
+++ b/tests/test_azure.py
@@ -16,7 +16,9 @@
 from object_store_connector.connector import ObjectStoreConnector
 
 
-class TestSource(ISourceConnector):
+
+
+class ObjectStoreConnector(ISourceConnector):
     def process(
         self,
         sc: SparkSession,
@@ -25,6 +27,7 @@ def process(
         metrics_collector: MetricsCollector,
     ) -> DataFrame:
         df = sc.read.format("json").load("tests/sample_data/nyt_data_100.json.gz")
+        # print(df.show())
         yield df
 
         df1 = sc.read.format("json").load("tests/sample_data/nyt_data_100.json")
@@ -55,25 +58,26 @@ def test_source_connector(self):
 
         config = yaml.safe_load(open(config_file_path))
 
-        config['connector_instance_id'] = "azure.new-york-taxi-data.1"
+        # config['connector_instance_id'] = "azure.new-york-taxi-data.1"
 
-        with open(
-            config_file_path, "w"
-        ) as config_file:
-            yaml.dump(config, config_file)
+        # with open(
+        #     config_file_path, "w"
+        # ) as config_file:
+        #     yaml.dump(config, config_file)
 
 
         self.assertEqual(os.path.exists(config_file_path), True)
 
         test_raw_topic = "azure.ingest"
         test_metrics_topic = "azure.metrics"
-
+        # import time
+        # time.sleep(100)
         kafka_consumer = KafkaConsumer(
             bootstrap_servers=config["kafka"]["broker-servers"],
-            group_id="test-group",
+            group_id="azure-test-group",
             enable_auto_commit=True,
         )
-
+        # time.sleep(100)
         trt_consumer = TopicPartition(test_raw_topic, 0)
         tmt_consumer = TopicPartition(test_metrics_topic, 0)
 
@@ -85,16 +89,20 @@ def test_source_connector(self):
 
         print("Done processing....0")
 
-        import time
-        time.sleep(300)
-
-        # metrics = []
-        # all_messages = kafka_consumer.poll(timeout_ms=10000)
+        # import time
+        # time.sleep(30)
+        all_messages = kafka_consumer.poll(timeout_ms=10000)
 
+        metrics = []
         # for topic_partition, messages in all_messages.items():
         #     for message in messages:
         #         if topic_partition.topic == test_metrics_topic:
         #             metrics.append(message.value)
 
-        # assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
-        # assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 1}
+        # Check number of messages
+        print(f"Number of messages in {test_metrics_topic}: {len(metrics)}")
+        for message in metrics:
+            print(message)
+        print("kkk",kafka_consumer.end_offsets([trt_consumer]))
+        assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
+        assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 1}
\ No newline at end of file

From cac0c0c82fc0776c908ac78ce84ba58337ab25aa Mon Sep 17 00:00:00 2001
From: Manaswini <143373040+saimanaswini-k@users.noreply.github.com>
Date: Mon, 27 May 2024 18:24:43 +0530
Subject: [PATCH 46/59] obsrv-labs/issue-tracker#2: Updated test case

---
 tests/test_s3.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tests/test_s3.py b/tests/test_s3.py
index db91c39..80d00b7 100644
--- a/tests/test_s3.py
+++ b/tests/test_s3.py
@@ -56,7 +56,7 @@ def test_source_connector(self):
 
         kafka_consumer = KafkaConsumer(
             bootstrap_servers=config["kafka"]["broker-servers"],
-            group_id="test-group",
+            group_id="s3-group",
             enable_auto_commit=True,
         )
 
@@ -76,10 +76,5 @@ def test_source_connector(self):
         #        if topic_partition.topic == test_metrics_topic:
         #            metrics.append(message.value)
 
-        # Check number of messages
-        print(f"Number of messages in {test_metrics_topic}: {len(metrics)}")
-        for message in metrics:
-            print(message)
-
         assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
         assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 1}

From cc0c85db7fed43566cde99405a5902e4f853bc76 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Mon, 27 May 2024 18:29:36 +0530
Subject: [PATCH 47/59] obsrv-labs/issue-tracker#6: cleanup

---
 tests/test_azure.py | 26 ++++----------------------
 1 file changed, 4 insertions(+), 22 deletions(-)

diff --git a/tests/test_azure.py b/tests/test_azure.py
index 9395b1a..097769a 100644
--- a/tests/test_azure.py
+++ b/tests/test_azure.py
@@ -46,38 +46,21 @@ def test_source_connector(self):
 
         connector = ObjectStoreConnector()
         config_file_path = os.path.join(os.path.dirname(__file__), "config/config.yaml")
-        # SourceConnector.process(connector=connector, config_file_path=config_file_path)
-         ## import the objectstorreconnector class
-
-        ## update the config file with your instances
-
-        ## call SourceConnector.process* - get the logic from __main__py.
-
-        # connector = TestSource()
-        # config_file_path = os.path.join(os.path.dirname(__file__), "config/config.yaml")
 
         config = yaml.safe_load(open(config_file_path))
 
-        # config['connector_instance_id'] = "azure.new-york-taxi-data.1"
-
-        # with open(
-        #     config_file_path, "w"
-        # ) as config_file:
-        #     yaml.dump(config, config_file)
-
 
         self.assertEqual(os.path.exists(config_file_path), True)
 
         test_raw_topic = "azure.ingest"
         test_metrics_topic = "azure.metrics"
-        # import time
-        # time.sleep(100)
+        
         kafka_consumer = KafkaConsumer(
             bootstrap_servers=config["kafka"]["broker-servers"],
             group_id="azure-test-group",
             enable_auto_commit=True,
         )
-        # time.sleep(100)
+        
         trt_consumer = TopicPartition(test_raw_topic, 0)
         tmt_consumer = TopicPartition(test_metrics_topic, 0)
 
@@ -89,8 +72,7 @@ def test_source_connector(self):
 
         print("Done processing....0")
 
-        # import time
-        # time.sleep(30)
+        
         all_messages = kafka_consumer.poll(timeout_ms=10000)
 
         metrics = []
@@ -103,6 +85,6 @@ def test_source_connector(self):
         print(f"Number of messages in {test_metrics_topic}: {len(metrics)}")
         for message in metrics:
             print(message)
-        print("kkk",kafka_consumer.end_offsets([trt_consumer]))
+        
         assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
         assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 1}
\ No newline at end of file

From d8c45ceb742759cc3ed18fb923cc8c8902c9967c Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Mon, 27 May 2024 18:38:59 +0530
Subject: [PATCH 48/59] obsrv-labs/issue-tracker#6:  fix: removed the
 with_bind_ports()

---
 tests/create_tables.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/create_tables.py b/tests/create_tables.py
index 83bd31e..dcbebc6 100644
--- a/tests/create_tables.py
+++ b/tests/create_tables.py
@@ -10,11 +10,10 @@
 
 def init_azurite():
     ## Init azurite
-    azurite= AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:latest").with_bind_ports(10000, 10000)
+    azurite= AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:latest")
     azurite.start()
 
     blob_port= azurite.get_exposed_port(10000)
-    # blob_port=10000
     conn_str=azurite.get_connection_string()
     acc_name=azurite.account_name
     acc_key=azurite.account_key

From bf86a6fdd6f5cd99a4c050e148e7f7c88c7e001c Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Mon, 27 May 2024 18:54:24 +0530
Subject: [PATCH 49/59] obsrv-labs/issue-tracker#6: cleanup

---
 tests/test_azure.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/tests/test_azure.py b/tests/test_azure.py
index 097769a..0d4fd58 100644
--- a/tests/test_azure.py
+++ b/tests/test_azure.py
@@ -70,21 +70,14 @@ def test_source_connector(self):
 
         SourceConnector.process(connector=connector, config_file_path=config_file_path)
 
-        print("Done processing....0")
-
         
-        all_messages = kafka_consumer.poll(timeout_ms=10000)
+        # all_messages = kafka_consumer.poll(timeout_ms=10000)
 
-        metrics = []
+        # metrics = []
         # for topic_partition, messages in all_messages.items():
         #     for message in messages:
         #         if topic_partition.topic == test_metrics_topic:
         #             metrics.append(message.value)
-
-        # Check number of messages
-        print(f"Number of messages in {test_metrics_topic}: {len(metrics)}")
-        for message in metrics:
-            print(message)
         
         assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
         assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 1}
\ No newline at end of file

From 07901d5b27a78d2aad8e2b6a9211add9f3b02fdb Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Tue, 28 May 2024 16:35:48 +0530
Subject: [PATCH 50/59] obsrv-labs/issue-tracker#6: feat: added the condition
 to check if the endpoint is from the local or from the azure account

---
 object_store_connector/provider/azure.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index a821bed..1e38855 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -22,10 +22,12 @@ def __init__(self, connector_config: str)-> None:
         self.account_key = connector_config["source"]["credentials"]["account_key"]
         self.container_name = connector_config["source"]["containername"]
         self.blob_endpoint = connector_config["source"]["blob_endpoint"]
-        self.prefix = (connector_config["source"]["prefix"]
-                       if "prefix" in connector_config["source"]
-                       else "/")
-        # self.obj_prefix= f"wasb//:{self.container_name}@"
+        self.prefix = (
+            connector_config["source"]["prefix"]
+            if "prefix" in connector_config["source"]
+            else "/"
+            )
+        
         
 
         self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
@@ -48,12 +50,15 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
         objects_info=[]
         if objects==None:
             raise Exception("No objects found")
-    
+        
         for obj in objects:
-            blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
-            
+            if self.blob_endpoint==(f"http://127.0.0.1:10000/{self.account_name}"):
+                blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
+                
+            else:
+                blob_location=f"https://{self.account_name}.blob.core.windows.net/{self.container_name/{obj['name']}}"
+                
             object_info = ObjectInfo(
-                    # location=f"{self.obj_prefix}{obj['name']}",
                     location=blob_location,
                     format=obj["name"].split(".")[-1],
                     file_size_kb=obj["size"] // 1024,

From 251e9dab2f75eeeabcdf1fcbab65a7970a578b4c Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Sun, 2 Jun 2024 09:02:01 +0530
Subject: [PATCH 51/59] obsrv-labs/issue-tracker#6: Cleanup

---
 object_store_connector/provider/azure.py |  5 +-
 tests/create_tables.py                   | 17 +++----
 tests/test_azure.py                      | 62 +++++++++++++++++++-----
 3 files changed, 61 insertions(+), 23 deletions(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 1e38855..26e475d 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -52,7 +52,9 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
             raise Exception("No objects found")
         
         for obj in objects:
-            if self.blob_endpoint==(f"http://127.0.0.1:10000/{self.account_name}"):
+            print("self.blob_endpoint",self.blob_endpoint)
+            host_ip=self.blob_endpoint.split("//")[-1].split("/")[0]
+            if self.blob_endpoint==(f"http://{host_ip}/{self.account_name}"):
                 blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
                 
             else:
@@ -66,6 +68,7 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
                     tags= self.fetch_tags(obj['name'], metrics_collector)
                 )
             objects_info.append(object_info.to_json())
+        
         return objects_info
 
     
diff --git a/tests/create_tables.py b/tests/create_tables.py
index dcbebc6..9238d39 100644
--- a/tests/create_tables.py
+++ b/tests/create_tables.py
@@ -10,7 +10,7 @@
 
 def init_azurite():
     ## Init azurite
-    azurite= AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:latest")
+    azurite= AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:latest").with_bind_ports(10000,10000)
     azurite.start()
 
     blob_port= azurite.get_exposed_port(10000)
@@ -26,19 +26,18 @@ def init_azurite():
 
     blob_service_client=BlobServiceClient.from_connection_string(conn_str)
     
-
+    print("Blob sr url", azure_conf)
     
     container_name="test-container"
 
     blob_service_client.create_container(container_name)
-    
-    bloblist = ["/home/vince/repogit/object-store-connector/tests/sample_data/nyt_data_100.json","/home/vince/repogit/object-store-connector/tests/sample_data/nyt_data_100.json.gz"]
-    for blob in bloblist:
-        blob_name=blob.split("/")[-1]
-        blob_client = blob_service_client.get_blob_client(container_name,blob=blob_name)
-        with open(blob, "rb") as data:
+     
+    folder_path="/home/vince/repogit/object-store-connector/tests/sample_data"
+    for filename in os.listdir(folder_path):    
+        filepath=os.path.join(folder_path,filename)
+        blob_client = blob_service_client.get_blob_client(container_name,blob=filename)
+        with open(filepath, "rb") as data:
             blob_client.upload_blob(data)
-    
     return azure_conf
 
 
diff --git a/tests/test_azure.py b/tests/test_azure.py
index 0d4fd58..524abb0 100644
--- a/tests/test_azure.py
+++ b/tests/test_azure.py
@@ -15,10 +15,10 @@
 
 from object_store_connector.connector import ObjectStoreConnector
 
+import json
 
 
-
-class ObjectStoreConnector(ISourceConnector):
+class TestSource(ISourceConnector):
     def process(
         self,
         sc: SparkSession,
@@ -27,7 +27,7 @@ def process(
         metrics_collector: MetricsCollector,
     ) -> DataFrame:
         df = sc.read.format("json").load("tests/sample_data/nyt_data_100.json.gz")
-        # print(df.show())
+        print(df.show())
         yield df
 
         df1 = sc.read.format("json").load("tests/sample_data/nyt_data_100.json")
@@ -66,18 +66,54 @@ def test_source_connector(self):
 
         kafka_consumer.assign([trt_consumer, tmt_consumer])
 
-        # kafka_consumer.seek_to_beginning()
+        kafka_consumer.seek_to_beginning()
 
         SourceConnector.process(connector=connector, config_file_path=config_file_path)
 
         
-        # all_messages = kafka_consumer.poll(timeout_ms=10000)
-
-        # metrics = []
-        # for topic_partition, messages in all_messages.items():
-        #     for message in messages:
-        #         if topic_partition.topic == test_metrics_topic:
-        #             metrics.append(message.value)
-        
+        all_messages = kafka_consumer.poll(timeout_ms=10000)
+
+        metrics = []
+        metric_names={
+                'num_api_calls': 0, 
+                'new_objects_discovered': 0
+            }
+        # num_api_calls=[]
+        for topic_partition, messages in all_messages.items():
+            for message in messages:
+                if topic_partition.topic == test_metrics_topic:
+                    # print("message of kafka",message.value)
+                    # message_val=json.loads(message.value.decode())
+                    # metric=message_val['edata']['metric']
+                    # print("metric of edata",metric)
+                    # if metric:
+                    #     print("num_api_calls",metric['num_api_calls'])
+                    # else:
+                    #     continue
+                    # metrics.append(message.value)
+                    msg_val=json.loads(message.value.decode())
+                    metrics.append(msg_val)
+                # print("msg_val",msg_val)
+        # print("metrics",metrics)
+        for metric in metrics:
+            metric_data = metric.get('edata', {}).get('metric', {})  # Get the 'metric' data from the current metric
+            for mn, val in metric_names.items():  # Iterate over metric_names dictionary
+                if mn in metric_data:  # Check if the current metric has the key 'mn'
+                    metric_names[mn] += metric_data[mn]
+        print(metric_names)
+            # if data['edata']['metric']['num_api_calls']:
+            #     print("num_call",data['edata']['metric']['num_api_calls']) 
+            # elif data['edata']['metric']['new_objects_discovered']:
+            #     print("new_obj",data['edata']['metric']['new_objects_discovered']) 
+            # else:
+            #     continue
+        count=0
         assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
-        assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 1}
\ No newline at end of file
+        count+=1
+        print("count",count)
+        assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 9}
+        count+=1
+        print("count",count)
+        # assert kafka_consumer.metrics([tmt_consumer])=={tmt_consumer:76}
+        # print("sddd",kafka_consumer.metrics([tmt_consumer]))
+        
\ No newline at end of file

From 78e50848a97ab350f00bfc70cab051f89e17dc59 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Sun, 2 Jun 2024 21:56:45 +0530
Subject: [PATCH 52/59] obsrv-labs/issue-tracker#6 feat:  added the assert for
 num_api_calls, new_objects_discovered and total_records_count

---
 tests/test_azure.py | 31 +++++++++----------------------
 1 file changed, 9 insertions(+), 22 deletions(-)

diff --git a/tests/test_azure.py b/tests/test_azure.py
index 524abb0..4aa3463 100644
--- a/tests/test_azure.py
+++ b/tests/test_azure.py
@@ -76,37 +76,24 @@ def test_source_connector(self):
         metrics = []
         metric_names={
                 'num_api_calls': 0, 
-                'new_objects_discovered': 0
+                'new_objects_discovered': 0,
+                'total_records_count':0
             }
         # num_api_calls=[]
         for topic_partition, messages in all_messages.items():
             for message in messages:
                 if topic_partition.topic == test_metrics_topic:
-                    # print("message of kafka",message.value)
-                    # message_val=json.loads(message.value.decode())
-                    # metric=message_val['edata']['metric']
-                    # print("metric of edata",metric)
-                    # if metric:
-                    #     print("num_api_calls",metric['num_api_calls'])
-                    # else:
-                    #     continue
-                    # metrics.append(message.value)
                     msg_val=json.loads(message.value.decode())
                     metrics.append(msg_val)
                 # print("msg_val",msg_val)
         # print("metrics",metrics)
         for metric in metrics:
-            metric_data = metric.get('edata', {}).get('metric', {})  # Get the 'metric' data from the current metric
-            for mn, val in metric_names.items():  # Iterate over metric_names dictionary
-                if mn in metric_data:  # Check if the current metric has the key 'mn'
+            metric_data = metric['edata']['metric']  
+            for mn, val in metric_names.items():  
+                if mn in metric_data:  
                     metric_names[mn] += metric_data[mn]
         print(metric_names)
-            # if data['edata']['metric']['num_api_calls']:
-            #     print("num_call",data['edata']['metric']['num_api_calls']) 
-            # elif data['edata']['metric']['new_objects_discovered']:
-            #     print("new_obj",data['edata']['metric']['new_objects_discovered']) 
-            # else:
-            #     continue
+           
         count=0
         assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
         count+=1
@@ -114,6 +101,6 @@ def test_source_connector(self):
         assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 9}
         count+=1
         print("count",count)
-        # assert kafka_consumer.metrics([tmt_consumer])=={tmt_consumer:76}
-        # print("sddd",kafka_consumer.metrics([tmt_consumer]))
-        
\ No newline at end of file
+        assert metric_names['num_api_calls'] == 5
+        assert metric_names['new_objects_discovered'] == 2
+        assert metric_names['total_records_count'] == 200
\ No newline at end of file

From f071daa8d9f51f21ce26ddfcfe276b8cca672c9d Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Mon, 3 Jun 2024 17:02:28 +0530
Subject: [PATCH 53/59] obsrv-labs/issue-tracker#6 cleanup

---
 object_store_connector/provider/azure.py |  5 ++---
 tests/create_tables.py                   |  3 +--
 tests/test_azure.py                      | 28 ++++++++++++------------
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 26e475d..fdf07ca 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -52,7 +52,6 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
             raise Exception("No objects found")
         
         for obj in objects:
-            print("self.blob_endpoint",self.blob_endpoint)
             host_ip=self.blob_endpoint.split("//")[-1].split("/")[0]
             if self.blob_endpoint==(f"http://{host_ip}/{self.account_name}"):
                 blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
@@ -88,7 +87,7 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
             elif file_format == "json":
                 df = sc.read.format("json").option("multiLine", False).load(object_path)
             elif file_format == "csv":
-                df = sc.read.format("csv").option("header", True).load(object_path)
+                df = sc.read.format("csv").option("header", True,mode = "PERMISSIVE", columnNameOfCorruptRecord = "_corrupt_record").load(object_path)
             elif file_format == "parquet":
                 df = sc.read.parquet(object_path)
             
@@ -241,7 +240,7 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
             existing_tags.update(new_dict)
             
             blob_client.set_blob_tags(existing_tags)
-            
+            api_calls += 1
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
             
             return True
diff --git a/tests/create_tables.py b/tests/create_tables.py
index 9238d39..a3772df 100644
--- a/tests/create_tables.py
+++ b/tests/create_tables.py
@@ -26,7 +26,7 @@ def init_azurite():
 
     blob_service_client=BlobServiceClient.from_connection_string(conn_str)
     
-    print("Blob sr url", azure_conf)
+    
     
     container_name="test-container"
 
@@ -172,7 +172,6 @@ def create_tables(config):
         cur.execute(ins_ds)
         cur.execute(ins_cr)
         cur.execute(ins_ci, (json.dumps(enc_config),))
-        # cur.execute("select * from connector_instances")
 
         conn.commit()
         conn.close()
diff --git a/tests/test_azure.py b/tests/test_azure.py
index 4aa3463..fec57fc 100644
--- a/tests/test_azure.py
+++ b/tests/test_azure.py
@@ -27,7 +27,7 @@ def process(
         metrics_collector: MetricsCollector,
     ) -> DataFrame:
         df = sc.read.format("json").load("tests/sample_data/nyt_data_100.json.gz")
-        print(df.show())
+    
         yield df
 
         df1 = sc.read.format("json").load("tests/sample_data/nyt_data_100.json")
@@ -77,30 +77,30 @@ def test_source_connector(self):
         metric_names={
                 'num_api_calls': 0, 
                 'new_objects_discovered': 0,
-                'total_records_count':0
+                'total_records_count':0,
+                'num_records':0,
+                'failed_records_count':0
             }
         # num_api_calls=[]
         for topic_partition, messages in all_messages.items():
             for message in messages:
-                if topic_partition.topic == test_metrics_topic:
+                if topic_partition.topic == test_raw_topic:
                     msg_val=json.loads(message.value.decode())
                     metrics.append(msg_val)
-                # print("msg_val",msg_val)
-        # print("metrics",metrics)
+                    
         for metric in metrics:
             metric_data = metric['edata']['metric']  
             for mn, val in metric_names.items():  
                 if mn in metric_data:  
                     metric_names[mn] += metric_data[mn]
-        print(metric_names)
-           
-        count=0
+      
         assert kafka_consumer.end_offsets([trt_consumer]) == {trt_consumer: 200}
-        count+=1
-        print("count",count)
+       
         assert kafka_consumer.end_offsets([tmt_consumer]) == {tmt_consumer: 9}
-        count+=1
-        print("count",count)
-        assert metric_names['num_api_calls'] == 5
+        
+        
+        assert metric_names['num_api_calls'] == 7
         assert metric_names['new_objects_discovered'] == 2
-        assert metric_names['total_records_count'] == 200
\ No newline at end of file
+        assert metric_names['total_records_count'] == 200
+        assert metric_names['num_records'] == 200
+        assert metric_names['failed_records_count'] == 0
\ No newline at end of file

From 9b1e642c5e861534f44ee6e5b594c36360e6a515 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Mon, 3 Jun 2024 17:21:54 +0530
Subject: [PATCH 54/59] obsrv-labs/issue-tracker#6: cleanup

---
 object_store_connector/provider/azure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index fdf07ca..e420068 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -87,7 +87,7 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
             elif file_format == "json":
                 df = sc.read.format("json").option("multiLine", False).load(object_path)
             elif file_format == "csv":
-                df = sc.read.format("csv").option("header", True,mode = "PERMISSIVE", columnNameOfCorruptRecord = "_corrupt_record").load(object_path)
+                df = sc.read.format("csv").option("header", True).load(object_path)
             elif file_format == "parquet":
                 df = sc.read.parquet(object_path)
             

From 0231afed645d215880c7750bb6651a3bf9a6c5c1 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Mon, 3 Jun 2024 17:55:13 +0530
Subject: [PATCH 55/59] obsrv-labs/issue-tracker#6: cleanup

---
 object_store_connector/provider/azure.py | 9 +++++++--
 tests/test_azure.py                      | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index e420068..25689f8 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -229,8 +229,13 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
         try:
             
             new_dict = {tag['key']: tag['value'] for tag in tags}
-        
-            stripped_file_path = object.get("location").lstrip("wasb://")
+
+            location = object.get("location")
+            if location.startswith("wasb://"):
+                stripped_file_path = location.lstrip("wasb://")
+            elif location.startswith("https://"):
+                stripped_file_path = location.lstrip("https://")
+            
             obj = stripped_file_path.split("/")[-1]
  
             blob_client = BlobClient.from_connection_string(
diff --git a/tests/test_azure.py b/tests/test_azure.py
index fec57fc..d16bde6 100644
--- a/tests/test_azure.py
+++ b/tests/test_azure.py
@@ -84,7 +84,7 @@ def test_source_connector(self):
         # num_api_calls=[]
         for topic_partition, messages in all_messages.items():
             for message in messages:
-                if topic_partition.topic == test_raw_topic:
+                if topic_partition.topic == test_metrics_topic:
                     msg_val=json.loads(message.value.decode())
                     metrics.append(msg_val)
                     

From 710f18f81725d608609507199c052bb9fb56e5f7 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Tue, 4 Jun 2024 12:19:51 +0530
Subject: [PATCH 56/59] 	obsrv-labs/issue-tracker#6: cleanup

---
 object_store_connector/provider/azure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 25689f8..34d14ca 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -57,7 +57,7 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
                 blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
                 
             else:
-                blob_location=f"https://{self.account_name}.blob.core.windows.net/{self.container_name/{obj['name']}}"
+                blob_location=f"https://{self.account_name}.blob.core.windows.net/{self.container_name}/{obj['name']}"
                 
             object_info = ObjectInfo(
                     location=blob_location,

From e90e90310d531c69f120786de57ba5a439ea560b Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Tue, 4 Jun 2024 15:31:22 +0530
Subject: [PATCH 57/59] obsrv-labs/issue-tracker#6: cleanup

---
 object_store_connector/connector.py              | 8 ++++----
 object_store_connector/provider/azure.py         | 4 ++--
 object_store_connector/provider/blob_provider.py | 2 +-
 object_store_connector/provider/s3.py            | 4 ++--
 tests/create_tables.py                           | 2 +-
 tests/test_azure.py                              | 4 +++-
 6 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/object_store_connector/connector.py b/object_store_connector/connector.py
index 9c7e2fd..d552863 100644
--- a/object_store_connector/connector.py
+++ b/object_store_connector/connector.py
@@ -3,19 +3,19 @@
 import time
 from typing import Any, Dict, Iterator
 
-from object_store_connector.models.object_info import ObjectInfo
+from models.object_info import ObjectInfo
 from obsrv.common import ObsrvException
 from obsrv.connector import ConnectorContext, MetricsCollector
 from obsrv.connector.batch import ISourceConnector
 from obsrv.models import ErrorData, ExecutionState, StatusCode
 from obsrv.utils import LoggerController
-from object_store_connector.provider.s3 import S3
+from provider.s3 import S3
 from pyspark.conf import SparkConf
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import lit
 
-from object_store_connector.provider.azure import AzureBlobStorage
-from object_store_connector.models.object_info import ObjectInfo
+from provider.azure import AzureBlobStorage
+from models.object_info import ObjectInfo
 logger = LoggerController(__name__)
 
 MAX_RETRY_COUNT = 10
diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index 34d14ca..a3e4c3d 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -2,8 +2,8 @@
 from typing import Dict, List, Any
 from azure.storage.blob import ContainerClient, BlobClient
 from pyspark.sql import DataFrame, SparkSession
-from object_store_connector.provider.blob_provider import BlobProvider
-from object_store_connector.models.object_info import ObjectInfo,Tag
+from provider.blob_provider import BlobProvider
+from models.object_info import ObjectInfo,Tag
 from pyspark.conf import SparkConf
 from obsrv.job.batch import get_base_conf
 from obsrv.connector import ConnectorContext, MetricsCollector
diff --git a/object_store_connector/provider/blob_provider.py b/object_store_connector/provider/blob_provider.py
index 1ac96fd..c4e7a7e 100644
--- a/object_store_connector/provider/blob_provider.py
+++ b/object_store_connector/provider/blob_provider.py
@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List
 
-from object_store_connector.models.object_info import ObjectInfo, Tag
+from models.object_info import ObjectInfo, Tag
 from obsrv.connector import MetricsCollector
 from pyspark.sql import DataFrame, SparkSession
 
diff --git a/object_store_connector/provider/s3.py b/object_store_connector/provider/s3.py
index 41a48a1..5b7793c 100644
--- a/object_store_connector/provider/s3.py
+++ b/object_store_connector/provider/s3.py
@@ -3,12 +3,12 @@
 
 import boto3
 from botocore.exceptions import BotoCoreError, ClientError
-from object_store_connector.models.object_info import ObjectInfo, Tag
+from models.object_info import ObjectInfo, Tag
 from obsrv.common import ObsrvException
 from obsrv.connector import ConnectorContext, MetricsCollector
 from obsrv.job.batch import get_base_conf
 from obsrv.models import ErrorData
-from object_store_connector.provider.blob_provider import BlobProvider
+from provider.blob_provider import BlobProvider
 from pyspark.conf import SparkConf
 from pyspark.sql import DataFrame, SparkSession
 
diff --git a/tests/create_tables.py b/tests/create_tables.py
index a3772df..c5d5773 100644
--- a/tests/create_tables.py
+++ b/tests/create_tables.py
@@ -9,7 +9,7 @@
 from azure.storage.blob import *
 
 def init_azurite():
-    ## Init azurite
+    
     azurite= AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:latest").with_bind_ports(10000,10000)
     azurite.start()
 
diff --git a/tests/test_azure.py b/tests/test_azure.py
index d16bde6..8adf627 100644
--- a/tests/test_azure.py
+++ b/tests/test_azure.py
@@ -1,6 +1,8 @@
 import os
 import unittest
 from typing import Any, Dict
+import sys
+sys.path.insert(0, '/home/vince/repogit/object-store-connector/object_store_connector')
 
 import pytest
 import yaml
@@ -81,7 +83,7 @@ def test_source_connector(self):
                 'num_records':0,
                 'failed_records_count':0
             }
-        # num_api_calls=[]
+        
         for topic_partition, messages in all_messages.items():
             for message in messages:
                 if topic_partition.topic == test_metrics_topic:

From 6c0dbb949c6ef5e64353e1f5f9aa558f7e1b7d19 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Fri, 7 Jun 2024 16:42:39 +0530
Subject: [PATCH 58/59] obsrv-labs/issue-tracker#6:  feat: checked with azure
 cred and cleanup

---
 object_store_connector/provider/azure.py |  36 ++-
 poetry.lock                              | 336 +++++++----------------
 tests/create_tables.py                   |  82 ++----
 tests/test_s3.py                         |   7 +-
 4 files changed, 142 insertions(+), 319 deletions(-)

diff --git a/object_store_connector/provider/azure.py b/object_store_connector/provider/azure.py
index a3e4c3d..515f17d 100644
--- a/object_store_connector/provider/azure.py
+++ b/object_store_connector/provider/azure.py
@@ -28,10 +28,14 @@ def __init__(self, connector_config: str)-> None:
             else "/"
             )
         
-        
+        if self.blob_endpoint=="core.windows.net":
+            self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};EndpointSuffix={self.blob_endpoint}" 
+        else:
+            self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
 
-        self.connection_string = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};BlobEndpoint={self.blob_endpoint}" 
         self.container_client = ContainerClient.from_connection_string(self.connection_string,self.container_name)
+        
+
 
 
     def get_spark_config(self, connector_config) -> SparkConf:
@@ -39,6 +43,8 @@ def get_spark_config(self, connector_config) -> SparkConf:
         conf.setAppName("ObsrvObjectStoreConnector")
         conf.set("spark.jars.packages", "org.apache.hadoop:hadoop-azure:3.3.1")
         conf.set("fs.azure.storage.accountAuthType", "SharedKey")
+        conf.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem")
+        conf.set(f"fs.azure.account.key.{self.account_name}.blob.core.windows.net", self.account_key)
         conf.set("fs.azure.storage.accountKey", connector_config["source"]["credentials"]["account_key"])
         return conf
 
@@ -52,12 +58,12 @@ def fetch_objects(self,ctx: ConnectorContext, metrics_collector: MetricsCollecto
             raise Exception("No objects found")
         
         for obj in objects:
-            host_ip=self.blob_endpoint.split("//")[-1].split("/")[0]
-            if self.blob_endpoint==(f"http://{host_ip}/{self.account_name}"):
-                blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
+            
+            if self.blob_endpoint==("core.windows.net"):
+                blob_location=f"wasbs://{self.container_name}@{self.account_name}.blob.core.windows.net/{obj['name']}"
                 
             else:
-                blob_location=f"https://{self.account_name}.blob.core.windows.net/{self.container_name}/{obj['name']}"
+                blob_location = f"wasb://{self.container_name}@storageemulator/{obj['name']}"
                 
             object_info = ObjectInfo(
                     location=blob_location,
@@ -78,9 +84,9 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
             {"key": "object_path", "value": object_path}
         ]
         
-        
+    
         api_calls, errors, records_count = 0, 0, 0
-
+       
         try:
             if file_format == "jsonl":
                 df = sc.read.format("json").load(object_path)
@@ -124,7 +130,7 @@ def read_object(self, object_path: str, sc: SparkSession, metrics_collector: Met
 
     def _list_blobs_in_container(self,ctx: ConnectorContext, metrics_collector) -> list:
         self.container_name = self.connector_config['source']['containername']
-        
+
         summaries = []
         continuation_token = None
         file_formats = {
@@ -188,7 +194,6 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             {"key": "method_name", "value": "get_blob_tags"},
             {"key": "object_path", "value": object_path}
         ]
-
         api_calls, errors = 0, 0
         try:
             blob_client = BlobClient.from_connection_string(
@@ -198,8 +203,6 @@ def fetch_tags(self, object_path: str, metrics_collector: MetricsCollector) -> L
             
             api_calls += 1
             metrics_collector.collect("num_api_calls", api_calls, addn_labels=labels)
-            for k, v in tags.items():
-                print(k, v)
             
             return [Tag(key,value) for key,value in tags.items()]
 
@@ -225,18 +228,11 @@ def update_tag(self,object: ObjectInfo, tags: list, metrics_collector: MetricsCo
             {"key": "object_path", "value": object.get('location')}
         ]
         api_calls, errors = 0, 0
-
         try:
             
             new_dict = {tag['key']: tag['value'] for tag in tags}
-
             location = object.get("location")
-            if location.startswith("wasb://"):
-                stripped_file_path = location.lstrip("wasb://")
-            elif location.startswith("https://"):
-                stripped_file_path = location.lstrip("https://")
-            
-            obj = stripped_file_path.split("/")[-1]
+            obj = location.split("/")[-1]
  
             blob_client = BlobClient.from_connection_string(
                 conn_str=self.connection_string, container_name=self.container_name, blob_name=obj
diff --git a/poetry.lock b/poetry.lock
index a2153ea..19b6ace 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -41,17 +41,17 @@ aio = ["azure-core[aio] (>=1.28.0)"]
 
 [[package]]
 name = "boto3"
-version = "1.34.105"
+version = "1.34.118"
 description = "The AWS SDK for Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "boto3-1.34.105-py3-none-any.whl", hash = "sha256:b633e8fbf7145bdb995ce68a27d096bb89fd393185b0e773418d81cd78db5a03"},
-    {file = "boto3-1.34.105.tar.gz", hash = "sha256:f2c11635be0de7b7c06eb606ece1add125e02d6ed521592294a0a21af09af135"},
+    {file = "boto3-1.34.118-py3-none-any.whl", hash = "sha256:e9edaf979fbe59737e158f2f0f3f0861ff1d61233f18f6be8ebb483905f24587"},
+    {file = "boto3-1.34.118.tar.gz", hash = "sha256:4eb8019421cb664a6fcbbee6152aa95a28ce8bbc1c4ee263871c09cdd58bf8ee"},
 ]
 
 [package.dependencies]
-botocore = ">=1.34.105,<1.35.0"
+botocore = ">=1.34.118,<1.35.0"
 jmespath = ">=0.7.1,<2.0.0"
 s3transfer = ">=0.10.0,<0.11.0"
 
@@ -60,13 +60,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
 
 [[package]]
 name = "botocore"
-version = "1.34.105"
+version = "1.34.118"
 description = "Low-level, data-driven core of boto 3."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "botocore-1.34.105-py3-none-any.whl", hash = "sha256:a459d060b541beecb50681e6e8a39313cca981e146a59ba7c5229d62f631a016"},
-    {file = "botocore-1.34.105.tar.gz", hash = "sha256:727d5d3e800ac8b705fca6e19b6fefa1e728a81d62a712df9bd32ed0117c740b"},
+    {file = "botocore-1.34.118-py3-none-any.whl", hash = "sha256:e3f6c5636a4394768e81e33a16f5c6ae7f364f512415d423f9b9dc67fc638df4"},
+    {file = "botocore-1.34.118.tar.gz", hash = "sha256:0a3d1ec0186f8b516deb39474de3d226d531f77f92a0f56ad79b80219db3ae9e"},
 ]
 
 [package.dependencies]
@@ -80,15 +80,26 @@ urllib3 = [
 [package.extras]
 crt = ["awscrt (==0.20.9)"]
 
+[[package]]
+name = "cachetools"
+version = "5.3.3"
+description = "Extensible memoizing collections and decorators"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"},
+    {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"},
+]
+
 [[package]]
 name = "certifi"
-version = "2024.2.2"
+version = "2024.6.2"
 description = "Python package for providing Mozilla's CA Bundle."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"},
-    {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"},
+    {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"},
+    {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"},
 ]
 
 [[package]]
@@ -254,127 +265,6 @@ files = [
     {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"},
 ]
 
-[[package]]
-name = "cachetools"
-version = "5.3.3"
-description = "Extensible memoizing collections and decorators"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"},
-    {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"},
-]
-
-[[package]]
-name = "certifi"
-version = "2024.2.2"
-description = "Python package for providing Mozilla's CA Bundle."
-optional = false
-python-versions = ">=3.6"
-files = [
-    {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"},
-    {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"},
-]
-
-[[package]]
-name = "charset-normalizer"
-version = "3.3.2"
-description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
-optional = false
-python-versions = ">=3.7.0"
-files = [
-    {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"},
-    {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"},
-    {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"},
-    {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"},
-    {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"},
-    {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"},
-    {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"},
-    {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"},
-]
-
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -388,71 +278,69 @@ files = [
 
 [[package]]
 name = "coverage"
-version = "7.5.1"
+version = "7.5.3"
 description = "Code coverage measurement for Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "coverage-7.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e"},
-    {file = "coverage-7.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f"},
-    {file = "coverage-7.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a"},
-    {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35"},
-    {file = "coverage-7.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e"},
-    {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223"},
-    {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e"},
-    {file = "coverage-7.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146"},
-    {file = "coverage-7.5.1-cp310-cp310-win32.whl", hash = "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228"},
-    {file = "coverage-7.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8"},
-    {file = "coverage-7.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428"},
-    {file = "coverage-7.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746"},
-    {file = "coverage-7.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3"},
-    {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2"},
-    {file = "coverage-7.5.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca"},
-    {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8"},
-    {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057"},
-    {file = "coverage-7.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987"},
-    {file = "coverage-7.5.1-cp311-cp311-win32.whl", hash = "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136"},
-    {file = "coverage-7.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd"},
-    {file = "coverage-7.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206"},
-    {file = "coverage-7.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34"},
-    {file = "coverage-7.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d"},
-    {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa"},
-    {file = "coverage-7.5.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e"},
-    {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572"},
-    {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07"},
-    {file = "coverage-7.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7"},
-    {file = "coverage-7.5.1-cp312-cp312-win32.whl", hash = "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19"},
-    {file = "coverage-7.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596"},
-    {file = "coverage-7.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7"},
-    {file = "coverage-7.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90"},
-    {file = "coverage-7.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e"},
-    {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5"},
-    {file = "coverage-7.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661"},
-    {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8"},
-    {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4"},
-    {file = "coverage-7.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d"},
-    {file = "coverage-7.5.1-cp38-cp38-win32.whl", hash = "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41"},
-    {file = "coverage-7.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de"},
-    {file = "coverage-7.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1"},
-    {file = "coverage-7.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece"},
-    {file = "coverage-7.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26"},
-    {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5"},
-    {file = "coverage-7.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601"},
-    {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be"},
-    {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f"},
-    {file = "coverage-7.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668"},
-    {file = "coverage-7.5.1-cp39-cp39-win32.whl", hash = "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981"},
-    {file = "coverage-7.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f"},
-    {file = "coverage-7.5.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312"},
-    {file = "coverage-7.5.1.tar.gz", hash = "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c"},
+    {file = "coverage-7.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a6519d917abb15e12380406d721e37613e2a67d166f9fb7e5a8ce0375744cd45"},
+    {file = "coverage-7.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aea7da970f1feccf48be7335f8b2ca64baf9b589d79e05b9397a06696ce1a1ec"},
+    {file = "coverage-7.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:923b7b1c717bd0f0f92d862d1ff51d9b2b55dbbd133e05680204465f454bb286"},
+    {file = "coverage-7.5.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62bda40da1e68898186f274f832ef3e759ce929da9a9fd9fcf265956de269dbc"},
+    {file = "coverage-7.5.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8b7339180d00de83e930358223c617cc343dd08e1aa5ec7b06c3a121aec4e1d"},
+    {file = "coverage-7.5.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:25a5caf742c6195e08002d3b6c2dd6947e50efc5fc2c2205f61ecb47592d2d83"},
+    {file = "coverage-7.5.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:05ac5f60faa0c704c0f7e6a5cbfd6f02101ed05e0aee4d2822637a9e672c998d"},
+    {file = "coverage-7.5.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:239a4e75e09c2b12ea478d28815acf83334d32e722e7433471fbf641c606344c"},
+    {file = "coverage-7.5.3-cp310-cp310-win32.whl", hash = "sha256:a5812840d1d00eafae6585aba38021f90a705a25b8216ec7f66aebe5b619fb84"},
+    {file = "coverage-7.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:33ca90a0eb29225f195e30684ba4a6db05dbef03c2ccd50b9077714c48153cac"},
+    {file = "coverage-7.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f81bc26d609bf0fbc622c7122ba6307993c83c795d2d6f6f6fd8c000a770d974"},
+    {file = "coverage-7.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7cec2af81f9e7569280822be68bd57e51b86d42e59ea30d10ebdbb22d2cb7232"},
+    {file = "coverage-7.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55f689f846661e3f26efa535071775d0483388a1ccfab899df72924805e9e7cd"},
+    {file = "coverage-7.5.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50084d3516aa263791198913a17354bd1dc627d3c1639209640b9cac3fef5807"},
+    {file = "coverage-7.5.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:341dd8f61c26337c37988345ca5c8ccabeff33093a26953a1ac72e7d0103c4fb"},
+    {file = "coverage-7.5.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ab0b028165eea880af12f66086694768f2c3139b2c31ad5e032c8edbafca6ffc"},
+    {file = "coverage-7.5.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5bc5a8c87714b0c67cfeb4c7caa82b2d71e8864d1a46aa990b5588fa953673b8"},
+    {file = "coverage-7.5.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:38a3b98dae8a7c9057bd91fbf3415c05e700a5114c5f1b5b0ea5f8f429ba6614"},
+    {file = "coverage-7.5.3-cp311-cp311-win32.whl", hash = "sha256:fcf7d1d6f5da887ca04302db8e0e0cf56ce9a5e05f202720e49b3e8157ddb9a9"},
+    {file = "coverage-7.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:8c836309931839cca658a78a888dab9676b5c988d0dd34ca247f5f3e679f4e7a"},
+    {file = "coverage-7.5.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:296a7d9bbc598e8744c00f7a6cecf1da9b30ae9ad51c566291ff1314e6cbbed8"},
+    {file = "coverage-7.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:34d6d21d8795a97b14d503dcaf74226ae51eb1f2bd41015d3ef332a24d0a17b3"},
+    {file = "coverage-7.5.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e317953bb4c074c06c798a11dbdd2cf9979dbcaa8ccc0fa4701d80042d4ebf1"},
+    {file = "coverage-7.5.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:705f3d7c2b098c40f5b81790a5fedb274113373d4d1a69e65f8b68b0cc26f6db"},
+    {file = "coverage-7.5.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1196e13c45e327d6cd0b6e471530a1882f1017eb83c6229fc613cd1a11b53cd"},
+    {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:015eddc5ccd5364dcb902eaecf9515636806fa1e0d5bef5769d06d0f31b54523"},
+    {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:fd27d8b49e574e50caa65196d908f80e4dff64d7e592d0c59788b45aad7e8b35"},
+    {file = "coverage-7.5.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:33fc65740267222fc02975c061eb7167185fef4cc8f2770267ee8bf7d6a42f84"},
+    {file = "coverage-7.5.3-cp312-cp312-win32.whl", hash = "sha256:7b2a19e13dfb5c8e145c7a6ea959485ee8e2204699903c88c7d25283584bfc08"},
+    {file = "coverage-7.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:0bbddc54bbacfc09b3edaec644d4ac90c08ee8ed4844b0f86227dcda2d428fcb"},
+    {file = "coverage-7.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f78300789a708ac1f17e134593f577407d52d0417305435b134805c4fb135adb"},
+    {file = "coverage-7.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b368e1aee1b9b75757942d44d7598dcd22a9dbb126affcbba82d15917f0cc155"},
+    {file = "coverage-7.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f836c174c3a7f639bded48ec913f348c4761cbf49de4a20a956d3431a7c9cb24"},
+    {file = "coverage-7.5.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:244f509f126dc71369393ce5fea17c0592c40ee44e607b6d855e9c4ac57aac98"},
+    {file = "coverage-7.5.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4c2872b3c91f9baa836147ca33650dc5c172e9273c808c3c3199c75490e709d"},
+    {file = "coverage-7.5.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dd4b3355b01273a56b20c219e74e7549e14370b31a4ffe42706a8cda91f19f6d"},
+    {file = "coverage-7.5.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f542287b1489c7a860d43a7d8883e27ca62ab84ca53c965d11dac1d3a1fab7ce"},
+    {file = "coverage-7.5.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:75e3f4e86804023e991096b29e147e635f5e2568f77883a1e6eed74512659ab0"},
+    {file = "coverage-7.5.3-cp38-cp38-win32.whl", hash = "sha256:c59d2ad092dc0551d9f79d9d44d005c945ba95832a6798f98f9216ede3d5f485"},
+    {file = "coverage-7.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:fa21a04112c59ad54f69d80e376f7f9d0f5f9123ab87ecd18fbb9ec3a2beed56"},
+    {file = "coverage-7.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5102a92855d518b0996eb197772f5ac2a527c0ec617124ad5242a3af5e25f85"},
+    {file = "coverage-7.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d1da0a2e3b37b745a2b2a678a4c796462cf753aebf94edcc87dcc6b8641eae31"},
+    {file = "coverage-7.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8383a6c8cefba1b7cecc0149415046b6fc38836295bc4c84e820872eb5478b3d"},
+    {file = "coverage-7.5.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9aad68c3f2566dfae84bf46295a79e79d904e1c21ccfc66de88cd446f8686341"},
+    {file = "coverage-7.5.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e079c9ec772fedbade9d7ebc36202a1d9ef7291bc9b3a024ca395c4d52853d7"},
+    {file = "coverage-7.5.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bde997cac85fcac227b27d4fb2c7608a2c5f6558469b0eb704c5726ae49e1c52"},
+    {file = "coverage-7.5.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:990fb20b32990b2ce2c5f974c3e738c9358b2735bc05075d50a6f36721b8f303"},
+    {file = "coverage-7.5.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3d5a67f0da401e105753d474369ab034c7bae51a4c31c77d94030d59e41df5bd"},
+    {file = "coverage-7.5.3-cp39-cp39-win32.whl", hash = "sha256:e08c470c2eb01977d221fd87495b44867a56d4d594f43739a8028f8646a51e0d"},
+    {file = "coverage-7.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:1d2a830ade66d3563bb61d1e3c77c8def97b30ed91e166c67d0632c018f380f0"},
+    {file = "coverage-7.5.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:3538d8fb1ee9bdd2e2692b3b18c22bb1c19ffbefd06880f5ac496e42d7bb3884"},
+    {file = "coverage-7.5.3.tar.gz", hash = "sha256:04aefca5190d1dc7a53a4c1a5a7f8568811306d7a8ee231c42fb69215571944f"},
 ]
 
 [package.extras]
 toml = ["tomli"]
 
 [[package]]
-<<<<<<< HEAD
-=======
 name = "cryptography"
 version = "42.0.7"
 description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
@@ -507,7 +395,6 @@ test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-co
 test-randomorder = ["pytest-randomly"]
 
 [[package]]
->>>>>>> azure-connector-tests
 name = "exceptiongroup"
 version = "1.2.1"
 description = "Backport of PEP 654 (exception groups)"
@@ -522,7 +409,6 @@ files = [
 test = ["pytest (>=6)"]
 
 [[package]]
-<<<<<<< HEAD
 name = "google-api-core"
 version = "2.19.0"
 description = "Google API client core library"
@@ -708,24 +594,22 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"]
 
 [[package]]
 name = "googleapis-common-protos"
-version = "1.63.0"
+version = "1.63.1"
 description = "Common protobufs used in Google APIs"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "googleapis-common-protos-1.63.0.tar.gz", hash = "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e"},
-    {file = "googleapis_common_protos-1.63.0-py2.py3-none-any.whl", hash = "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632"},
+    {file = "googleapis-common-protos-1.63.1.tar.gz", hash = "sha256:c6442f7a0a6b2a80369457d79e6672bb7dcbaab88e0848302497e3ec80780a6a"},
+    {file = "googleapis_common_protos-1.63.1-py2.py3-none-any.whl", hash = "sha256:0e1c2cdfcbc354b76e4a211a35ea35d6926a835cba1377073c4861db904a1877"},
 ]
 
 [package.dependencies]
-protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0"
+protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0"
 
 [package.extras]
 grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"]
 
 [[package]]
-=======
->>>>>>> azure-connector-tests
 name = "idna"
 version = "3.7"
 description = "Internationalized Domain Names in Applications (IDNA)"
@@ -799,7 +683,6 @@ dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
 [[package]]
-<<<<<<< HEAD
 name = "proto-plus"
 version = "1.23.0"
 description = "Beautiful, Pythonic protocol buffers."
@@ -861,7 +744,7 @@ files = [
 [package.dependencies]
 pyasn1 = ">=0.4.6,<0.7.0"
 
-=======
+[[package]]
 name = "pycparser"
 version = "2.22"
 description = "C parser in Python"
@@ -872,16 +755,15 @@ files = [
     {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
 ]
 
->>>>>>> azure-connector-tests
 [[package]]
 name = "pytest"
-version = "8.2.0"
+version = "8.2.2"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"},
-    {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"},
+    {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"},
+    {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"},
 ]
 
 [package.dependencies]
@@ -889,11 +771,7 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""}
 exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
-<<<<<<< HEAD
-pluggy = ">=1.4,<2.0"
-=======
 pluggy = ">=1.5,<2.0"
->>>>>>> azure-connector-tests
 tomli = {version = ">=1", markers = "python_version < \"3.11\""}
 
 [package.extras]
@@ -915,23 +793,13 @@ six = ">=1.5"
 
 [[package]]
 name = "requests"
-<<<<<<< HEAD
-version = "2.31.0"
-description = "Python HTTP for Humans."
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
-    {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
-=======
-version = "2.32.1"
+version = "2.32.3"
 description = "Python HTTP for Humans."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "requests-2.32.1-py3-none-any.whl", hash = "sha256:21ac9465cdf8c1650fe1ecde8a71669a93d4e6f147550483a2967d08396a56a5"},
-    {file = "requests-2.32.1.tar.gz", hash = "sha256:eb97e87e64c79e64e5b8ac75cee9dd1f97f49e289b083ee6be96268930725685"},
->>>>>>> azure-connector-tests
+    {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"},
+    {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
 ]
 
 [package.dependencies]
@@ -944,7 +812,6 @@ urllib3 = ">=1.21.1,<3"
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
 use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
-<<<<<<< HEAD
 [[package]]
 name = "rsa"
 version = "4.9"
@@ -959,8 +826,6 @@ files = [
 [package.dependencies]
 pyasn1 = ">=0.1.3"
 
-=======
->>>>>>> azure-connector-tests
 [[package]]
 name = "s3transfer"
 version = "0.10.1"
@@ -1001,7 +866,17 @@ files = [
 ]
 
 [[package]]
-<<<<<<< HEAD
+name = "typing-extensions"
+version = "4.12.1"
+description = "Backported and Experimental Type Hints for Python 3.8+"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "typing_extensions-4.12.1-py3-none-any.whl", hash = "sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a"},
+    {file = "typing_extensions-4.12.1.tar.gz", hash = "sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1"},
+]
+
+[[package]]
 name = "urllib3"
 version = "1.26.18"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
@@ -1017,18 +892,6 @@ brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotl
 secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
 socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 
-=======
-name = "typing-extensions"
-version = "4.11.0"
-description = "Backported and Experimental Type Hints for Python 3.8+"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"},
-    {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"},
-]
-
->>>>>>> azure-connector-tests
 [[package]]
 name = "urllib3"
 version = "2.2.1"
@@ -1048,10 +911,5 @@ zstd = ["zstandard (>=0.18.0)"]
 
 [metadata]
 lock-version = "2.0"
-<<<<<<< HEAD
 python-versions = "^3.8"
-content-hash = "aba102d48a694f410cb09cd8d862442328bf01203a1355072d3a35cf41dad874"
-=======
-python-versions = "^3.10"
-content-hash = "bfee156550f4d9533314ff0c47dd53e872e764a3640e9711a85be6efdda0d52e"
->>>>>>> azure-connector-tests
+content-hash = "16b9fc2ffee90628fc2779e27e879cee391c930b8695a26faf00c8f167153af4"
diff --git a/tests/create_tables.py b/tests/create_tables.py
index 6986ae5..cd76e39 100644
--- a/tests/create_tables.py
+++ b/tests/create_tables.py
@@ -1,13 +1,16 @@
 import json
 import os
-<<<<<<< HEAD
-import psycopg2
-import yaml
-from obsrv.utils import EncryptionUtil
 from testcontainers.minio import MinioContainer
 from minio import Minio
 from minio.error import S3Error
 
+import psycopg2
+import yaml
+
+from obsrv.utils import EncryptionUtil
+from testcontainers.azurite import AzuriteContainer
+from azure.storage.blob import *
+
 def init_minio(connector_config):
 
     config=json.loads(connector_config)
@@ -19,6 +22,11 @@ def init_minio(connector_config):
         secure=False
 
     )
+    minio_container= MinioContainer("minio/minio:latest")
+    minio_container.start()
+    print("get_clinet",minio_container.get_client())
+    print("get_config",minio_container.get_config())
+    print("get_host_ip",minio_container.get_container_host_ip())
     bucket_name = config['source']['bucket']
 
     #Create Bucket
@@ -31,8 +39,14 @@ def init_minio(connector_config):
     except S3Error as e:
         print(f"Error: {e}")
 
+    # folder_name = "tests/sample_data"
+    # file_path = os.path.join(os.getcwd(), folder_name)
+    # file_path="/home/vince/repogit/object-store-connector/tests/sample_data"
+
+    # for filename in os.listdir(file_path):    
+    #     filepath=os.path.join(file_path,filename)
     object_name="data.json"
-    file_path = "/root/GitRepo/obsrv-python-sdk/tests/sample_data/nyt_data_100.json"
+    file_path = "/home/vince/repogit/object-store-connector/tests/sample_data/nyt_data_100.json"
     c_type="application/json"
  
     #Upload Objects
@@ -52,14 +66,8 @@ def init_minio(connector_config):
         print(f"Unexpected error: {e}")
 
     return minio_conf
-=======
 
-import psycopg2
-import yaml
 
-from obsrv.utils import EncryptionUtil
-from testcontainers.azurite import AzuriteContainer
-from azure.storage.blob import *
 
 def init_azurite():
     
@@ -85,6 +93,8 @@ def init_azurite():
 
     blob_service_client.create_container(container_name)
      
+    # folder_name = "tests/sample_data"
+    # folder_path = os.path.join(os.getcwd(), folder_name)
     folder_path="/home/vince/repogit/object-store-connector/tests/sample_data"
     for filename in os.listdir(folder_path):    
         filepath=os.path.join(folder_path,filename)
@@ -94,12 +104,10 @@ def init_azurite():
     return azure_conf
 
 
->>>>>>> azure-connector-tests
 
 
 def create_tables(config):
     enc = EncryptionUtil(config["obsrv_encryption_key"])
-<<<<<<< HEAD
     minio_container= MinioContainer("minio/minio:latest")
     minio_container.start()
     
@@ -121,11 +129,9 @@ def create_tables(config):
     # Initialize Minio client
     minio_conf = init_minio(connector_config)
 
-=======
 
     azure_conf = init_azurite()
    
->>>>>>> azure-connector-tests
     datasets = """
         CREATE TABLE IF NOT EXISTS datasets (
             id TEXT PRIMARY KEY,
@@ -196,31 +202,19 @@ def create_tables(config):
 
     ins_ds = """
         INSERT INTO datasets (id, dataset_id, type, name, validation_config, extraction_config, dedup_config, data_schema, denorm_config, router_config, dataset_config, tags, data_version, status, created_by, updated_by, created_date, updated_date, published_date) VALUES
-<<<<<<< HEAD
-        ('new-york-taxi-data', 'new-york-taxi-data', 'dataset', 'new-york-taxi-data', '{"validate": true, "mode": "Strict", "validation_mode": "Strict"}', '{"is_batch_event": false}', '{"drop_duplicates": true, "dedup_key": "tripID", "dedup_period": 604800}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"tripID":{"type":"string","suggestions":[{"message":"The Property tripID appears to be uuid format type.","advice":"Suggest to not to index the high cardinal columns","resolutionType":"DEDUP","severity":"LOW","path":"properties.tripID"}],"arrival_format":"text","data_type":"string"}},"additionalProperties":false}', '{}', '{"topic": "new-york-taxi-data"}', '{"data_key": "", "timestamp_key": "tpep_pickup_datetime", "exclude_fields": [], "entry_topic": "s3.ingest", "redis_db_host": "obsrv-dedup-redis-master.redis.svc.cluster.local", "redis_db_port": 6379, "index_data": true, "redis_db": 0}', '{}', '1', 'Live', 'SYSTEM', 'SYSTEM', '2024-03-27 06:48:35.993478', '2024-03-27 06:48:35.993478', '2024-03-27 06:48:35.993478');
-=======
         ('new-york-taxi-data', 'new-york-taxi-data', 'dataset', 'new-york-taxi-data', '{"validate": true, "mode": "Strict", "validation_mode": "Strict"}', '{"is_batch_event": false}', '{"drop_duplicates": true, "dedup_key": "tripID", "dedup_period": 604800}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"tripID":{"type":"string","suggestions":[{"message":"The Property tripID appears to be uuid format type.","advice":"Suggest to not to index the high cardinal columns","resolutionType":"DEDUP","severity":"LOW","path":"properties.tripID"}],"arrival_format":"text","data_type":"string"}},"additionalProperties":false}', '{}', '{"topic": "new-york-taxi-data"}', '{"data_key": "", "timestamp_key": "tpep_pickup_datetime", "exclude_fields": [], "entry_topic": "azure.ingest", "redis_db_host": "obsrv-dedup-redis-master.redis.svc.cluster.local", "redis_db_port": 6379, "index_data": true, "redis_db": 0}', '{}', '1', 'Live', 'SYSTEM', 'SYSTEM', '2024-03-27 06:48:35.993478', '2024-03-27 06:48:35.993478', '2024-03-27 06:48:35.993478');
->>>>>>> azure-connector-tests
     """
 
     # ins_ds = """
     #     INSERT INTO datasets (id, dataset_id, type, name, dataset_config, status, created_by, updated_by) VALUES
-<<<<<<< HEAD
     #     ('new-york-taxi-data', 'new-york-taxi-data', 'dataset', 'new-york-taxi-data', '{"entry_topic": "test.ingest"}', 'Live', 'SYSTEM', 'SYSTEM');
-=======
     #     ('new-york-taxi-data', 'new-york-taxi-data', 'dataset', 'new-york-taxi-data', '{"entry_topic": "azure.ingest"}', 'Live', 'SYSTEM', 'SYSTEM');
->>>>>>> azure-connector-tests
     # """
 
     ins_cr = """
         INSERT INTO connector_registry (id, version, type, category, name, description, technology, licence, owner, iconURL, status, created_by, updated_by, updated_date) VALUES
-<<<<<<< HEAD
-        ('s3.1', '1', 'source', 'object', 'test_reader', 'test_reader', 'Python', 'Apache 2.0', 'ravi@obsrv.ai', 'http://localhost', 'Live', 'SYSTEM', 'SYSTEM', now());
-    """
-
-
-=======
-        ('azure.1', '1', 'source', 'object', 'test_reader', 'test_reader', 'Python', 'Apache 2.0', 'ravi@obsrv.ai', 'http://localhost', 'Live', 'SYSTEM', 'SYSTEM', now());
+        ('s3.1', '1', 'source', 'object', 'test_reader', 'test_reader', 'Python', 'Apache 2.0', 'ravi@obsrv.ai', 'http://localhost', 'Live', 'SYSTEM', 'SYSTEM', now()),
+     ('azure.1', '1', 'source', 'object', 'test_reader', 'test_reader', 'Python', 'Apache 2.0', 'ravi@obsrv.ai', 'http://localhost', 'Live', 'SYSTEM', 'SYSTEM', now());
     """
 
     connector_config = json.dumps(
@@ -235,21 +229,16 @@ def create_tables(config):
         "blob_endpoint":azure_conf['blobEndpoint'],
         "prefix":"/"
         }})
->>>>>>> azure-connector-tests
     enc_config = enc.encrypt(connector_config)
 
     ins_ci = """
         INSERT INTO connector_instances (id, dataset_id, connector_id, connector_type, connector_config, operations_config, status, connector_state, connector_stats, created_by, updated_by, created_date, updated_date, published_date) VALUES
-<<<<<<< HEAD
         ('s3.new-york-taxi-data.1', 'new-york-taxi-data', 's3.1', 'source', %s, '{}', 'Live', '{}', '{}', 'SYSTEM', 'SYSTEM', now(), now(), now()
-        );
-    """
+        ),
   
-=======
         ('azure.new-york-taxi-data.1', 'new-york-taxi-data', 'azure.1', 'source', %s, '{}', 'Live', '{}', '{}', 'SYSTEM', 'SYSTEM', now(), now(), now()
         );
     """
->>>>>>> azure-connector-tests
 
     with open(
         os.path.join(os.path.dirname(__file__), "config/config.yaml"), "r"
@@ -264,26 +253,6 @@ def create_tables(config):
         )
 
         cur = conn.cursor()
-<<<<<<< HEAD
-        try : 
-            cur.execute(datasets)
-            cur.execute(connector_registry)
-            cur.execute(connector_instances)
-            cur.execute(indexes)
-            cur.execute(ins_ds)
-            cur.execute(ins_cr)
-            cur.execute(ins_ci, (json.dumps(enc_config),))
-
-            conn.commit()
-        except Exception as e:
-            print(f"Error occurred: {e}")
-            conn.rollback()
-        finally:
-            conn.close()
-            minio_container.stop()
-
-        return
-=======
 
         cur.execute(datasets)
         cur.execute(connector_registry)
@@ -291,8 +260,7 @@ def create_tables(config):
         cur.execute(indexes)
         cur.execute(ins_ds)
         cur.execute(ins_cr)
-        cur.execute(ins_ci, (json.dumps(enc_config),))
+        cur.execute(ins_ci, (json.dumps(enc_config),(json.dumps(enc_config))))
 
         conn.commit()
         conn.close()
->>>>>>> azure-connector-tests
diff --git a/tests/test_s3.py b/tests/test_s3.py
index 80d00b7..3c3abc7 100644
--- a/tests/test_s3.py
+++ b/tests/test_s3.py
@@ -2,7 +2,7 @@
 import unittest
 from typing import Any, Dict
 import sys
-sys.path.insert(0, '/root/GitRepo/object-store-connector/object_store_connector')
+sys.path.insert(0, '/home/vince/repogit/object-store-connector/object_store_connector')
 
 # from connector import Info
 
@@ -12,7 +12,8 @@
 from kafka import KafkaConsumer, TopicPartition
 from pyspark.conf import SparkConf
 from pyspark.sql import DataFrame, SparkSession
-from connector import ObjectStoreConnector
+#
+from object_store_connector.connector import ObjectStoreConnector
 from obsrv.connector import ConnectorContext, MetricsCollector
 from obsrv.connector.batch import ISourceConnector, SourceConnector
 from obsrv.job.batch import get_base_conf
@@ -20,7 +21,7 @@
 
 
 
-class ObjectStoreConnector(ISourceConnector):
+class TestSource(ISourceConnector):
     def process(
         self,
         sc: SparkSession,

From 74f73bbefde980f25777599ec28f098d3151d2d2 Mon Sep 17 00:00:00 2001
From: vinayak-sanketika <vinayakad@sanketika.in>
Date: Fri, 7 Jun 2024 16:44:38 +0530
Subject: [PATCH 59/59] obsrv-labs/issue-tracker#6: feat :cleanup

---
 tests/create_tables.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tests/create_tables.py b/tests/create_tables.py
index cd76e39..885fe84 100644
--- a/tests/create_tables.py
+++ b/tests/create_tables.py
@@ -22,11 +22,7 @@ def init_minio(connector_config):
         secure=False
 
     )
-    minio_container= MinioContainer("minio/minio:latest")
-    minio_container.start()
-    print("get_clinet",minio_container.get_client())
-    print("get_config",minio_container.get_config())
-    print("get_host_ip",minio_container.get_container_host_ip())
+   
     bucket_name = config['source']['bucket']
 
     #Create Bucket