-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcollector.py
148 lines (109 loc) · 7.59 KB
/
collector.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
import requests
import json
import aiohttp
# The Class Redfin is taken from https://github.com/reteps/redfin
# converted it to async for faster run time.
class Redfin:
def __init__(self):
self.base = 'https://redfin.com/stingray/'
self.user_agent_header = {
'user-agent': 'redfin'
}
async def meta_request(self, url, kwargs):
async with aiohttp.ClientSession() as session:
async with session.get(
self.base + url, params=kwargs, headers=self.user_agent_header
) as response:
if response.status != 200:
raise aiohttp.ClientError(f"Request failed with status {response.status}")
response_text = await response.text()
return json.loads(response_text[4:])
async def meta_property(self, url, kwargs, page=False):
if page:
kwargs['pageType'] = 3
return await self.meta_request('api/home/details/' + url, {
'accessLevel': 1,
**kwargs
})
# Url Requests
async def initial_info(self, url, **kwargs):
return await self.meta_request('api/home/details/initialInfo', {'path': url, **kwargs})
async def page_tags(self, url, **kwargs):
return await self.meta_request('api/home/details/v1/pagetagsinfo', {'path': url, **kwargs})
async def primary_region(self, url, **kwargs):
return await self.meta_request('api/home/details/primaryRegionInfo', {'path': url, **kwargs})
# Search
async def search(self, query, **kwargs):
return await self.meta_request('do/location-autocomplete', {'location': query, 'v': 2, **kwargs})
# Property ID Requests
async def below_the_fold(self, property_id, **kwargs):
return await self.meta_property('belowTheFold', {'propertyId': property_id, **kwargs}, page=True)
async def hood_photos(self, property_id, **kwargs):
return await self.meta_request('api/home/details/hood-photos', {'propertyId': property_id, **kwargs})
async def more_resources(self, property_id, **kwargs):
return await self.meta_request('api/home/details/moreResourcesInfo', {'propertyId': property_id, **kwargs})
async def page_header(self, property_id, **kwargs):
return await self.meta_request('api/home/details/homeDetailsPageHeaderInfo', {'propertyId': property_id, **kwargs})
async def property_comments(self, property_id, **kwargs):
return await self.meta_request('api/v1/home/details/propertyCommentsInfo', {'propertyId': property_id, **kwargs})
async def building_details_page(self, property_id, **kwargs):
return await self.meta_request('api/building/details-page/v1', {'propertyId': property_id, **kwargs})
async def owner_estimate(self, property_id, **kwargs):
return await self.meta_request('api/home/details/owner-estimate', {'propertyId': property_id, **kwargs})
async def claimed_home_seller_data(self, property_id, **kwargs):
return await self.meta_request('api/home/details/claimedHomeSellerData', {'propertyId': property_id, **kwargs})
async def cost_of_home_ownership(self, property_id, **kwargs):
return await self.meta_request('do/api/costOfHomeOwnershipDetails', {'propertyId': property_id, **kwargs})
# Listing ID Requests
async def floor_plans(self, listing_id, **kwargs):
return await self.meta_request('api/home/details/listing/floorplans', {'listingId': listing_id, **kwargs})
async def tour_list_date_picker(self, listing_id, **kwargs):
return await self.meta_request('do/tourlist/getDatePickerData', {'listingId': listing_id, **kwargs})
# Table ID Requests
async def shared_region(self, table_id, **kwargs):
return await self.meta_request('api/region/shared-region-info', {'tableId': table_id, 'regionTypeId': 2, 'mapPageTypeId': 1, **kwargs})
# Property Requests
async def similar_listings(self, property_id, listing_id, **kwargs):
return await self.meta_property('similars/listings', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def similar_sold(self, property_id, listing_id, **kwargs):
return await self.meta_property('similars/solds', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def nearby_homes(self, property_id, listing_id, **kwargs):
return await self.meta_property('nearbyhomes', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def above_the_fold(self, property_id, listing_id, **kwargs):
return await self.meta_property('aboveTheFold', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def property_parcel(self, property_id, listing_id, **kwargs):
return await self.meta_property('propertyParcelInfo', {'propertyId': property_id, 'listingId': listing_id, **kwargs}, page=True)
async def activity(self, property_id, listing_id, **kwargs):
return await self.meta_property('activityInfo', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def customer_conversion_info_off_market(self, property_id, listing_id, **kwargs):
return await self.meta_property('customerConversionInfo/offMarket', {'propertyId': property_id, 'listingId': listing_id, **kwargs}, page=True)
async def rental_estimate(self, property_id, listing_id, **kwargs):
return await self.meta_property('rental-estimate', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def avm_historical(self, property_id, listing_id, **kwargs):
return await self.meta_property('avmHistoricalData', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def info_panel(self, property_id, listing_id, **kwargs):
return await self.meta_property('mainHouseInfoPanelInfo', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def descriptive_paragraph(self, property_id, listing_id, **kwargs):
return await self.meta_property('descriptiveParagraph', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def avm_details(self, property_id, listing_id, **kwargs):
return await self.meta_property('avm', {'propertyId': property_id, 'listingId': listing_id, **kwargs})
async def tour_insights(self, property_id, listing_id, **kwargs):
return await self.meta_property('tourInsights', {'propertyId': property_id, 'listingId': listing_id, **kwargs}, page=True)
async def stats(self, property_id, listing_id, region_id, **kwargs):
return await self.meta_property('stats', {'regionId': region_id, 'propertyId': property_id, 'listingId': listing_id, 'regionTypeId': 2, **kwargs})
async def price_history(self, property_id, listing_id=None, **kwargs):
"""
Added additional function here to better suit the project needs.
Grabs the estimated historical price history for a given property from Redfin.
Args:
property_id (int): The unique identifier of the property.
listing_id (int, optional): The unique identifier of the listing for the property. Defaults to None.
**kwargs: Additional parameters to be passed to the meta_property function.
Returns:
dict: A dictionary containing the historical price data for the property.
"""
params = {'propertyId': property_id}
if listing_id:
params['listingId'] = listing_id
params.update(kwargs)
return await self.meta_property('avmHistoricalData', params)