-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmain.py
118 lines (93 loc) · 3.89 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
import sys
import time
import schedule
from settings import rounds, init_value, round_increase, advanced
from web import get_map_properties, get_advanced_map_properties
from properties import url_properties
from spreadsheet import get_number_of_sheets, get_map_url, insert_data
# Schedule for automatic updates
def scheduler():
schedule.every().day.at("09:00").do(initializer)
schedule.every().day.at("15:00").do(initializer)
schedule.every().day.at("21:00").do(initializer)
while True:
schedule.run_pending()
time.sleep(45)
# Initializing the program by formatting arguments
def initializer():
# Get number of sheets
number_of_sheets = get_number_of_sheets()
# Loop through the sheets
for x in range(number_of_sheets):
current_map = get_map_url(x)
if not current_map == '':
# Split the url and url variables
url = current_map.split('?')[0]
url_variables = current_map.split('?')[1].split('&')
# Execute the program
result = run(url, url_variables)
# Sorting results (x[0] = bedrooms || x[1] = price || x[2] = price/bedroom || x[3] = m2)
sorted_result = sorted(result, key=lambda x: x[1], reverse=False)
# Insert data into the sheet
insert_data(x, sorted_result)
# Run
def run(url, url_variables):
# Variables
result = []
min_bedrooms = 0
# Loop through the rounds
for x in range(rounds):
# Variables for issue handling
issue_counter = 0
extract_properties_issues = True
# Variables for iterating number of bedrooms and cost
min_bedrooms = min_bedrooms + 1
max_amount = init_value + (round_increase * x)
# Format the url properties
internal_url = url_properties(url, url_variables, min_bedrooms, max_amount)
# Get properties off finn
if advanced:
# Issue handling loop for property extraction
while extract_properties_issues and issue_counter < 3:
try:
properties = get_advanced_map_properties(internal_url)
extract_properties_issues = False
except:
issue_counter += 1
print('There were issues during property extraction. Trying again...')
# Append results to list
for y in range(len(properties)):
exist = False
price_per_bedroom = int(properties[y][5]) / int(properties[y][4])
d = [properties[y][4], properties[y][5], str(price_per_bedroom), properties[y][1], properties[y][2], properties[y][3], properties[y][0], internal_url]
for z in range(len(result)):
if result[z][0:6] == d[0:6]:
exist = True
if not exist:
result.append(d)
else:
print('This property is already included in the result list.')
else:
# Issue handling loop for property extraction
while extract_properties_issues and issue_counter < 3:
try:
properties = get_map_properties(internal_url)
except:
issue_counter += 1
print('There were issues during property extraction. Trying again...')
# Append results to list
for y in range(len(properties)):
d = [min_bedrooms, max_amount, 'https://www.finn.no/' + properties[y], internal_url]
result.append(d)
# Printing status
print('Current map progress: ' + str(x + 1) + '/' + str(rounds) + '...')
# Return properties found for current map
return result
# Handle execution
if __name__ == "__main__":
argument = sys.argv
try:
if argument[1] == 'auto':
scheduler()
except:
initializer()