Merge pull request #714 from OpenUpSA/bug-719-error-reading-from-s3

Set S3_URL_PREFIX to link to bulk downloads

municipal_finance/settings.py

@@ -456,13 +456,14 @@
 AWS_ACCESS_KEY_ID = env.str("AWS_ACCESS_KEY_ID", "")
 AWS_SECRET_ACCESS_KEY = env.str("AWS_SECRET_ACCESS_KEY", "")
 AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME", "")
-AWS_S3_CUSTOM_DOMAIN = "%s.s3.amazonaws.com" % AWS_STORAGE_BUCKET_NAME
+AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
 AWS_S3_OBJECT_PARAMETERS = {
     "CacheControl": "max-age=86400",
 }
 AWS_S3_ENDPOINT_URL = env.str("AWS_S3_ENDPOINT_URL", None)
 AWS_DEFAULT_ACL = "public-read"
 AWS_BUCKET_ACL = "public-read"
+AWS_REGION = env.str("AWS_REGION", "eu-west-1")
 UPDATE_BULK_DOWNLOADS = env.str("UPDATE_BULK_DOWNLOADS", False)
 BULK_DOWNLOAD_DIR = env.str("BULK_DOWNLOAD_DIR", "")
 

municipal_finance/storage.py

@@ -1,6 +1,8 @@
 from storages.backends.s3boto3 import S3Boto3Storage
+from municipal_finance import settings
 
 
 class MediaStorage(S3Boto3Storage):
     location = "media"
     file_overwrite = False
+    region_name = settings.AWS_REGION

municipal_finance/update/aged_creditor_facts_v2.py

@@ -85,6 +85,6 @@ def row_to_obj(self, row):
         )
 
 
-def update_aged_creditor_facts_v2(update_obj, batch_size):
+def update_aged_creditor_facts_v2(update_obj, batch_size, **kwargs):
     updater = AgedCreditorFactsUpdater(update_obj, batch_size)
     updater.update()

municipal_finance/update/audit_opinion_facts.py

@@ -52,6 +52,6 @@ def row_to_obj(self, row):
         )
 
 
-def update_audit_opinion_facts(update_obj, batch_size):
+def update_audit_opinion_facts(update_obj, batch_size, **kwargs):
     updater = AuditOpinionFactsUpdater(update_obj, batch_size)
     updater.update()

municipal_finance/update/capital_facts_v2.py

@@ -80,6 +80,6 @@ def row_to_obj(self, row):
         )
 
 
-def update_capital_facts_v2(update_obj, batch_size):
+def update_capital_facts_v2(update_obj, batch_size, **kwargs):
     updater = CapitalFactsV2Updater(update_obj, batch_size)
     updater.update()

municipal_finance/update/cash_flow_v2.py

@@ -75,6 +75,6 @@ def row_to_obj(self, row: CashFlowFactRow):
         )
 
 
-def update_cash_flow_v2(update_obj, batch_size):
+def update_cash_flow_v2(update_obj, batch_size, **kwargs):
     updater = CashFlowFactsV2Updater(update_obj, batch_size)
     updater.update()

municipal_finance/update/financial_position_facts_v2.py

@@ -70,6 +70,6 @@ def row_to_obj(self, row):
         )
 
 
-def update_financial_position_facts_v2(update_obj, batch_size):
+def update_financial_position_facts_v2(update_obj, batch_size, **kwargs):
     updater = FinancialPositionFactsUpdater(update_obj, batch_size)
     updater.update()

municipal_finance/update/grant_facts_v2.py

@@ -70,6 +70,6 @@ def row_to_obj(self, row):
         )
 
 
-def update_grant_facts_v2(update_obj, batch_size):
+def update_grant_facts_v2(update_obj, batch_size, **kwargs):
     updater = GrantFactsUpdater(update_obj, batch_size)
     updater.update()

municipal_finance/update/income_expenditure_v2.py

@@ -79,7 +79,7 @@ def row_to_obj(self, row: IncomeExpenditureFactRow):
         )
 
 
-def update_income_expenditure_v2(update_obj, batch_size):
+def update_income_expenditure_v2(update_obj, batch_size, **kwargs):
     updater = IncomeExpenditureFactsV2Updater(
         update_obj, batch_size,
     )

municipal_finance/update/municipal_staff_contacts.py

@@ -58,6 +58,6 @@ def row_to_obj(self, row: MunicipalStaffContactRow):
         )
 
 
-def update_municipal_staff_contacts(update_obj, batch_size):
+def update_municipal_staff_contacts(update_obj, batch_size, **kwargs):
     updater = MunicipalStaffContactsUpdater(update_obj, batch_size)
     updater.update()

municipal_finance/update/repairs_maintenance_v2.py

@@ -75,7 +75,7 @@ def row_to_obj(self, row: RepairsMaintenanceFactRow):
         )
 
 
-def update_repairs_maintenance_v2(update_obj, batch_size):
+def update_repairs_maintenance_v2(update_obj, batch_size, **kwargs):
     updater = RepairsMaintenanceFactsV2Updater(
         update_obj, batch_size,
     )

municipal_finance/update/uifw_expense_facts.py

@@ -56,6 +56,6 @@ def row_to_obj(self, row):
         )
 
 
-def update_uifw_expense_facts(update_obj, batch_size):
+def update_uifw_expense_facts(update_obj, batch_size, **kwargs):
     updater = FinancialPositionFactsUpdater(update_obj, batch_size)
     updater.update()

municipal_finance/update/utils.py

@@ -160,5 +160,5 @@ def update(self):
                     objects = cls.facts_cls.objects.bulk_create(objects)
                     self.update_obj.inserted += len(objects)
             # Save the status of the update
-            config.IS_SCORECARD_COMPILED  = False
+            config.IS_SCORECARD_COMPILED = False
             self.update_obj.save(update_fields=["deleted", "inserted"])

municipal_finance/views.py

@@ -120,7 +120,7 @@ def docs(request):
     return render(request, 'docs.html', {
         'cubes': cubes,
         'bulk_downloads': bulk_downloads,
-        'storage_url': settings.AWS_S3_ENDPOINT_URL,
+        'storage_url': settings.S3_URL_PREFIX,
     })
 
 

requirements.txt

@@ -1,8 +1,8 @@
 arrow==0.15.5
 babbage==0.3.6
 blessed==1.17.2
-boto3==1.16.63
-botocore==1.19.63
+boto3==1.17.112
+botocore==1.20.112
 certifi==2022.12.7
 chardet==3.0.4
 decorator==4.0.9
@@ -47,7 +47,7 @@ pytz==2017.3
 PyYAML==5.4
 requests==2.26.0
 requests-futures==0.9.7
-s3transfer==0.3.3
+s3transfer==0.4.2
 sentry-sdk==1.26.0
 simplegeneric==0.8.1
 six==1.10.0