From 52e63028b348495f0cde50cab6d980d1187e763e Mon Sep 17 00:00:00 2001 From: JD Bothma Date: Thu, 3 Dec 2020 18:46:22 +0200 Subject: [PATCH 1/8] Restore template used by data explorer site --- scorecard/templates/_videos.html | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scorecard/templates/_videos.html diff --git a/scorecard/templates/_videos.html b/scorecard/templates/_videos.html new file mode 100644 index 000000000..b52423563 --- /dev/null +++ b/scorecard/templates/_videos.html @@ -0,0 +1,15 @@ +{% load static %} + +
+
+ +
+ + +
From 98ad0117766c856d05e570464b711c9cbba97785 Mon Sep 17 00:00:00 2001 From: JD Bothma Date: Thu, 3 Dec 2020 18:47:09 +0200 Subject: [PATCH 2/8] Move _videos template to municipal_finance app where it's now solely used --- {scorecard => municipal_finance}/templates/_videos.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {scorecard => municipal_finance}/templates/_videos.html (100%) diff --git a/scorecard/templates/_videos.html b/municipal_finance/templates/_videos.html similarity index 100% rename from scorecard/templates/_videos.html rename to municipal_finance/templates/_videos.html From 0022504b6abc2a7c6f5253f408b5a219cdfed835 Mon Sep 17 00:00:00 2001 From: JD Bothma Date: Thu, 3 Dec 2020 18:47:55 +0200 Subject: [PATCH 3/8] Remove duplicate formats dir that's actually used at municipal_finance --- formats/__init__.py | 0 formats/en_ZA/__init__.py | 0 formats/en_ZA/formats.py | 1 - 3 files changed, 1 deletion(-) delete mode 100644 formats/__init__.py delete mode 100644 formats/en_ZA/__init__.py delete mode 100644 formats/en_ZA/formats.py diff --git a/formats/__init__.py b/formats/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/formats/en_ZA/__init__.py b/formats/en_ZA/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/formats/en_ZA/formats.py b/formats/en_ZA/formats.py deleted file mode 100644 index d26ea6eff..000000000 --- a/formats/en_ZA/formats.py +++ /dev/null @@ -1 +0,0 @@ -THOUSAND_SEPARATOR = ' ' From 33d9cf37ce3f77ba4d354538d6783af28efeb74d Mon Sep 17 00:00:00 2001 From: JD Bothma Date: Thu, 3 Dec 2020 18:48:20 +0200 Subject: [PATCH 4/8] Remove now-unused runtime.txt - replaced by Dockerfile --- runtime.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 runtime.txt diff --git a/runtime.txt b/runtime.txt deleted file mode 100644 index 02d0df5e8..000000000 --- a/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.6.3 From ee7eb92be827b1ae6d2b552d19d146023280a9f3 Mon Sep 17 00:00:00 2001 From: JD Bothma Date: Thu, 3 Dec 2020 18:48:46 +0200 Subject: [PATCH 5/8] Remove now-unsed search_indexes - replaced by postgres search --- search_indexes/backends.py | 11 --- search_indexes/documents.py | 79 ---------------- search_indexes/search.py | 15 ---- search_indexes/serializers.py | 33 ------- search_indexes/signals.py | 1 - search_indexes/urls.py | 17 ---- search_indexes/views.py | 82 ----------------- search_indexes/viewsets.py | 165 ---------------------------------- 8 files changed, 403 deletions(-) delete mode 100644 search_indexes/backends.py delete mode 100644 search_indexes/documents.py delete mode 100644 search_indexes/search.py delete mode 100644 search_indexes/serializers.py delete mode 100644 search_indexes/signals.py delete mode 100644 search_indexes/urls.py delete mode 100644 search_indexes/views.py delete mode 100644 search_indexes/viewsets.py diff --git a/search_indexes/backends.py b/search_indexes/backends.py deleted file mode 100644 index 54def98e1..000000000 --- a/search_indexes/backends.py +++ /dev/null @@ -1,11 +0,0 @@ -from elasticsearch_dsl import A -from rest_framework.filters import BaseFilterBackend - -class AggregationsBackend(BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - s = queryset - - a = A("sum", field="total_forecast_budget") - s.aggs.metric("myval", "sum", field="total_forecast_budget") - return queryset - diff --git a/search_indexes/documents.py b/search_indexes/documents.py deleted file mode 100644 index 7843c29b6..000000000 --- a/search_indexes/documents.py +++ /dev/null @@ -1,79 +0,0 @@ -from django_elasticsearch_dsl import Document, fields -from django_elasticsearch_dsl.registries import registry -from infrastructure.models import Project - -budget_phase = fields.ObjectField(properties={ - "code": fields.TextField(), - "name": fields.TextField() -}) - -financial_year = fields.ObjectField(properties={ - "budget_year": fields.TextField(), -}) - - -@registry.register_document -class ProjectDocument(Document): - - #geography = fields.ObjectField(properties={ - # "name": fields.TextField(), - # "province_name": fields.TextField(), - #}) - - #expenditure = fields.NestedField(properties={ - # "budget_phase": budget_phase, - # "financial_year": financial_year, - # "amount": fields.DoubleField(), - #}) - - - project_description = fields.KeywordField() - project_type = fields.KeywordField() - function = fields.KeywordField() - province = fields.KeywordField() - municipality = fields.KeywordField() - - # TODO year currently hardcoded - need to figure out the best way to handle this - total_forecast_budget = fields.DoubleField() - def prepare_total_forecast_budget(self, instance): - - qs = instance.expenditure.filter(financial_year__budget_year="2019/2020") - if qs.count() > 0: - return qs.first().amount - return 0 - - def prepare_province(self, instance): - return instance.geography.province_name - - def prepare_municipality(self, instance): - return instance.geography.name - - def prepare_project_type(self, instance): - return instance.project_type - - def get_queryset(self): - return (super(ProjectDocument, self) - .get_queryset() - .select_related("geography") - .prefetch_related("expenditure__budget_year", "expenditure__financial_Year") - ) - - class Index: - name = "projects" - settings = {"number_of_shards": 1, - "number_of_replicas": 0} - - class Django: - model = Project - - fields = [ - "project_number", - "mtsf_service_outcome", - "iudf", - "own_strategic_objectives", - "asset_class", - "asset_subclass", - "ward_location", - "longitude", - "latitude", - ] diff --git a/search_indexes/search.py b/search_indexes/search.py deleted file mode 100644 index 404065423..000000000 --- a/search_indexes/search.py +++ /dev/null @@ -1,15 +0,0 @@ -#from elasticsearch_dsl import FacetedSearch, TermsFacet, A -# -#class ProjectSearch(FacetedSearch): -# fields = ["project_description", "project_number", "mtsf_service_outcome", "iudf", "own_strategic_objectives", "asset_class", "asset_subclass"] -# facets = { -# "type": TermsFacet(field="project_type"), -# "function": TermsFacet(field="function"), -# "province": TermsFacet(field="province"), -# "municipality": TermsFacet(field="municipality"), -# "municipality_budget": TermsFacet(field="municipality", metric=(A("sum", field="total_forecast_budget"))), -# } -# -# def scan(self): -# s = super().search() -# return self._s.scan() diff --git a/search_indexes/serializers.py b/search_indexes/serializers.py deleted file mode 100644 index 019038d10..000000000 --- a/search_indexes/serializers.py +++ /dev/null @@ -1,33 +0,0 @@ -import json - -from rest_framework import serializers -from django_elasticsearch_dsl_drf.serializers import DocumentSerializer - -from .documents import ProjectDocument - -class ProjectDocumentSerializer(DocumentSerializer): - id = serializers.SerializerMethodField() - - def get_id(self, obj): - return int(obj.meta.id) - - class Meta(object): - - document = ProjectDocument - fields = ( - "project_description", - "project_number", - "function", - "project_type", - "province", - "municipality", - "mtsf_service_outcome", - "iudf", - "own_strategic_objectives", - "asset_class", - "asset_subclass", - "ward_location", - "total_forecast_budget", - "longitude", - "latitude", - ) diff --git a/search_indexes/signals.py b/search_indexes/signals.py deleted file mode 100644 index e14179eef..000000000 --- a/search_indexes/signals.py +++ /dev/null @@ -1 +0,0 @@ -# TODO To be implemented diff --git a/search_indexes/urls.py b/search_indexes/urls.py deleted file mode 100644 index 55118b9e2..000000000 --- a/search_indexes/urls.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.conf.urls import url, include -from rest_framework.routers import DefaultRouter - -from .viewsets import ProjectDocumentView -from . import views - - -router = DefaultRouter() -projects = router.register(r"projects", - ProjectDocumentView, - basename="projectdocument" -) - -urlpatterns = [ - url(r'^', include(router.urls)), - url(r'new_search/', views.ProjectView.as_view()), -] diff --git a/search_indexes/views.py b/search_indexes/views.py deleted file mode 100644 index 766af21ab..000000000 --- a/search_indexes/views.py +++ /dev/null @@ -1,82 +0,0 @@ -from django.db.models import Count, Sum -from django.db.models import F -from django.contrib.postgres.search import SearchQuery -from rest_framework.decorators import api_view -from rest_framework.response import Response -from rest_framework import generics -from rest_framework.pagination import PageNumberPagination - -from infrastructure import models, serializers - -class ProjectView(generics.ListCreateAPIView): - queryset = models.Project.objects.all() - serializer_class = serializers.ProjectSerializer - pagination_class = PageNumberPagination - fieldmap = { - "municipality": "geography__name", - "function": "function", - "type": "project_type", - "province": "geography__province_name" - } - - def list(self, request): - search_query = request.GET.get("q", "") - - queryset = self.get_queryset() - queryset = self.add_filters(queryset, request.GET) - queryset = self.text_search(queryset, search_query) - facets = self.get_facets(queryset) - aggregations = self.aggregations(queryset, request.GET) - - queryset = self.paginate_queryset(queryset) - serializer_class = self.get_serializer_class() - serializer = serializer_class(queryset, many=True) - data = { - "projects": serializer.data, - "facets": facets, - "aggregations": aggregations - } - - return self.get_paginated_response(data) - - def text_search(self, qs, text): - if len(text) == 0: - return qs - - return qs.filter(content_search=SearchQuery(text)) - - def aggregations(self, qs, params): - # TODO - not sure where to put these magic values - financial_year = params.get("financial_year", "2017/2018") - budget_phase = params.get("budget_phase", "Audited Outcome") - - return { - "total": qs.total_value(financial_year, budget_phase) - - } - - def add_filters(self, qs, params): - query_dict = {} - for k, v in ProjectView.fieldmap.items(): - if k in params: - query_dict[v] = params[k] - - return qs.filter(**query_dict) - - def get_facets(self, qs): - def facet_query(field): - field_name = F(field) - return qs.values(key=F(field)).annotate(count=Count(field)) - - - facet_muni = facet_query("geography__name") - facet_type = facet_query("project_type") - facet_function = facet_query("function") - facet_province = facet_query("geography__province_name") - js = { - "municipality": facet_muni, - "type": facet_type, - "function": facet_function, - "province": facet_province - } - return js diff --git a/search_indexes/viewsets.py b/search_indexes/viewsets.py deleted file mode 100644 index dc563a999..000000000 --- a/search_indexes/viewsets.py +++ /dev/null @@ -1,165 +0,0 @@ -from django_elasticsearch_dsl_drf.constants import ( - LOOKUP_FILTER_TERMS, - LOOKUP_FILTER_RANGE, - LOOKUP_FILTER_PREFIX, - LOOKUP_FILTER_WILDCARD, - LOOKUP_QUERY_IN, - LOOKUP_QUERY_GT, - LOOKUP_QUERY_GTE, - LOOKUP_QUERY_LT, - LOOKUP_QUERY_LTE, - LOOKUP_QUERY_EXCLUDE, -) - -from django_elasticsearch_dsl_drf.filter_backends import ( - FilteringFilterBackend, - IdsFilterBackend, - OrderingFilterBackend, - DefaultOrderingFilterBackend, - CompoundSearchFilterBackend, - FacetedSearchFilterBackend, - aggregations, -) - -from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet -from django_elasticsearch_dsl_drf.viewsets import DocumentViewSet -from django_elasticsearch_dsl_drf.pagination import PageNumberPagination -from elasticsearch_dsl import TermsFacet -from elasticsearch_dsl.faceted_search import Facet - -from .documents import ProjectDocument -from .serializers import ProjectDocumentSerializer - -from .backends import AggregationsBackend - -class MunicipalBudgetFacet(TermsFacet): - def get_aggregation(self): - agg = super(MunicipalBudgetFacet, self).get_aggregation() - - - """ - Return the aggregation object. - """ - agg = A(self.agg_type, **self._params) - if self._metric: - agg.metric('metric', self._metric) - return agg - -class SumFacet(Facet): - agg_type = 'sum' - -from rest_framework.decorators import action -class ProjectDocumentView(DocumentViewSet): - - document = ProjectDocument - serializer_class = ProjectDocumentSerializer - pagination_class = PageNumberPagination - lookup_field = "id" - - filter_backends = [ - FilteringFilterBackend, - IdsFilterBackend, - OrderingFilterBackend, - DefaultOrderingFilterBackend, - CompoundSearchFilterBackend, - FacetedSearchFilterBackend, - AggregationsBackend, - ] - - # Define search fields - search_fields = ( - #"id", - "project_description", - "project_number", - "function", - "project_type", - "province", - "municipality", - "mtsf_service_outcome", - "iudf", - "own_strategic_objectives", - "asset_class", - "asset_subclass", - ) - - # Define filter fields - filter_fields = { - "id": { - "field": "id", - # Note, that we limit the lookups of id field in this example, - # to `range`, `in`, `gt`, `gte`, `lt` and `lte` filters. - "lookups": [ - LOOKUP_FILTER_RANGE, - LOOKUP_QUERY_IN, - LOOKUP_QUERY_GT, - LOOKUP_QUERY_GTE, - LOOKUP_QUERY_LT, - LOOKUP_QUERY_LTE, - ], - }, - "function": "function", - "project_type": "project_type", - "province": "province", - "municipality": "municipality", - "pages": { - "field": "pages", - # Note, that we limit the lookups of `pages` field in this - # example, to `range`, `gt`, `gte`, `lt` and `lte` filters. - "lookups": [ - LOOKUP_FILTER_RANGE, - LOOKUP_QUERY_GT, - LOOKUP_QUERY_GTE, - LOOKUP_QUERY_LT, - LOOKUP_QUERY_LTE, - ], - }, - } - - faceted_search_fields = { - #"id": { - # "field": "id", - #}, - "province": { - "field": "province", - #"facet": TermsFacet, - "enabled": True, - }, - "municipality": { - "field": "municipality", - "facet": TermsFacet, - "enabled": True, - "options": { - "size" : 300 - } - }, - "functions": { - "field": "function", - "facet": TermsFacet, - "enabled": True, - "options": { - "size" : 300 - } - }, - "project_type": { - "field": "project_type", - "facet": TermsFacet, - "enabled": True - }, - "total_budget": { - "field": "total_forecast_budget", - "facet": SumFacet, - "enabled": True - }, - } - - ## Define ordering fields - ordering_fields = { - # "id": "id", - "total_forecast_budget": "total_forecast_budget", - "province": "province", - "project_description": "project_description", - "function": "function", - "project_type": "project_type", - } - ## Specify default ordering - ordering = ("-total_forecast_budget",) From 88f35b1bac442dd03620e314031be20c38833f6e Mon Sep 17 00:00:00 2001 From: JD Bothma Date: Thu, 3 Dec 2020 20:10:40 +0200 Subject: [PATCH 6/8] Basic pdf download test --- .../fixtures/tests/test_profile_and_pdf.json | 1856 +++++++++++++++++ scorecard/tests/test_pdf.py | 10 + 2 files changed, 1866 insertions(+) create mode 100644 scorecard/fixtures/tests/test_profile_and_pdf.json create mode 100644 scorecard/tests/test_pdf.py diff --git a/scorecard/fixtures/tests/test_profile_and_pdf.json b/scorecard/fixtures/tests/test_profile_and_pdf.json new file mode 100644 index 000000000..8117427e6 --- /dev/null +++ b/scorecard/fixtures/tests/test_profile_and_pdf.json @@ -0,0 +1,1856 @@ +[ +{ + "model": "scorecard.geography", + "pk": 154, + "fields": { + "geo_level": "municipality", + "geo_code": "BUF", + "name": "Buffalo City", + "long_name": "Buffalo City, Eastern Cape", + "square_kms": 2751.69154949282, + "parent_level": "province", + "parent_code": "EC", + "province_name": "Eastern Cape", + "province_code": "EC", + "category": "A", + "miif_category": "A", + "population": 781026, + "postal_address_1": "P O BOX 134", + "postal_address_2": "EAST LONDON", + "postal_address_3": "5200", + "street_address_1": "Trust Bank Centre", + "street_address_2": "C/O Oxford & North Street", + "street_address_3": "East London", + "street_address_4": "5200", + "phone_number": "043 705 2000", + "fax_number": "043 743 8568", + "url": "http://www.buffalocity.gov.za" + } +}, +{ + "model": "municipal_finance.municipalityprofile", + "pk": "BUF", + "fields": { + "data": { + "indicators": { + "wasteful_exp": { + "ref": { + "url": "http://mfma.treasury.gov.za/Circulars/Pages/Circular71.aspx", + "title": "Circular 71" + }, + "values": [ + { + "date": 2019, + "rating": "bad", + "result": 2.74 + }, + { + "date": 2018, + "rating": "bad", + "result": 6.16 + }, + { + "date": 2017, + "rating": "bad", + "result": 12.16 + }, + { + "date": 2016, + "rating": "bad", + "result": 7.11 + } + ], + "result_type": "%" + }, + "cash_coverage": { + "ref": { + "url": "http://mfma.treasury.gov.za/Media_Releases/The%20state%20of%20local%20government%20finances/Pages/default.aspx", + "title": "State of Local Government Finances" + }, + "values": [ + { + "date": 2019, + "rating": "ave", + "result": 2.1 + }, + { + "date": 2018, + "rating": "good", + "result": 3.6 + }, + { + "date": 2017, + "rating": "good", + "result": 3.6 + }, + { + "date": 2016, + "rating": "good", + "result": 5.2 + } + ], + "result_type": "months" + }, + "current_ratio": { + "ref": { + "url": "http://mfma.treasury.gov.za/Circulars/Pages/Circular71.aspx", + "title": "Circular 71" + }, + "values": [ + { + "date": 2019, + "year": 2019, + "assets": 2766495585.0, + "rating": "good", + "result": 1.61, + "amount_type": "AUDA", + "liabilities": 1714855209.0 + }, + { + "date": 2018, + "year": 2018, + "assets": 3119778100.0, + "rating": "good", + "result": 1.79, + "amount_type": "AUDA", + "liabilities": 1741644859.0 + }, + { + "date": 2017, + "year": 2017, + "assets": 2995989615.0, + "rating": "good", + "result": 2.29, + "amount_type": "AUDA", + "liabilities": 1309999921.0 + }, + { + "date": 2016, + "year": 2016, + "assets": 3665738322.0, + "rating": "good", + "result": 2.31, + "amount_type": "AUDA", + "liabilities": 1588602305.0 + } + ], + "result_type": "ratio" + }, + "op_budget_diff": { + "ref": { + "url": "http://mfma.treasury.gov.za/Media_Releases/Reports%20to%20Parliament/Pages/default.aspx", + "title": "Over and under spending reports to parliament" + }, + "values": [ + { + "date": 2019, + "rating": "good", + "result": 4.4, + "overunder": "over" + }, + { + "date": 2018, + "rating": "good", + "result": 2.1, + "overunder": "over" + }, + { + "date": 2017, + "rating": "ave", + "result": -6.0, + "overunder": "under" + }, + { + "date": 2016, + "rating": "good", + "result": -3.0, + "overunder": "under" + } + ], + "result_type": "%" + }, + "cap_budget_diff": { + "ref": { + "url": "http://mfma.treasury.gov.za/Media_Releases/Reports%20to%20Parliament/Pages/default.aspx", + "title": "Over and under spending reports to parliament" + }, + "values": [ + { + "date": 2019, + "rating": "bad", + "result": -16.15, + "overunder": "under" + }, + { + "date": 2018, + "rating": "bad", + "result": -24.95, + "overunder": "under" + }, + { + "date": 2017, + "rating": "ave", + "result": -13.91, + "overunder": "under" + }, + { + "date": 2016, + "rating": "ave", + "result": -14.28, + "overunder": "under" + } + ], + "result_type": "%" + }, + "liquidity_ratio": { + "ref": { + "url": "http://mfma.treasury.gov.za/RegulationsandGazettes/Municipal%20Budget%20and%20Reporting%20Regulations/Pages/default.aspx", + "title": "Municipal Budget and Reporting Regulations" + }, + "values": [ + { + "cash": 247013522.0, + "date": "2019", + "year": 2019, + "rating": "bad", + "result": 0.68, + "amount_type": "ACT", + "liabilities": 1714855209.0, + "call_investment_deposits": 924619393.0 + }, + { + "cash": 165103636.0, + "date": "2018", + "year": 2018, + "rating": "good", + "result": 1.05, + "amount_type": "ACT", + "liabilities": 1741644859.0, + "call_investment_deposits": 1660392952.0 + }, + { + "cash": 24591070.0, + "date": "2017", + "year": 2017, + "rating": "good", + "result": 1.29, + "amount_type": "ACT", + "liabilities": 1309999921.0, + "call_investment_deposits": 1665510900.0 + }, + { + "cash": 222736132.0, + "date": "2016", + "year": 2016, + "rating": "good", + "result": 1.49, + "amount_type": "ACT", + "liabilities": 1588602305.0, + "call_investment_deposits": 2151164102.0 + } + ], + "result_type": "ratio" + }, + "revenue_sources": { + "ref": { + "url": "http://mfma.treasury.gov.za/Media_Releases/LGESDiscussions/Pages/default.aspx", + "title": "Local Government Equitable Share" + }, + "year": 2019, + "local": { + "items": [ + { + "item.code": "0200", + "amount.sum": 1405020223.0, + "item.label": "Property Rates", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "0300", + "amount.sum": 0.0, + "item.label": "Property Rates - Penalties And Collection Charges", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "0400", + "amount.sum": 2823912533.0, + "item.label": "Service Charges", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "0700", + "amount.sum": 20704445.0, + "item.label": "Rent Of Facilities And Equipment", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "0800", + "amount.sum": 98690423.0, + "item.label": "Interest Earned - External Investments", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "1000", + "amount.sum": 67093405.0, + "item.label": "Interest Earned - Outstanding Debtors", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "1100", + "amount.sum": 0.0, + "item.label": "Dividends Received", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "1300", + "amount.sum": 24938282.0, + "item.label": "Fines", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "1400", + "amount.sum": 14300355.0, + "item.label": "Licenses and Permits", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "1500", + "amount.sum": 26198150.0, + "item.label": "Agency Services", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "1700", + "amount.sum": 746927378.0, + "item.label": "Other Revenue", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "1800", + "amount.sum": 0.0, + "item.label": "Gain On Disposal Of Property, Plant & Equipment", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + } + ], + "amount": 5227785194.0, + "percent": 73.14 + }, + "rating": "ave", + "government": { + "items": [ + { + "item.code": "1600", + "amount.sum": 1025375160.0, + "item.label": "Transfers Recognised - Operating", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + }, + { + "item.code": "1610", + "amount.sum": 894057135.0, + "item.label": "Transfers Recognised - Capital", + "amount_type.code": "AUDA", + "financial_year_end.year": 2019 + } + ], + "amount": 1919432295.0, + "percent": 26.86 + } + }, + "cash_at_year_end": { + "ref": { + "url": "http://mfma.treasury.gov.za/Media_Releases/The%20state%20of%20local%20government%20finances/Pages/default.aspx", + "title": "State of Local Government Finances" + }, + "values": [ + { + "date": 2019, + "rating": "good", + "result": 1171632912.0 + }, + { + "date": 2018, + "rating": "good", + "result": 1825496588.0 + }, + { + "date": 2017, + "rating": "good", + "result": 1690101970.0 + }, + { + "date": 2016, + "rating": "good", + "result": 2373900195.0 + } + ], + "result_type": "R" + }, + "revenue_breakdown": { + "values": [ + { + "date": "2019", + "item": "Property rates", + "amount": 1405020223.0, + "percent": 19.66, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Service Charges", + "amount": 2823912533.0, + "percent": 39.51, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Rental income", + "amount": 20704445.0, + "percent": 0.29, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Interest and investments", + "amount": 165783828.0, + "percent": 2.32, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Fines", + "amount": 24938282.0, + "percent": 0.35, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Licenses and Permits", + "amount": 14300355.0, + "percent": 0.2, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Agency services", + "amount": 26198150.0, + "percent": 0.37, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Government Transfers for Operating Expenses", + "amount": 1025375160.0, + "percent": 14.35, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Government Transfers for Capital Expenses", + "amount": 894057135.0, + "percent": 12.51, + "amount_type": "AUDA" + }, + { + "date": "2019", + "item": "Other", + "amount": 746927378.0, + "percent": 10.45, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Property rates", + "amount": 1006114406.0, + "percent": 15.95, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Service Charges", + "amount": 2576078780.0, + "percent": 40.83, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Rental income", + "amount": 20067719.0, + "percent": 0.32, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Interest and investments", + "amount": 176012022.0, + "percent": 2.79, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Fines", + "amount": 23698183.0, + "percent": 0.38, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Licenses and Permits", + "amount": 14249685.0, + "percent": 0.23, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Agency services", + "amount": 25682604.0, + "percent": 0.41, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Government Transfers for Operating Expenses", + "amount": 817569465.0, + "percent": 12.96, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Government Transfers for Capital Expenses", + "amount": 930587543.0, + "percent": 14.75, + "amount_type": "AUDA" + }, + { + "date": "2018", + "item": "Other", + "amount": 719837903.0, + "percent": 11.41, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Property rates", + "amount": 957618439.0, + "percent": 15.84, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Service Charges", + "amount": 2593541588.0, + "percent": 42.89, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Rental income", + "amount": 16424005.0, + "percent": 0.27, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Interest and investments", + "amount": 198436890.0, + "percent": 3.28, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Fines", + "amount": 16895710.0, + "percent": 0.28, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Licenses and Permits", + "amount": 14225199.0, + "percent": 0.24, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Agency services", + "amount": 0.0, + "percent": 0, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Government Transfers for Operating Expenses", + "amount": 1304827290.0, + "percent": 21.58, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Government Transfers for Capital Expenses", + "amount": 669780334.0, + "percent": 11.08, + "amount_type": "AUDA" + }, + { + "date": "2017", + "item": "Other", + "amount": 274544055.0, + "percent": 4.54, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Property rates", + "amount": 906093625.0, + "percent": 14.62, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Service Charges", + "amount": 2758688507.0, + "percent": 44.52, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Rental income", + "amount": 16583410.0, + "percent": 0.27, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Interest and investments", + "amount": 187367781.0, + "percent": 3.02, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Fines", + "amount": 5593754.0, + "percent": 0.09, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Licenses and Permits", + "amount": 12611826.0, + "percent": 0.2, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Agency services", + "amount": 0.0, + "percent": 0, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Government Transfers for Operating Expenses", + "amount": 1334131275.0, + "percent": 21.53, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Government Transfers for Capital Expenses", + "amount": 670393964.0, + "percent": 10.82, + "amount_type": "AUDA" + }, + { + "date": "2016", + "item": "Other", + "amount": 305253324.0, + "percent": 4.93, + "amount_type": "AUDA" + }, + { + "date": "2020 budget", + "item": null, + "amount": null, + "percent": null, + "amount_type": "ORGB" + } + ] + }, + "rep_maint_perc_ppe": { + "ref": { + "url": "http://mfma.treasury.gov.za/Circulars/Pages/Circular71.aspx", + "title": "Circular 71" + }, + "values": [ + { + "date": 2019, + "rating": "bad", + "result": 1.95 + }, + { + "date": 2018, + "rating": "bad", + "result": 1.91 + }, + { + "date": 2017, + "rating": "bad", + "result": 2.35 + }, + { + "date": 2016, + "rating": "bad", + "result": 2.58 + } + ], + "result_type": "%" + }, + "expenditure_trends_staff": { + "values": [ + { + "date": 2019, + "rating": "", + "result": 29.94 + }, + { + "date": 2018, + "rating": "", + "result": 30.9 + }, + { + "date": 2017, + "rating": "", + "result": 29.12 + }, + { + "date": 2016, + "rating": "", + "result": 25.57 + } + ], + "result_type": "%" + }, + "expenditure_trends_contracting": { + "values": [ + { + "date": 2019, + "rating": "", + "result": 0.09 + }, + { + "date": 2018, + "rating": "", + "result": 0.05 + }, + { + "date": 2017, + "rating": "", + "result": 0.02 + }, + { + "date": 2016, + "rating": "", + "result": 0.0 + } + ], + "result_type": "%" + }, + "current_debtors_collection_rate": { + "ref": { + "url": "http://mfma.treasury.gov.za/RegulationsandGazettes/Municipal%20Budget%20and%20Reporting%20Regulations/Pages/default.aspx", + "title": "Municipal Budget and Reporting Regulations" + }, + "values": [ + { + "date": "2019", + "year": 2019, + "rating": "good", + "result": 100.0, + "amount_type": "AUDA" + }, + { + "date": "2018", + "year": 2018, + "rating": "good", + "result": 100.0, + "amount_type": "AUDA" + }, + { + "date": "2017", + "year": 2017, + "rating": "good", + "result": 100.0, + "amount_type": "AUDA" + }, + { + "date": "2016", + "year": 2016, + "rating": "good", + "result": 100.0, + "amount_type": "AUDA" + } + ], + "result_type": "%" + }, + "expenditure_functional_breakdown": { + "values": [ + { + "date": "2016", + "item": "Community & Social Services", + "amount": 131367526.0, + "percent": 2.38 + }, + { + "date": "2016", + "item": "Electricity ", + "amount": 1584720583.0, + "percent": 28.72 + }, + { + "date": "2016", + "item": "Environmental Protection", + "amount": 107629270.0, + "percent": 1.95 + }, + { + "date": "2016", + "item": "Governance, Administration, Planning and Development", + "amount": 1157894137.0, + "percent": 20.98 + }, + { + "date": "2016", + "item": "Health", + "amount": 30925459.0, + "percent": 0.56 + }, + { + "date": "2016", + "item": "Housing", + "amount": 216632801.0, + "percent": 3.93 + }, + { + "date": "2016", + "item": "Other", + "amount": 15864509.0, + "percent": 0.29 + }, + { + "date": "2016", + "item": "Public Safety", + "amount": 281241920.0, + "percent": 5.1 + }, + { + "date": "2016", + "item": "Road Transport", + "amount": 552458354.0, + "percent": 10.01 + }, + { + "date": "2016", + "item": "Sport And Recreation", + "amount": 74197499.0, + "percent": 1.34 + }, + { + "date": "2016", + "item": "Waste Management", + "amount": 334139398.0, + "percent": 6.06 + }, + { + "date": "2016", + "item": "Waste Water Management", + "amount": 376260078.0, + "percent": 6.82 + }, + { + "date": "2016", + "item": "Water", + "amount": 654617060.0, + "percent": 11.86 + }, + { + "date": "2017", + "item": "Community & Social Services", + "amount": 116398218.0, + "percent": 2.08 + }, + { + "date": "2017", + "item": "Electricity ", + "amount": 1666907223.0, + "percent": 29.83 + }, + { + "date": "2017", + "item": "Environmental Protection", + "amount": 121351704.0, + "percent": 2.17 + }, + { + "date": "2017", + "item": "Governance, Administration, Planning and Development", + "amount": 1315680257.0, + "percent": 23.55 + }, + { + "date": "2017", + "item": "Health", + "amount": 33294971.0, + "percent": 0.6 + }, + { + "date": "2017", + "item": "Housing", + "amount": 171503113.0, + "percent": 3.07 + }, + { + "date": "2017", + "item": "Other", + "amount": 15816005.0, + "percent": 0.28 + }, + { + "date": "2017", + "item": "Public Safety", + "amount": 302303547.0, + "percent": 5.41 + }, + { + "date": "2017", + "item": "Road Transport", + "amount": 543621626.0, + "percent": 9.73 + }, + { + "date": "2017", + "item": "Sport And Recreation", + "amount": 70286521.0, + "percent": 1.26 + }, + { + "date": "2017", + "item": "Waste Management", + "amount": 302401528.0, + "percent": 5.41 + }, + { + "date": "2017", + "item": "Waste Water Management", + "amount": 326507893.0, + "percent": 5.84 + }, + { + "date": "2017", + "item": "Water", + "amount": 601544259.0, + "percent": 10.77 + }, + { + "date": "2018", + "item": "Community & Social Services", + "amount": 99349812.0, + "percent": 1.64 + }, + { + "date": "2018", + "item": "Electricity ", + "amount": 1854018426.0, + "percent": 30.52 + }, + { + "date": "2018", + "item": "Environmental Protection", + "amount": 23069203.0, + "percent": 0.38 + }, + { + "date": "2018", + "item": "Governance, Administration, Planning and Development", + "amount": 1511189246.0, + "percent": 24.88 + }, + { + "date": "2018", + "item": "Health", + "amount": 36345730.0, + "percent": 0.6 + }, + { + "date": "2018", + "item": "Housing", + "amount": 105092324.0, + "percent": 1.73 + }, + { + "date": "2018", + "item": "Other", + "amount": 80109352.0, + "percent": 1.32 + }, + { + "date": "2018", + "item": "Public Safety", + "amount": 86797609.0, + "percent": 1.43 + }, + { + "date": "2018", + "item": "Road Transport", + "amount": 738605124.0, + "percent": 12.16 + }, + { + "date": "2018", + "item": "Sport And Recreation", + "amount": 290312610.0, + "percent": 4.78 + }, + { + "date": "2018", + "item": "Waste Management", + "amount": 322768854.0, + "percent": 5.31 + }, + { + "date": "2018", + "item": "Waste Water Management", + "amount": 303304887.0, + "percent": 4.99 + }, + { + "date": "2018", + "item": "Water", + "amount": 623452817.0, + "percent": 10.26 + }, + { + "date": "2019", + "item": "Community & Social Services", + "amount": 116150339.0, + "percent": 1.7 + }, + { + "date": "2019", + "item": "Electricity ", + "amount": 2027332017.0, + "percent": 29.61 + }, + { + "date": "2019", + "item": "Environmental Protection", + "amount": 24492542.0, + "percent": 0.36 + }, + { + "date": "2019", + "item": "Governance, Administration, Planning and Development", + "amount": 1599450956.0, + "percent": 23.36 + }, + { + "date": "2019", + "item": "Health", + "amount": 41263053.0, + "percent": 0.6 + }, + { + "date": "2019", + "item": "Housing", + "amount": 57396554.0, + "percent": 0.84 + }, + { + "date": "2019", + "item": "Other", + "amount": 86862442.0, + "percent": 1.27 + }, + { + "date": "2019", + "item": "Public Safety", + "amount": 480926026.0, + "percent": 7.02 + }, + { + "date": "2019", + "item": "Road Transport", + "amount": 803260389.0, + "percent": 11.73 + }, + { + "date": "2019", + "item": "Sport And Recreation", + "amount": 317602546.0, + "percent": 4.64 + }, + { + "date": "2019", + "item": "Waste Management", + "amount": 390106815.0, + "percent": 5.7 + }, + { + "date": "2019", + "item": "Waste Water Management", + "amount": 301628432.0, + "percent": 4.41 + }, + { + "date": "2019", + "item": "Water", + "amount": 599866525.0, + "percent": 8.76 + } + ] + } + }, + "demarcation": { + "land_lost": [], + "land_gained": [] + }, + "muni_contact": { + "url": "http://www.buffalocity.gov.za", + "phone_number": "043 705 2000", + "street_address_1": "Trust Bank Centre", + "street_address_2": "C/O Oxford & North Street", + "street_address_3": "East London", + "street_address_4": "5200" + }, + "mayoral_staff": { + "officials": [ + { + "name": "Xola Pakati", + "role": "Mayor/Executive Mayor", + "email": "execmayor@buffalocity.gov.za", + "title": "Mr", + "secretary": { + "name": "Philasande Pula", + "role": "Secretary of Mayor/Executive Mayor", + "email": "philasandep@buffalocity.gov.za", + "title": "Ms", + "fax_number": "043 743 9040", + "office_phone": "043 705 1072" + }, + "fax_number": "043 743 9040", + "office_phone": "043 705 1901" + }, + { + "name": "Andile Sihlahla", + "role": "Municipal Manager", + "email": "andiles@buffalocity.gov.za", + "title": "Mr", + "secretary": { + "name": "Phindiswa Sululu", + "role": "Secretary of Municipal Manager", + "email": "phindiswas@buffalocity.gov.za", + "title": "Ms", + "fax_number": "043 722 6126", + "office_phone": "043 705 1901" + }, + "fax_number": "043 722 6126", + "office_phone": "043 705 1046" + }, + { + "name": "Zoliswa Patience Matana", + "role": "Deputy Mayor/Executive Mayor", + "email": "zolswam@buffalocity.gov.za", + "title": "Ms", + "secretary": { + "name": "Princess Feni", + "role": "Secretary of Deputy Mayor/Executive Mayor", + "email": "princessf@buffalocity.gov.za", + "title": "Ms", + "fax_number": "086 545 1288", + "office_phone": "043 705 2899" + }, + "fax_number": "086 545 1288", + "office_phone": "043 705 2899" + }, + { + "name": "Ntsikelelo Sigcau", + "role": "Chief Financial Officer", + "email": "ntsikelelos@buffalocity.gov.za", + "title": "Mr", + "secretary": { + "name": "Candice Bahlmann", + "role": "Secretary of Financial Manager", + "email": "candiceb@buffalocity.gov.za", + "title": "Ms", + "fax_number": "043 743 9141", + "office_phone": "043 705 1887" + }, + "fax_number": "043 742 2443", + "office_phone": "043 705 3329" + } + ], + "updated_date": "October 2020" + }, + "audit_opinions": { + "values": [ + { + "date": 2019, + "rating": "qualified", + "result": "Qualified", + "report_url": "http://mfma.treasury.gov.za/Documents/07.%20Audit%20Reports/2018-19/01.%20Metros/BUF%20Buffalo%20City/BUF%20Buffalo%20City%20Audit%20report%202018-19.pdf" + }, + { + "date": 2018, + "rating": "qualified", + "result": "Qualified", + "report_url": "http://mfma.treasury.gov.za/Documents/07.%20Audit%20Reports/2017-18/01.%20Metros/BUF%20Buffalo%20City/BUF%20Buffalo%20City%20Consolidated%20Audit%20report%202017-18.pdf" + }, + { + "date": 2017, + "rating": "unqualified_emphasis_of_matter", + "result": "Unqualified - Emphasis of Matter items", + "report_url": "http://mfma.treasury.gov.za/Documents/07.%20Audit%20Reports/2016-17/01.%20Metros/BUF%20Buffalo%20City/BUF%20Buffalo%20City%20Audit%20report%202016-17.pdf" + }, + { + "date": 2016, + "rating": "qualified", + "result": "Qualified", + "report_url": "http://mfma.treasury.gov.za/Documents/07.%20Audit%20Reports/2015-16/01.%20Metros/BUF%20Buffalo%20City" + } + ] + } + } + } +}, +{ + "model": "municipal_finance.mediangroup", + "pk": "national", + "fields": { + "data": { + "wasteful_exp": { + "A": { + "2016": 8.31, + "2017": 10.625, + "2018": 7.8950000000000005, + "2019": 6.66 + } + }, + "cash_coverage": { + "A": { + "2016": 2.85, + "2017": 2.3, + "2018": 2.3, + "2019": 1.6 + } + }, + "current_ratio": { + "A": { + "2016": 1.5, + "2017": 1.57, + "2018": 1.305, + "2019": 1.295 + } + }, + "op_budget_diff": { + "A": { + "2016": 0.25, + "2017": -5.2, + "2018": -0.09999999999999987, + "2019": 1.4000000000000001 + } + }, + "cap_budget_diff": { + "A": { + "2016": -7.475, + "2017": -21.595, + "2018": -21.555, + "2019": -17.15 + } + }, + "liquidity_ratio": { + "A": { + "2016": 0.81, + "2017": 0.75, + "2018": 0.65, + "2019": 0.505 + } + }, + "cash_at_year_end": { + "A": { + "2016": 1779320159.5, + "2017": 1929708886.5, + "2018": 2109104675.0, + "2019": 2137349494.5 + } + }, + "rep_maint_perc_ppe": { + "A": { + "2016": 3.63, + "2017": 2.5949999999999998, + "2018": 2.245, + "2019": 2.445 + } + }, + "expenditure_trends_staff": { + "A": { + "2016": 26.735, + "2017": 29.245, + "2018": 29.07, + "2019": 28.995 + } + }, + "expenditure_trends_contracting": { + "A": { + "2016": 6.015, + "2017": 5.205, + "2018": 5.1000000000000005, + "2019": 5.035 + } + }, + "current_debtors_collection_rate": { + "A": { + "2016": 98.945, + "2017": 94.735, + "2018": 95.825, + "2019": 97.7 + } + } + } + } +}, +{ + "model": "municipal_finance.mediangroup", + "pk": "provincial", + "fields": { + "data": { + "wasteful_exp": { + "EC": { + "A": { + "2016": 7.11, + "2017": 12.16, + "2018": 6.16, + "2019": 2.74 + } + } + }, + "cash_coverage": { + "EC": { + "A": { + "2016": 5.2, + "2017": 3.6, + "2018": 3.6, + "2019": 2.1 + } + } + }, + "current_ratio": { + "EC": { + "A": { + "2016": 2.31, + "2017": 2.29, + "2018": 1.79, + "2019": 1.61 + } + } + }, + "op_budget_diff": { + "EC": { + "A": { + "2016": -3.0, + "2017": -6.0, + "2018": 2.1, + "2019": 4.4 + } + } + }, + "cap_budget_diff": { + "EC": { + "A": { + "2016": -14.28, + "2017": -13.91, + "2018": -24.95, + "2019": -16.15 + } + } + }, + "liquidity_ratio": { + "EC": { + "A": { + "2016": 1.49, + "2017": 1.29, + "2018": 1.05, + "2019": 0.68 + } + } + }, + "cash_at_year_end": { + "EC": { + "A": { + "2016": 2373900195.0, + "2017": 1690101970.0, + "2018": 1825496588.0, + "2019": 1171632912.0 + } + } + }, + "rep_maint_perc_ppe": { + "EC": { + "A": { + "2016": 2.58, + "2017": 2.35, + "2018": 1.91, + "2019": 1.95 + } + } + }, + "expenditure_trends_staff": { + "EC": { + "A": { + "2016": 25.57, + "2017": 29.12, + "2018": 30.9, + "2019": 29.94 + } + } + }, + "expenditure_trends_contracting": { + "EC": { + "A": { + "2016": 0.0, + "2017": 0.02, + "2018": 0.05, + "2019": 0.09 + } + } + }, + "current_debtors_collection_rate": { + "EC": { + "A": { + "2016": 100.0, + "2017": 100.0, + "2018": 100.0, + "2019": 100.0 + } + } + } + } + } +}, +{ + "model": "municipal_finance.ratingcountgroup", + "pk": "national", + "fields": { + "data": { + "wasteful_exp": { + "A": { + "2016": { + "bad": 2 + }, + "2017": { + "bad": 2 + }, + "2018": { + "bad": 2 + }, + "2019": { + "bad": 2 + } + } + }, + "cash_coverage": { + "A": { + "2016": { + "bad": 1, + "good": 1 + }, + "2017": { + "bad": 1, + "good": 1 + }, + "2018": { + "bad": 1, + "good": 1 + }, + "2019": { + "ave": 2 + } + } + }, + "current_ratio": { + "A": { + "2016": { + "bad": 1, + "good": 1 + }, + "2017": { + "bad": 1, + "good": 1 + }, + "2018": { + "bad": 1, + "good": 1 + }, + "2019": { + "bad": 1, + "good": 1 + } + } + }, + "op_budget_diff": { + "A": { + "2016": { + "good": 2 + }, + "2017": { + "ave": 1, + "good": 1 + }, + "2018": { + "good": 2 + }, + "2019": { + "good": 2 + } + } + }, + "cap_budget_diff": { + "A": { + "2016": { + "ave": 1, + "good": 1 + }, + "2017": { + "ave": 1, + "bad": 1 + }, + "2018": { + "bad": 2 + }, + "2019": { + "bad": 2 + } + } + }, + "liquidity_ratio": { + "A": { + "2016": { + "bad": 1, + "good": 1 + }, + "2017": { + "bad": 1, + "good": 1 + }, + "2018": { + "bad": 1, + "good": 1 + }, + "2019": { + "bad": 2 + } + } + }, + "cash_at_year_end": { + "A": { + "2016": { + "good": 2 + }, + "2017": { + "good": 2 + }, + "2018": { + "good": 2 + }, + "2019": { + "good": 2 + } + } + }, + "rep_maint_perc_ppe": { + "A": { + "2016": { + "bad": 2 + }, + "2017": { + "bad": 2 + }, + "2018": { + "bad": 2 + }, + "2019": { + "bad": 2 + } + } + }, + "expenditure_trends_staff": { + "A": { + "2016": { + "": 2 + }, + "2017": { + "": 2 + }, + "2018": { + "": 2 + }, + "2019": { + "": 2 + } + } + }, + "expenditure_trends_contracting": { + "A": { + "2016": { + "": 2 + }, + "2017": { + "": 2 + }, + "2018": { + "": 2 + }, + "2019": { + "": 2 + } + } + }, + "current_debtors_collection_rate": { + "A": { + "2016": { + "good": 2 + }, + "2017": { + "bad": 1, + "good": 1 + }, + "2018": { + "bad": 1, + "good": 1 + }, + "2019": { + "good": 2 + } + } + } + } + } +}, +{ + "model": "municipal_finance.ratingcountgroup", + "pk": "provincial", + "fields": { + "data": { + "wasteful_exp": { + "EC": { + "A": { + "2016": { + "bad": 1 + }, + "2017": { + "bad": 1 + }, + "2018": { + "bad": 1 + }, + "2019": { + "bad": 1 + } + } + } + }, + "cash_coverage": { + "EC": { + "A": { + "2016": { + "good": 1 + }, + "2017": { + "good": 1 + }, + "2018": { + "good": 1 + }, + "2019": { + "ave": 1 + } + } + } + }, + "current_ratio": { + "EC": { + "A": { + "2016": { + "good": 1 + }, + "2017": { + "good": 1 + }, + "2018": { + "good": 1 + }, + "2019": { + "good": 1 + } + } + } + }, + "op_budget_diff": { + "EC": { + "A": { + "2016": { + "good": 1 + }, + "2017": { + "ave": 1 + }, + "2018": { + "good": 1 + }, + "2019": { + "good": 1 + } + } + } + }, + "cap_budget_diff": { + "EC": { + "A": { + "2016": { + "ave": 1 + }, + "2017": { + "ave": 1 + }, + "2018": { + "bad": 1 + }, + "2019": { + "bad": 1 + } + } + } + }, + "liquidity_ratio": { + "EC": { + "A": { + "2016": { + "good": 1 + }, + "2017": { + "good": 1 + }, + "2018": { + "good": 1 + }, + "2019": { + "bad": 1 + } + } + } + }, + "cash_at_year_end": { + "EC": { + "A": { + "2016": { + "good": 1 + }, + "2017": { + "good": 1 + }, + "2018": { + "good": 1 + }, + "2019": { + "good": 1 + } + } + } + }, + "rep_maint_perc_ppe": { + "EC": { + "A": { + "2016": { + "bad": 1 + }, + "2017": { + "bad": 1 + }, + "2018": { + "bad": 1 + }, + "2019": { + "bad": 1 + } + } + } + }, + "expenditure_trends_staff": { + "EC": { + "A": { + "2016": { + "": 1 + }, + "2017": { + "": 1 + }, + "2018": { + "": 1 + }, + "2019": { + "": 1 + } + } + } + }, + "expenditure_trends_contracting": { + "EC": { + "A": { + "2016": { + "": 1 + }, + "2017": { + "": 1 + }, + "2018": { + "": 1 + }, + "2019": { + "": 1 + } + } + } + }, + "current_debtors_collection_rate": { + "EC": { + "A": { + "2016": { + "good": 1 + }, + "2017": { + "good": 1 + }, + "2018": { + "good": 1 + }, + "2019": { + "good": 1 + } + } + } + } + } + } +} +] diff --git a/scorecard/tests/test_pdf.py b/scorecard/tests/test_pdf.py new file mode 100644 index 000000000..06a4e1550 --- /dev/null +++ b/scorecard/tests/test_pdf.py @@ -0,0 +1,10 @@ +from django.contrib.staticfiles.testing import StaticLiveServerTestCase +from django.urls import reverse +import io + +class PDFTests(StaticLiveServerTestCase): + fixtures = ['tests/test_profile_and_pdf'] + + def test_pdf_download(self): + response = self.client.get("/profiles/municipality-BUF-buffalo-city/") + f = io.BytesIO(response.content) From 0d775edfbfba2398af487a1091af284be9928dda Mon Sep 17 00:00:00 2001 From: JD Bothma Date: Thu, 3 Dec 2020 22:10:01 +0200 Subject: [PATCH 7/8] WIP adding PDF test but I have to hardcode the url because build_absolute_uri thinks port is 80 --- docker-compose.yml | 2 ++ requirements.txt | 1 + scorecard/tests/test_pdf.py | 16 ++++++++++++++-- scorecard/views.py | 1 + 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index da37afeb0..eaec3cb8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,8 @@ services: - minio stdin_open: true tty: true + extra_hosts: + - "testserver:127.0.0.1" postgres: image: postgres:11.5 diff --git a/requirements.txt b/requirements.txt index 23413a56e..29351d313 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,6 +48,7 @@ psycogreen==1.0 psycopg2==2.7.3.1 ptyprocess==0.5.1 pycrypto==2.6.1 +PyPDF2==1.26.0 pyScss==1.3.7 python-dateutil==2.5.2 pytz==2017.3 diff --git a/scorecard/tests/test_pdf.py b/scorecard/tests/test_pdf.py index 06a4e1550..cd71939f3 100644 --- a/scorecard/tests/test_pdf.py +++ b/scorecard/tests/test_pdf.py @@ -1,10 +1,22 @@ from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.urls import reverse import io +import PyPDF2 +import tempfile + class PDFTests(StaticLiveServerTestCase): fixtures = ['tests/test_profile_and_pdf'] + port = 9000 def test_pdf_download(self): - response = self.client.get("/profiles/municipality-BUF-buffalo-city/") - f = io.BytesIO(response.content) + response = self.client.get(f"{ self.live_server_url }/profiles/municipality-BUF-buffalo-city.pdf") + temp_file = tempfile.NamedTemporaryFile() + temp_file.write(response.content) + temp_file.flush() + temp_file.seek(0) + pdfReader = PyPDF2.PdfFileReader(temp_file) + self.assertEqual(11, pdfReader.numPages) + pageObj = pdfReader.getPage(0) + print(pageObj.extractText()) + self.assertTrue("Buffalo City" in pageObj.extractText()) diff --git a/scorecard/views.py b/scorecard/views.py index 16e86d501..1059df292 100644 --- a/scorecard/views.py +++ b/scorecard/views.py @@ -218,6 +218,7 @@ def get(self, request, *args, **kwargs): self.geo_code, self.geo.slug, ) + #url = f"http://localhost:9000{path}" url = request.build_absolute_uri(path) # !!! This relies on GeographyDetailView validating the user-provided # input to the path to avoid arbitraty command execution From 48a46fc7e339dd0105fa45350f650daf59c28e4d Mon Sep 17 00:00:00 2001 From: JD Bothma Date: Thu, 3 Dec 2020 23:16:11 +0200 Subject: [PATCH 8/8] Add basic GUI test for the profile page --- makepdf.js => assets/js/makepdf.js | 0 docker-compose.yml | 3 --- packages.txt | 2 ++ requirements.txt | 2 +- scorecard/tests/test_pdf.py | 22 ---------------------- scorecard/tests/test_ui.py | 23 +++++++++++++++++++++++ scorecard/views.py | 3 +-- 7 files changed, 27 insertions(+), 28 deletions(-) rename makepdf.js => assets/js/makepdf.js (100%) delete mode 100644 scorecard/tests/test_pdf.py create mode 100644 scorecard/tests/test_ui.py diff --git a/makepdf.js b/assets/js/makepdf.js similarity index 100% rename from makepdf.js rename to assets/js/makepdf.js diff --git a/docker-compose.yml b/docker-compose.yml index eaec3cb8e..dbc42c7e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,8 +36,6 @@ services: - minio stdin_open: true tty: true - extra_hosts: - - "testserver:127.0.0.1" postgres: image: postgres:11.5 @@ -56,7 +54,6 @@ services: args: USER_ID: ${USER_ID:-1001} GROUP_ID: ${GROUP_ID:-1001} - dockerfile: Dockerfile command: bin/wait-for-postgres.sh python manage.py runserver 0.0.0.0:8002 # command: bin/wait-for-postgres.sh gunicorn --limit-request-line 7168 --worker-class gevent municipal_finance.wsgi:application -t 600 --log-file - -b 0.0.0.0:8002 volumes: diff --git a/packages.txt b/packages.txt index dd5c2590e..8cd3a15cf 100644 --- a/packages.txt +++ b/packages.txt @@ -7,3 +7,5 @@ libpq-dev git # chromium for its dependencies, while we actually use the puppeteer chromium chromium +# webdriver for selenium testing +chromium-driver diff --git a/requirements.txt b/requirements.txt index 29351d313..f3ed14d14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,6 @@ psycogreen==1.0 psycopg2==2.7.3.1 ptyprocess==0.5.1 pycrypto==2.6.1 -PyPDF2==1.26.0 pyScss==1.3.7 python-dateutil==2.5.2 pytz==2017.3 @@ -56,6 +55,7 @@ PyYAML==5.1.1 requests==2.20.0 requests-futures==0.9.7 s3transfer==0.3.3 +selenium==3.141.0 sentry-sdk==0.19.1 simplegeneric==0.8.1 six==1.10.0 diff --git a/scorecard/tests/test_pdf.py b/scorecard/tests/test_pdf.py deleted file mode 100644 index cd71939f3..000000000 --- a/scorecard/tests/test_pdf.py +++ /dev/null @@ -1,22 +0,0 @@ -from django.contrib.staticfiles.testing import StaticLiveServerTestCase -from django.urls import reverse -import io -import PyPDF2 -import tempfile - - -class PDFTests(StaticLiveServerTestCase): - fixtures = ['tests/test_profile_and_pdf'] - port = 9000 - - def test_pdf_download(self): - response = self.client.get(f"{ self.live_server_url }/profiles/municipality-BUF-buffalo-city.pdf") - temp_file = tempfile.NamedTemporaryFile() - temp_file.write(response.content) - temp_file.flush() - temp_file.seek(0) - pdfReader = PyPDF2.PdfFileReader(temp_file) - self.assertEqual(11, pdfReader.numPages) - pageObj = pdfReader.getPage(0) - print(pageObj.extractText()) - self.assertTrue("Buffalo City" in pageObj.extractText()) diff --git a/scorecard/tests/test_ui.py b/scorecard/tests/test_ui.py new file mode 100644 index 000000000..6c4a6449e --- /dev/null +++ b/scorecard/tests/test_ui.py @@ -0,0 +1,23 @@ +from django.contrib.staticfiles.testing import StaticLiveServerTestCase +from django.urls import reverse +from selenium import webdriver + + +class ProfileTest(StaticLiveServerTestCase): + fixtures = ['tests/test_profile_and_pdf'] + + def test_profile(self): + options = webdriver.ChromeOptions() + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-setuid-sandbox") + options.add_argument("--headless") + driver = webdriver.Chrome(options=options) + + driver.get(f"{self.live_server_url}/profiles/municipality-BUF-buffalo-city/") + + self.assertEquals("Buffalo City", driver.find_element_by_css_selector(".page-heading__title").text) + + browser_logs = driver.get_log("browser") + browser_errors = [entry for entry in browser_logs if entry['level'] == 'SEVERE'] + self.assertEquals(0, len(browser_errors), browser_errors) diff --git a/scorecard/views.py b/scorecard/views.py index 1059df292..f9ca99494 100644 --- a/scorecard/views.py +++ b/scorecard/views.py @@ -218,11 +218,10 @@ def get(self, request, *args, **kwargs): self.geo_code, self.geo.slug, ) - #url = f"http://localhost:9000{path}" url = request.build_absolute_uri(path) # !!! This relies on GeographyDetailView validating the user-provided # input to the path to avoid arbitraty command execution - command = ["node", "makepdf.js", url] + command = ["node", "assets/js/makepdf.js", url] try: completed_process = subprocess.run( command,