Skip to content

Commit

Permalink
Cleanup use of word boundary
Browse files Browse the repository at this point in the history
  • Loading branch information
@norkans7
norkans7 committed Jan 9, 2025
1 parent 3795832 commit 5176976
Show file tree
Hide file tree
Showing 10 changed files with 80 additions and 182 deletions.
8 changes: 4 additions & 4 deletions temba/api/v2/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,11 +408,11 @@ class BoundariesEndpoint(ListAPIMixin, BaseEndpoint):
A `GET` returns the boundaries for your organization with the following fields. To include geometry,
specify `geometry=true`.
* **osm_id** - the OSM ID for this boundary prefixed with the element type (string).
* **name** - the name of the administrative boundary (string).
* **parent** - the id of the containing parent of this boundary or null if this boundary is a country (string).
* **osm_id** - the OSM ID for this location prefixed with the element type (string).
* **name** - the name of the administrative location (string).
* **parent** - the id of the containing parent of this location or null if this location is a country (string).
* **level** - the level: 0 for country, 1 for state, 2 for district (int).
* **geometry** - the geometry for this boundary, which will usually be a MultiPolygon (GEOJSON).
* **geometry** - the geometry for this location, which will usually be a MultiPolygon (GEOJSON).
**Note that including geometry may produce a very large result so it is recommended to cache the results on the
client side.**
Expand Down
38 changes: 19 additions & 19 deletions temba/locations/management/commands/import_geojson.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument("files", nargs="+")
parser.add_argument(
"--country", dest="country", default=None, help="Only process the boundary files for this country osm id"
"--country", dest="country", default=None, help="Only process the location files for this country osm id"
)

def import_file(self, filename, file):
Expand Down Expand Up @@ -79,11 +79,11 @@ def import_file(self, filename, file):
continue

# try to find existing admin level by osm_id
boundary = Location.objects.filter(osm_id=osm_id)
location = Location.objects.filter(osm_id=osm_id)

# didn't find it? what about by name?
if not boundary:
boundary = Location.objects.filter(parent=parent, name__iexact=name)
if not location:
location = Location.objects.filter(parent=parent, name__iexact=name)

# skip over items with no geometry
if not feature["geometry"] or not feature["geometry"]["coordinates"]:
Expand All @@ -105,18 +105,18 @@ def import_file(self, filename, file):
kwargs["simplified_geometry"] = geometry

# if this is an update, just update with those fields
if boundary:
if location:
if not parent:
kwargs["path"] = name
else:
kwargs["path"] = parent.path + Location.PADDED_PATH_SEPARATOR + name

self.stdout.write(self.style.SUCCESS(f" ** updating {name} ({osm_id})"))
boundary = boundary.first()
boundary.update(**kwargs)
location = location.first()
location.update(**kwargs)

# update any children
boundary.update_path()
location.update_path()

# otherwise, this is new, so create it
else:
Expand All @@ -126,19 +126,19 @@ def import_file(self, filename, file):
# keep track of this osm_id
seen_osm_ids.add(osm_id)

# now remove any unseen boundaries
# now remove any unseen locations
if osm_id:
last_boundary = Location.objects.filter(osm_id=osm_id).first()
if last_boundary:
self.stdout.write(self.style.SUCCESS(f" ** removing unseen boundaries ({osm_id})"))
country = last_boundary.get_root()
unseen_boundaries = country.get_descendants().filter(level=level).exclude(osm_id__in=seen_osm_ids)
last_location = Location.objects.filter(osm_id=osm_id).first()
if last_location:
self.stdout.write(self.style.SUCCESS(f" ** removing unseen locations ({osm_id})"))
country = last_location.get_root()
unseen_locations = country.get_descendants().filter(level=level).exclude(osm_id__in=seen_osm_ids)
deleted_count = 0
for unseen_boundary in unseen_boundaries:
unseen_boundary.release()
for unseen_location in unseen_locations:
unseen_location.release()
deleted_count += 1
if deleted_count > 0:
self.stdout.write(f" ** Unseen boundaries removed: {deleted_count}")
self.stdout.write(f" ** Unseen locations removed: {deleted_count}")

return country, seen_osm_ids
else:
Expand Down Expand Up @@ -193,7 +193,7 @@ def handle(self, *args, **options):
if country is None:
return

# remove all other unseen boundaries from the database for the country
# remove all other unseen locations from the database for the country
with connection.cursor() as cursor:
cursor.execute(
"""
Expand All @@ -217,7 +217,7 @@ def handle(self, *args, **options):
""",
(country.id, list(updated_osm_ids)),
)
self.stdout.write(self.style.SUCCESS(f"Other unseen boundaries removed: {cursor.rowcount}"))
self.stdout.write(self.style.SUCCESS(f"Other unseen locations removed: {cursor.rowcount}"))

if country:
self.stdout.write(self.style.SUCCESS((f" ** updating paths for all of {country.name}")))
Expand Down
103 changes: 0 additions & 103 deletions temba/locations/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,116 +46,13 @@ class AdminBoundary(MPTTModel, models.Model):
objects = NoGeometryManager()
geometries = GeometryManager()

@staticmethod
def get_geojson_dump(name, features):
# build a feature collection
feature_collection = geojson.FeatureCollection(features)
return geojson.dumps({"name": name, "geometry": feature_collection})

def as_json(self, org):
result = dict(osm_id=self.osm_id, name=self.name, level=self.level, aliases="", path=self.path)

if self.parent:
result["parent_osm_id"] = self.parent.osm_id

aliases = "\n".join(sorted([alias.name for alias in self.aliases.filter(org=org)]))
result["aliases"] = aliases
return result

def get_geojson_feature(self):
return geojson.Feature(
properties=dict(name=self.name, osm_id=self.osm_id, id=self.pk, level=self.level),
zoomable=True if self.children.all() else False,
geometry=None if not self.simplified_geometry else geojson.loads(self.simplified_geometry.geojson),
)

def get_geojson(self):
return AdminBoundary.get_geojson_dump(self.name, [self.get_geojson_feature()])

def get_children_geojson(self):
children = []
for child in self.children.all():
children.append(child.get_geojson_feature())
return AdminBoundary.get_geojson_dump(self.name, children)

def update(self, **kwargs):
AdminBoundary.objects.filter(id=self.id).update(**kwargs)

# update our object values so that self is up to date
for key, value in kwargs.items():
setattr(self, key, value)

def update_path(self):
if self.level == 0:
self.path = self.name
self.save(update_fields=("path",))

def _update_child_paths(boundary):
boundaries = AdminBoundary.objects.filter(parent=boundary).only("name", "parent__path")
boundaries.update(
path=Concat(Value(boundary.path), Value(" %s " % AdminBoundary.PATH_SEPARATOR), F("name"))
)
for boundary in boundaries:
_update_child_paths(boundary)

_update_child_paths(self)

def update_aliases(self, org, user, aliases: list):
siblings = self.parent.children.all()

self.aliases.filter(org=org).delete() # delete any existing aliases for this workspace

for new_alias in aliases:
assert new_alias and len(new_alias) < AdminBoundary.MAX_NAME_LEN

# aliases are only allowed to exist on one boundary with same parent at a time
BoundaryAlias.objects.filter(name=new_alias, boundary__in=siblings, org=org).delete()

BoundaryAlias.create(org, user, self, new_alias)

def release(self):
for child_boundary in AdminBoundary.objects.filter(parent=self): # pragma: no cover
child_boundary.release()

self.aliases.all().delete()
self.delete()

Check warning on line 54 in temba/locations/models.py

View check run for this annotation

Codecov / codecov/patch

temba/locations/models.py#L53-L54 

Added lines #L53 - L54 were not covered by tests

@classmethod
def create(cls, osm_id, name, level, parent=None, **kwargs):
"""
Create method that takes care of creating path based on name and parent
"""
path = name
if parent is not None:
path = parent.path + AdminBoundary.PADDED_PATH_SEPARATOR + name

return AdminBoundary.objects.create(osm_id=osm_id, name=name, level=level, parent=parent, path=path, **kwargs)

@classmethod
def strip_last_path(cls, path):
"""
Strips the last part of the passed in path. Throws if there is no separator
"""
parts = path.split(AdminBoundary.PADDED_PATH_SEPARATOR)
if len(parts) <= 1: # pragma: no cover
raise Exception("strip_last_path called without a path to strip")

return AdminBoundary.PADDED_PATH_SEPARATOR.join(parts[:-1])

@classmethod
def get_by_path(cls, org, path):
cache = getattr(org, "_abs", {})

if not cache:
setattr(org, "_abs", cache)

boundary = cache.get(path)
if not boundary:
boundary = AdminBoundary.objects.filter(path=path).first()
cache[path] = boundary

return boundary

def __str__(self):
return self.name

Check warning on line 57 in temba/locations/models.py

View check run for this annotation

Codecov / codecov/patch

temba/locations/models.py#L57 

Added line #L57 was not covered by tests

Expand Down
32 changes: 16 additions & 16 deletions temba/locations/tests/test_geojson.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ class ImportGeoJSONtest(TembaTest):
}]
}"""

data_geojson_level_1_new_boundary = """{
data_geojson_level_1_new_location = """{
"type": "FeatureCollection",
"features": [{
"type": "Feature",
Expand Down Expand Up @@ -183,7 +183,7 @@ def test_ok_filename_admin(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing R188933admin0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen boundaries (R1000)\nOther unseen boundaries removed: 0\n ** updating paths for all of Granica\n",
"=== parsing R188933admin0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen locations (R1000)\nOther unseen locations removed: 0\n ** updating paths for all of Granica\n",
)

self.assertEqual(Location.objects.count(), 1)
Expand All @@ -195,7 +195,7 @@ def test_ok_filename_admin_level_with_country_prefix(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing R188933admin0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen boundaries (R1000)\nOther unseen boundaries removed: 0\n ** updating paths for all of Granica\n",
"=== parsing R188933admin0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen locations (R1000)\nOther unseen locations removed: 0\n ** updating paths for all of Granica\n",
)

self.assertEqual(Location.objects.count(), 1)
Expand All @@ -207,7 +207,7 @@ def test_ok_filename_admin_level(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing admin_level_0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen boundaries (R1000)\nOther unseen boundaries removed: 0\n ** updating paths for all of Granica\n",
"=== parsing admin_level_0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen locations (R1000)\nOther unseen locations removed: 0\n ** updating paths for all of Granica\n",
)

self.assertEqual(Location.objects.count(), 1)
Expand Down Expand Up @@ -240,7 +240,7 @@ def test_feature_multipolygon_geometry(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing admin_level_0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen boundaries (R1000)\nOther unseen boundaries removed: 0\n ** updating paths for all of Granica\n",
"=== parsing admin_level_0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen locations (R1000)\nOther unseen locations removed: 0\n ** updating paths for all of Granica\n",
)

self.assertOSMIDs({"R1000"})
Expand All @@ -267,7 +267,7 @@ def test_feature_with_parent(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing admin_level_0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen boundaries (R1000)\n=== parsing admin_level_1_simplified.json\n ** adding Međa 2 (R2000)\n ** removing unseen boundaries (R2000)\n=== parsing admin_level_2_simplified.json\n ** adding Međa 55 (R55000)\n ** removing unseen boundaries (R55000)\nOther unseen boundaries removed: 0\n ** updating paths for all of Granica\n",
"=== parsing admin_level_0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen locations (R1000)\n=== parsing admin_level_1_simplified.json\n ** adding Međa 2 (R2000)\n ** removing unseen locations (R2000)\n=== parsing admin_level_2_simplified.json\n ** adding Međa 55 (R55000)\n ** removing unseen locations (R55000)\nOther unseen locations removed: 0\n ** updating paths for all of Granica\n",
)

self.assertOSMIDs({"R1000", "R2000", "R55000"})
Expand Down Expand Up @@ -299,12 +299,12 @@ def test_update_features_with_parent(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing admin_level_0_simplified.json\n ** updating Granica (R1000)\n ** removing unseen boundaries (R1000)\n=== parsing admin_level_1_simplified.json\n ** updating Međa 2 (R2000)\n ** removing unseen boundaries (R2000)\nOther unseen boundaries removed: 0\n ** updating paths for all of Granica\n",
"=== parsing admin_level_0_simplified.json\n ** updating Granica (R1000)\n ** removing unseen locations (R1000)\n=== parsing admin_level_1_simplified.json\n ** updating Međa 2 (R2000)\n ** removing unseen locations (R2000)\nOther unseen locations removed: 0\n ** updating paths for all of Granica\n",
)

self.assertOSMIDs({"R1000", "R2000"})

def test_remove_unseen_boundaries(self):
def test_remove_unseen_locations(self):
# insert features in the database
geojson_data = [self.data_geojson_level_0, self.data_geojson_level_1]

Expand All @@ -320,8 +320,8 @@ def test_remove_unseen_boundaries(self):

LocationAlias.create(self.org, self.admin, Location.objects.get(osm_id="R2000"), "My Alias")

# update data, and add a new boundary
geojson_data = [self.data_geojson_level_0, self.data_geojson_level_1_new_boundary]
# update data, and add a new location
geojson_data = [self.data_geojson_level_0, self.data_geojson_level_1_new_location]

with patch("builtins.open") as mock_file:
mock_file.return_value.__enter__ = lambda filename: filename
Expand All @@ -333,13 +333,13 @@ def test_remove_unseen_boundaries(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing admin_level_0_simplified.json\n ** updating Granica (R1000)\n ** removing unseen boundaries (R1000)\n=== parsing admin_level_1_simplified.json\n ** adding Međa 3 (R3000)\n ** removing unseen boundaries (R3000)\n ** Unseen boundaries removed: 1\nOther unseen boundaries removed: 0\n ** updating paths for all of Granica\n",
"=== parsing admin_level_0_simplified.json\n ** updating Granica (R1000)\n ** removing unseen locations (R1000)\n=== parsing admin_level_1_simplified.json\n ** adding Međa 3 (R3000)\n ** removing unseen locations (R3000)\n ** Unseen locations removed: 1\nOther unseen locations removed: 0\n ** updating paths for all of Granica\n",
)

self.assertOSMIDs({"R1000", "R3000"})

def test_remove_other_unseen_boundaries(self):
# other unseen boundaries are boundaries which have not been updated in any way for a country
def test_remove_other_unseen_locations(self):
# other unseen locations are locations which have not been updated in any way for a country

# insert features in the database
geojson_data = [self.data_geojson_level_0, self.data_geojson_level_1]
Expand All @@ -354,7 +354,7 @@ def test_remove_other_unseen_boundaries(self):

self.assertOSMIDs({"R1000", "R2000"})

# update data, and add a new boundary
# update data, and add a new location
geojson_data = [self.data_geojson_level_0]

with patch("builtins.open") as mock_file:
Expand All @@ -367,7 +367,7 @@ def test_remove_other_unseen_boundaries(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing admin_level_0_simplified.json\n ** updating Granica (R1000)\n ** removing unseen boundaries (R1000)\nOther unseen boundaries removed: 1\n ** updating paths for all of Granica\n",
"=== parsing admin_level_0_simplified.json\n ** updating Granica (R1000)\n ** removing unseen locations (R1000)\nOther unseen locations removed: 1\n ** updating paths for all of Granica\n",
)

self.assertOSMIDs({"R1000"})
Expand All @@ -385,7 +385,7 @@ def test_zipfiles_parsing(self):

self.assertEqual(
captured_output.getvalue(),
"=== parsing admin_level_0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen boundaries (R1000)\nOther unseen boundaries removed: 0\n ** updating paths for all of Granica\n",
"=== parsing admin_level_0_simplified.json\n ** adding Granica (R1000)\n ** removing unseen locations (R1000)\nOther unseen locations removed: 0\n ** updating paths for all of Granica\n",
)

self.assertOSMIDs({"R1000"})
Expand Down
Loading

0 comments on commit 5176976

Please sign in to comment.