Skip to content

Commit

Permalink
refactor for text fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@sekharpanja
sekharpanja committed Oct 31, 2023
1 parent ac86ddd commit ee660cf
Showing 1 changed file with 74 additions and 98 deletions.
172 changes: 74 additions & 98 deletions dataworkspace/dataworkspace/apps/datasets/migrations/0155_catalogues_replace_dit_dbt.py
View file
Original file line number Diff line number Diff line change
@@ -1,113 +1,89 @@
import re
from django.db import migrations
from django.db.models import F, Func, Value

skip_items = [
"great.gov.uk value for money survey responses",
"DDaT Recruitment Survey",
"Regional inbound enquiries 2018-2020",
"DDaT Return to the office survey",
"DIT MS Teams feedback survey",
"Civil Service Leadership Academy Training Records",
"Jobs supported by UK exports",
"DIT staff and contractors: leavers",
"HR People Data merged 13 month rolling",
"People Data: Cyber team report",
"DIT staff and contractors: leavers",
"DIT people data: joiners and leavers",
"DIT Return to office: Personal Risk Assessment (PRA) data",
]
dit = "DIT"
dbt = "DBT"
dit_full = "Department for International Trade"
dbt_full = "Department for Business and Trade"


def replace_func(field_name, find_str, replace_str):
return Func(F(field_name), Value(find_str), Value(replace_str), function="replace")


def search_and_replace_model(model):
skip_items = [
"great.gov.uk value for money survey responses",
"DDaT Recruitment Survey",
"Regional inbound enquiries 2018-2020",
"DDaT Return to the office survey",
"DIT MS Teams feedback survey",
"Civil Service Leadership Academy Training Records",
"Jobs supported by UK exports",
"DIT staff and contractors: leavers",
"HR People Data merged 13 month rolling",
"People Data: Cyber team report",
"DIT staff and contractors: leavers",
"DIT people data: joiners and leavers",
"DIT Return to office: Personal Risk Assessment (PRA) data",
]

for dataset in model.objects.all():
if dataset.name in skip_items:
continue
dataset.name = re.sub(dataset.name, "DIT", "DBT", flags=re.IGNORECASE)
dataset.name = re.sub(
dataset.name,
"Department of International Trade",
"Department for Business and Trade",
flags=re.IGNORECASE,
)

dataset.short_description = re.sub(
dataset.short_description, "DIT", "DBT", flags=re.IGNORECASE
)
dataset.short_description = re.sub(
dataset.short_description,
"Department of International Trade",
"Department for Business and Trade",
flags=re.IGNORECASE,
)

dataset.description = re.sub(dataset.description, "DIT", "DBT", flags=re.IGNORECASE)
dataset.description = re.sub(
dataset.description,
"Department of International Trade",
"Department for Business and Trade",
flags=re.IGNORECASE,
)

dataset.retention_policy = re.sub(
dataset.retention_policy, "DIT", "DBT", flags=re.IGNORECASE
)
dataset.retention_policy = re.sub(
dataset.retention_policy,
"Department of International Trade",
"Department for Business and Trade",
flags=re.IGNORECASE,
)

dataset.personal_data = re.sub(dataset.personal_data, "DIT", "DBT", flags=re.IGNORECASE)
dataset.personal_data = re.sub(
dataset.personal_data,
"Department of International Trade",
"Department for Business and Trade",
flags=re.IGNORECASE,
)

dataset.license = re.sub(dataset.license, "DIT", "DBT", flags=re.IGNORECASE)
dataset.license = re.sub(
dataset.license,
"Department of International Trade",
"Department for Business and Trade",
flags=re.IGNORECASE,
)

dataset.restrictions_on_usage = re.sub(
dataset.restrictions_on_usage, "DIT", "DBT", flags=re.IGNORECASE
)
dataset.restrictions_on_usage = re.sub(
dataset.restrictions_on_usage,
"Department of International Trade",
"Department for Business and Trade",
flags=re.IGNORECASE,
)

eligibility_criteria = list(dataset.eligibility_criteria.all())
eligibility_criteria = [
re.sub(criteria, "DIT", "DBT", flags=re.IGNORECASE)
for criteria in eligibility_criteria
]
eligibility_criteria = [
re.sub(
criteria,
"Department of International Trade",
"Department for Business and Trade",
flags=re.IGNORECASE,
)
for criteria in eligibility_criteria
]
model.objects.exclude(name__in=skip_items).update(
name=replace_func("name", dit, dbt),
short_description=replace_func("short_description", dit, dbt),
description=replace_func("description", dit, dbt),
licence=replace_func("licence", dit, dbt),
restrictions_on_usage=replace_func("restrictions_on_usage", dit, dbt),
)

model.objects.exclude(name__in=skip_items).update(
name=replace_func("name", dit_full, dbt_full),
short_description=replace_func("short_description", dit_full, dbt_full),
description=replace_func("description", dit_full, dbt_full),
licence=replace_func("licence", dit_full, dbt_full),
restrictions_on_usage=replace_func("restrictions_on_usage", dit_full, dbt_full),
)


def search_and_replace_specific_fields(model):
model.objects.exclude(name__in=skip_items).update(
retention_policy=replace_func("retention_policy", dit, dbt),
personal_data=replace_func("personal_data", dit, dbt),
)
model.objects.exclude(name__in=skip_items).update(
retention_policy=replace_func("retention_policy", dit_full, dbt_full),
personal_data=replace_func("personal_data", dit_full, dbt_full),
)
for dataset in model.objects.exclude(name__in=skip_items).all():
if dataset.eligibility_criteria:
eligibility_criteria = list(dataset.eligibility_criteria)
eligibility_criteria = [
re.sub(criteria, dit, dbt, flags=re.IGNORECASE)
for criteria in eligibility_criteria
]
eligibility_criteria = [
re.sub(
criteria,
dit_full,
dbt_full,
flags=re.IGNORECASE,
)
for criteria in eligibility_criteria
]
dataset.eligibility_criteria = eligibility_criteria
dataset.save()


def search_and_replace_catalogue_items(apps, _):
model = apps.get_model("datasets", "app_dataset")
model = apps.get_model("datasets", "DataSet")
search_and_replace_model(model)
search_and_replace_specific_fields(model)

model = apps.get_model("datasets", "datasets_visualisationcatalogueitem")
model = apps.get_model("datasets", "VisualisationCatalogueItem")
search_and_replace_model(model)
search_and_replace_specific_fields(model)

model = apps.get_model("datasets", "app_referencedataset")
model = apps.get_model("datasets", "ReferenceDataset")
search_and_replace_model(model)


Expand Down

0 comments on commit ee660cf

Please sign in to comment.