Skip to content

Commit

Permalink
Import Transition Network groups
Browse files Browse the repository at this point in the history
Closes #389.

I would have loved to use our json config-based importer for this,
but it doesn’t handle group imports yet, and modifying it to inherit
from BaseConstituencyGroupListImportCommand rather than
BaseImportFromDataFrameCommand was beyond the scope of this ticket.
  • Loading branch information
zarino authored and struan committed Jan 27, 2025
1 parent c4f1170 commit d028ca8
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 0 deletions.
52 changes: 52 additions & 0 deletions hub/management/commands/generate_transition_network_groups_csv.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
from django.conf import settings

import pandas as pd
import requests

from .base_generators import BaseLatLonGeneratorCommand


class Command(BaseLatLonGeneratorCommand):
help = (
"Generate CSV file of Transition Network groups with constituency information"
)
message = "Generating a CSV of Transition Network groups"

out_file = settings.BASE_DIR / "data" / "transition_network_groups.csv"

row_name = "group_name"
uses_gss = True

def get_dataframe(self):
url = "https://maps.transitionnetwork.org/wp-json/cds/v1/initiatives/?country=GB&per_page=999"
try:
response = requests.get(url)
response.raise_for_status()
data = response.json()
except requests.RequestException as e:
print(f"Error reading remote file {url}: {e}")
return None

rows = []
for i, item in enumerate(data["body"]):
location = item.get("location", {})
contact = item.get("contact", {})
if location.get("lat") and location.get("lng"):
rows.append(
{
"transition_network_id": item.get("id"),
"transition_network_url": item.get("url"),
"group_name": item.get("title"),
"group_url": contact.get("website"),
"group_facebook": contact.get("facebook"),
"lat_lon": f"{location.get('lat')},{location.get('lng')}",
"lat": location.get("lat"),
"lon": location.get("lng"),
}
)
else:
print(
f"Group {i}, {item.get('title')} has no lat/lon so will be ignored"
)

return pd.DataFrame(rows)
102 changes: 102 additions & 0 deletions hub/management/commands/import_transition_network_groups.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
from datetime import date

from django.conf import settings

import pandas as pd

from hub.models import DataSet

from .base_importers import (
BaseConstituencyGroupListImportCommand,
MultipleAreaTypesMixin,
)


class Command(MultipleAreaTypesMixin, BaseConstituencyGroupListImportCommand):
help = "Import Transition Network groups"
message = "Importing Transition Network groups"

data_file = settings.BASE_DIR / "data" / "transition_network_groups.csv"

uses_gss = True
area_types = ["WMC", "WMC23", "STC", "DIS"]
cons_col_map = {
"WMC": "WMC",
"WMC23": "WMC23",
"STC": "STC",
"DIS": "DIS",
}
defaults = {
"label": "Transition Network groups",
"data_type": "json",
"category": "movement",
"subcategory": "groups",
"release_date": str(date.today()),
"source_label": "Data from Transition Network contributors under Open Database Licence.",
"source": "https://maps.transitionnetwork.org/",
"source_type": "api",
"data_url": "https://maps.transitionnetwork.org/wp-json/cds/v1/initiatives/?country=GB&per_page=999",
"table": "areadata",
"default_value": {},
"exclude_countries": [],
"is_filterable": False,
"is_shadable": False,
"comparators": DataSet.comparators_default(),
"unit_type": "point",
"unit_distribution": "point",
}

count_defaults = {
"label": "Number of Transition Network groups",
"data_type": "integer",
"category": "movement",
"release_date": str(date.today()),
"source_label": "Data from Transition Network contributors under Open Database Licence.",
"source": "https://maps.transitionnetwork.org/",
"source_type": "api",
"data_url": "https://maps.transitionnetwork.org/wp-json/cds/v1/initiatives/?country=GB&per_page=999",
"table": "areadata",
"default_value": 0,
"exclude_countries": [],
"is_filterable": True,
"is_shadable": True,
"comparators": DataSet.numerical_comparators(),
"unit_type": "point",
"unit_distribution": "point",
}

data_sets = {
"constituency_transition_groups": {
"defaults": defaults,
},
"constituency_transition_group_count": {
"defaults": count_defaults,
},
}

group_data_type = "constituency_transition_groups"
count_data_type = "constituency_transition_group_count"

def get_df(self):
if self.data_file.exists() is False:
return None

df = pd.read_csv(self.data_file)

# Remove columns we don't need, and rename one we do.
df = df.rename(
columns={
"transition_network_url": "url" # for automatic chip linking in template
}
).drop(
columns=[
"transition_network_id",
"group_url",
"group_facebook",
]
)

return df

def get_group_json(self, row):
return row[["group_name", "url"]].dropna().to_dict()
1 change: 1 addition & 0 deletions hub/views/area.py
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ def get_context_data(self, **kwargs):
"constituency_nt_properties_count": "constituency_nt_properties",
"constituency_wi_group_count": "constituency_wi_groups",
"constituency_foe_group_count": "constituency_foe_groups",
"constituency_transition_group_count": "constituency_transition_groups",
"power_postcodes_count": "power_postcodes",
"tcc_open_letter_signatories_count": "tcc_open_letter_signatories",
"wildlife_trusts_reserves_count": "wildlife_trusts_reserves",
Expand Down

0 comments on commit d028ca8

Please sign in to comment.