-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Closes #389. I would have loved to use our json config-based importer for this, but it doesn’t handle group imports yet, and modifying it to inherit from BaseConstituencyGroupListImportCommand rather than BaseImportFromDataFrameCommand was beyond the scope of this ticket.
- Loading branch information
Showing
3 changed files
with
155 additions
and
0 deletions.
There are no files selected for viewing
52 changes: 52 additions & 0 deletions
52
hub/management/commands/generate_transition_network_groups_csv.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
from django.conf import settings | ||
|
||
import pandas as pd | ||
import requests | ||
|
||
from .base_generators import BaseLatLonGeneratorCommand | ||
|
||
|
||
class Command(BaseLatLonGeneratorCommand): | ||
help = ( | ||
"Generate CSV file of Transition Network groups with constituency information" | ||
) | ||
message = "Generating a CSV of Transition Network groups" | ||
|
||
out_file = settings.BASE_DIR / "data" / "transition_network_groups.csv" | ||
|
||
row_name = "group_name" | ||
uses_gss = True | ||
|
||
def get_dataframe(self): | ||
url = "https://maps.transitionnetwork.org/wp-json/cds/v1/initiatives/?country=GB&per_page=999" | ||
try: | ||
response = requests.get(url) | ||
response.raise_for_status() | ||
data = response.json() | ||
except requests.RequestException as e: | ||
print(f"Error reading remote file {url}: {e}") | ||
return None | ||
|
||
rows = [] | ||
for i, item in enumerate(data["body"]): | ||
location = item.get("location", {}) | ||
contact = item.get("contact", {}) | ||
if location.get("lat") and location.get("lng"): | ||
rows.append( | ||
{ | ||
"transition_network_id": item.get("id"), | ||
"transition_network_url": item.get("url"), | ||
"group_name": item.get("title"), | ||
"group_url": contact.get("website"), | ||
"group_facebook": contact.get("facebook"), | ||
"lat_lon": f"{location.get('lat')},{location.get('lng')}", | ||
"lat": location.get("lat"), | ||
"lon": location.get("lng"), | ||
} | ||
) | ||
else: | ||
print( | ||
f"Group {i}, {item.get('title')} has no lat/lon so will be ignored" | ||
) | ||
|
||
return pd.DataFrame(rows) |
102 changes: 102 additions & 0 deletions
102
hub/management/commands/import_transition_network_groups.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
from datetime import date | ||
|
||
from django.conf import settings | ||
|
||
import pandas as pd | ||
|
||
from hub.models import DataSet | ||
|
||
from .base_importers import ( | ||
BaseConstituencyGroupListImportCommand, | ||
MultipleAreaTypesMixin, | ||
) | ||
|
||
|
||
class Command(MultipleAreaTypesMixin, BaseConstituencyGroupListImportCommand): | ||
help = "Import Transition Network groups" | ||
message = "Importing Transition Network groups" | ||
|
||
data_file = settings.BASE_DIR / "data" / "transition_network_groups.csv" | ||
|
||
uses_gss = True | ||
area_types = ["WMC", "WMC23", "STC", "DIS"] | ||
cons_col_map = { | ||
"WMC": "WMC", | ||
"WMC23": "WMC23", | ||
"STC": "STC", | ||
"DIS": "DIS", | ||
} | ||
defaults = { | ||
"label": "Transition Network groups", | ||
"data_type": "json", | ||
"category": "movement", | ||
"subcategory": "groups", | ||
"release_date": str(date.today()), | ||
"source_label": "Data from Transition Network contributors under Open Database Licence.", | ||
"source": "https://maps.transitionnetwork.org/", | ||
"source_type": "api", | ||
"data_url": "https://maps.transitionnetwork.org/wp-json/cds/v1/initiatives/?country=GB&per_page=999", | ||
"table": "areadata", | ||
"default_value": {}, | ||
"exclude_countries": [], | ||
"is_filterable": False, | ||
"is_shadable": False, | ||
"comparators": DataSet.comparators_default(), | ||
"unit_type": "point", | ||
"unit_distribution": "point", | ||
} | ||
|
||
count_defaults = { | ||
"label": "Number of Transition Network groups", | ||
"data_type": "integer", | ||
"category": "movement", | ||
"release_date": str(date.today()), | ||
"source_label": "Data from Transition Network contributors under Open Database Licence.", | ||
"source": "https://maps.transitionnetwork.org/", | ||
"source_type": "api", | ||
"data_url": "https://maps.transitionnetwork.org/wp-json/cds/v1/initiatives/?country=GB&per_page=999", | ||
"table": "areadata", | ||
"default_value": 0, | ||
"exclude_countries": [], | ||
"is_filterable": True, | ||
"is_shadable": True, | ||
"comparators": DataSet.numerical_comparators(), | ||
"unit_type": "point", | ||
"unit_distribution": "point", | ||
} | ||
|
||
data_sets = { | ||
"constituency_transition_groups": { | ||
"defaults": defaults, | ||
}, | ||
"constituency_transition_group_count": { | ||
"defaults": count_defaults, | ||
}, | ||
} | ||
|
||
group_data_type = "constituency_transition_groups" | ||
count_data_type = "constituency_transition_group_count" | ||
|
||
def get_df(self): | ||
if self.data_file.exists() is False: | ||
return None | ||
|
||
df = pd.read_csv(self.data_file) | ||
|
||
# Remove columns we don't need, and rename one we do. | ||
df = df.rename( | ||
columns={ | ||
"transition_network_url": "url" # for automatic chip linking in template | ||
} | ||
).drop( | ||
columns=[ | ||
"transition_network_id", | ||
"group_url", | ||
"group_facebook", | ||
] | ||
) | ||
|
||
return df | ||
|
||
def get_group_json(self, row): | ||
return row[["group_name", "url"]].dropna().to_dict() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters