From 810d1ab61bff11c72ea9ba0c8ecd9aba8852d922 Mon Sep 17 00:00:00 2001 From: Ari Hershowitz Date: Mon, 12 Apr 2021 09:35:25 -0700 Subject: [PATCH] Feature/bill committee (#301) * Add committees to billjson metadata * Use committee data --- UPDATES_CELERY.adoc | 6 +-- server_py/flatgov/bills/handler.py | 5 -- .../migrations/0002_bill_committees_dict.py | 18 +++++++ server_py/flatgov/bills/models.py | 1 + server_py/flatgov/bills/views.py | 1 + server_py/flatgov/common/billdata.py | 48 ++++++++++++++++++- server_py/flatgov/common/constants.py | 3 +- server_py/flatgov/common/relatedBills.py | 8 ++-- server_py/flatgov/flatgov/dev.py | 1 + server_py/flatgov/templates/bills/detail.html | 5 +- 10 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 server_py/flatgov/bills/migrations/0002_bill_committees_dict.py diff --git a/UPDATES_CELERY.adoc b/UPDATES_CELERY.adoc index c5721e3..5567080 100644 --- a/UPDATES_CELERY.adoc +++ b/UPDATES_CELERY.adoc @@ -207,9 +207,9 @@ Celery tasks: uscongress:: update uscongress bill and metadata statementAdminPolicy:: updates statements of administration policy from the current White House (OMB) website -committeereport:: TODO -cbo:: TODO -crs:: TODO +committeereport:: update committee reports associated with bills +cbo:: update cbo reports associated with bills +crs:: update crs reports associated with bills #### US Congress Task diff --git a/server_py/flatgov/bills/handler.py b/server_py/flatgov/bills/handler.py index 41f8cd4..7b85f72 100644 --- a/server_py/flatgov/bills/handler.py +++ b/server_py/flatgov/bills/handler.py @@ -38,13 +38,8 @@ def get_bill(self, *args, **kwargs): 'sponsor': sponsor, 'cosponsors_dict': cosponsors_dict, } - # bill = Bill.objects.get_or_create( - # bill_congress_type_number=self.congress, - # default=bill_data - # ) return bill_data def get_cosponsors(self, *args, **kwargs): - bill = kwargs.get('bill') cosponsors = self.data.get('cosponsors') return cosponsors diff --git a/server_py/flatgov/bills/migrations/0002_bill_committees_dict.py b/server_py/flatgov/bills/migrations/0002_bill_committees_dict.py new file mode 100644 index 0000000..364446d --- /dev/null +++ b/server_py/flatgov/bills/migrations/0002_bill_committees_dict.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.8 on 2021-04-09 23:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bills', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='bill', + name='committees_dict', + field=models.JSONField(default=list), + ), + ] diff --git a/server_py/flatgov/bills/models.py b/server_py/flatgov/bills/models.py index 6ccad38..9f00636 100644 --- a/server_py/flatgov/bills/models.py +++ b/server_py/flatgov/bills/models.py @@ -24,6 +24,7 @@ class Bill(models.Model): related_bills = models.JSONField(default=list) related_dict = models.JSONField(default=dict) cosponsors_dict = models.JSONField(default=list) + committees_dict = models.JSONField(default=list) es_similarity = models.JSONField(default=list) es_similar_bills_dict = models.JSONField(default=dict) diff --git a/server_py/flatgov/bills/views.py b/server_py/flatgov/bills/views.py index 4dbd178..e31d8ed 100644 --- a/server_py/flatgov/bills/views.py +++ b/server_py/flatgov/bills/views.py @@ -125,6 +125,7 @@ def get_qs_related_bill(self): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['cosponsors_dict'] = self.get_cosponsors_dict() + context['committees_dict'] = self.object.committees_dict context['cosponsors'] = self.get_cosponsors() context['statements'] = self.get_related_statements() context['committees'] = self.get_related_committees() diff --git a/server_py/flatgov/common/billdata.py b/server_py/flatgov/common/billdata.py index 6c71131..0f122c3 100644 --- a/server_py/flatgov/common/billdata.py +++ b/server_py/flatgov/common/billdata.py @@ -5,6 +5,7 @@ import sys, os, argparse, logging, re, json, gzip from typing import Dict from functools import reduce +from bills.models import Bill from common import constants, utils @@ -191,13 +192,56 @@ def loadBillsMeta(billMetaPath = constants.PATH_TO_BILLS_META, zip = True): return billsMeta -def saveBillsMeta(billsMeta: Dict, metaPath = constants.PATH_TO_BILLS_META, zip = True): +def saveBillsMeta(billsMeta: Dict, metaPath = constants.PATH_TO_BILLS_META, zip = False): with open(metaPath, 'w') as f: json.dump(billsMeta, f) if zip: with gzip.open(metaPath + '.gz', 'wt', encoding="utf-8") as zipfile: json.dump(billsMeta, zipfile) +def saveBillsMetaToDb(): + billsMeta = loadBillsMeta(billMetaPath = constants.PATH_TO_BILLS_META_GO, zip = False) + for billnumber, billdata in billsMeta.items(): + billCongressTypeNumber = billdata.get('bill_congress_type_number','') + if not billCongressTypeNumber: + continue + billNumberMatch = constants.BILL_NUMBER_REGEX_COMPILED.match(billCongressTypeNumber) + [congress, billType, numberOfBill, billVersion, billTypeNumber] = ["" for x in range(5)] + if billNumberMatch and billNumberMatch.groups(): + [congress, billType, numberOfBill, billVersion] = billNumberMatch.groups() + else: + continue + print('Loading: ' + billCongressTypeNumber) + metadata = { + 'type': billType, + 'congress': int(congress), + 'number': numberOfBill, + } + + for key, value in metadata.items(): + if value: + billdata[key] = value + + billdata['cosponsors_dict'] = billdata.get('cosponsors', []) + billdata['committees_dict'] = billdata.get('committees', []) + keys = billdata.keys() + if 'cosponsors' in keys: + del billdata['cosponsors'] + if 'committees' in keys: + del billdata['committees'] + + # Avoid not null constraint + if not billdata.get('related_bills'): + billdata['related_bills'] = [] + + if not billdata.get('cosponsors_dict'): + billdata['cosponsors_dict'] = [] + + if not billdata.get('committees_dict'): + billdata['committees_dict'] = [] + + Bill.objects.update_or_create(bill_congress_type_number=billnumber, defaults=billdata) + def updateBillsMeta(billsMeta= {}): def addToBillsMeta(dirName: str, fileName: str): billDict = loadJSON(os.path.join(dirName, fileName)) @@ -216,7 +260,9 @@ def addToBillsMeta(dirName: str, fileName: str): billsMeta[billCongressTypeNumber]['titles'] = [title.get('title') for title in titles] billsMeta[billCongressTypeNumber]['titles_whole_bill'] = [title.get('title') for title in titles if not title.get('is_for_portion')] cosponsors = getCosponsors(fileDict=billDict, includeFields=['name', 'bioguide_id']) + committees = billDict.get('committees', []) billsMeta[billCongressTypeNumber]['cosponsors'] = cosponsors + billsMeta[billCongressTypeNumber]['committees'] = committees # TODO convert bill_id to billnumber billsMeta[billCongressTypeNumber]['related_bills'] = billDict.get('related_bills') diff --git a/server_py/flatgov/common/constants.py b/server_py/flatgov/common/constants.py index 459b1eb..a5410ec 100644 --- a/server_py/flatgov/common/constants.py +++ b/server_py/flatgov/common/constants.py @@ -34,7 +34,8 @@ with open(PATH_BILL_FULL_JSON, 'r') as f: BILL_FULL_MAPPING = json.load(f) -PATH_TO_BILLS_META = settings.PATH_TO_BILLS_META +PATH_TO_BILLS_META = settings.PATH_TO_BILLS_META +PATH_TO_BILLS_META_GO = settings.PATH_TO_BILLS_META_GO PATH_TO_CONGRESSDATA_DIR = settings.CONGRESS_DATA_PATH PATH_TO_DATA_DIR = settings.PATH_TO_DATA_DIR PATH_TO_CONGRESSDATA_XML_DIR = settings.PATH_TO_CONGRESSDATA_XML_DIR diff --git a/server_py/flatgov/common/relatedBills.py b/server_py/flatgov/common/relatedBills.py index a633105..2d26110 100644 --- a/server_py/flatgov/common/relatedBills.py +++ b/server_py/flatgov/common/relatedBills.py @@ -177,11 +177,11 @@ def addSponsors(): def makeAndSaveRelatedBills(titlesIndex = loadTitlesIndex(), remake = False): if not os.path.isdir(PATH_TO_RELATEDBILLS_DIR): os.mkdir(PATH_TO_RELATEDBILLS_DIR) - logger.info('Adding same titles') + logger.info('Adding same titles (in stored json)') addSameTitles(titlesIndex=titlesIndex) - logger.info('Adding similar titles') + logger.info('Adding similar titles (in stored json)') addSimilarTitles(noYearTitlesIndex=loadTitlesIndex(titleIndexPath=PATH_TO_NOYEAR_TITLES_INDEX)) - logger.info('Adding related bills from GPO data') + logger.info('Adding related bills from GPO data (in stored json)') addGPORelatedBills() - logger.info('Adding sponsor info') + logger.info('Adding sponsor info (in Bill model of db)') addSponsors() diff --git a/server_py/flatgov/flatgov/dev.py b/server_py/flatgov/flatgov/dev.py index 6849f80..fbd00ed 100644 --- a/server_py/flatgov/flatgov/dev.py +++ b/server_py/flatgov/flatgov/dev.py @@ -40,6 +40,7 @@ PATH_TO_BILLS_META = os.path.join(BASE_DIR, 'billsMeta.json') +PATH_TO_BILLS_META_GO = os.path.join(BASE_DIR, 'billMetaGo.json') PATH_TO_CONGRESSDATA_DIR = CONGRESS_DATA_PATH PATH_TO_DATA_DIR = os.getenv('PATH_TO_DATA_DIR', os.path.join('/', *"/usr/local/share/xcential/public/data".split('/'))) PATH_TO_CONGRESSDATA_XML_DIR = os.getenv('PATH_TO_CONGRESSDATA_XML_DIR', os.path.join('/', *"/usr/local/share/xcential/public/data/116/dtd".split('/'))) diff --git a/server_py/flatgov/templates/bills/detail.html b/server_py/flatgov/templates/bills/detail.html index 9bb2d3c..7a48802 100644 --- a/server_py/flatgov/templates/bills/detail.html +++ b/server_py/flatgov/templates/bills/detail.html @@ -58,8 +58,9 @@

Committees and Cosponsors


Committees of Jurisdiction:
- House Administration - House Oversight + {% for committeeItem in committees_dict %} + {{committeeItem.committee}} + {% endfor %}