Feature/bill committee (#301)

* Add committees to billjson metadata

* Use committee data

UPDATES_CELERY.adoc

@@ -207,9 +207,9 @@ Celery tasks:
 
 uscongress:: update uscongress bill and metadata
 statementAdminPolicy:: updates statements of administration policy from the current White House (OMB) website
-committeereport:: TODO
-cbo:: TODO
-crs:: TODO
+committeereport:: update committee reports associated with bills 
+cbo:: update cbo reports associated with bills 
+crs:: update crs reports associated with bills 
 
 #### US Congress Task
 

server_py/flatgov/bills/handler.py

@@ -38,13 +38,8 @@ def get_bill(self, *args, **kwargs):
             'sponsor': sponsor,
             'cosponsors_dict': cosponsors_dict,
         }
-        # bill = Bill.objects.get_or_create(
-        #     bill_congress_type_number=self.congress,
-        #     default=bill_data
-        # )
         return bill_data
 
     def get_cosponsors(self, *args, **kwargs):
-        bill = kwargs.get('bill')
         cosponsors = self.data.get('cosponsors')
         return cosponsors

server_py/flatgov/bills/migrations/0002_bill_committees_dict.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.8 on 2021-04-09 23:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bills', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='bill',
+            name='committees_dict',
+            field=models.JSONField(default=list),
+        ),
+    ]

server_py/flatgov/bills/models.py

@@ -24,6 +24,7 @@ class Bill(models.Model):
     related_bills = models.JSONField(default=list)
     related_dict = models.JSONField(default=dict)
     cosponsors_dict = models.JSONField(default=list)
+    committees_dict = models.JSONField(default=list)
     es_similarity = models.JSONField(default=list)
     es_similar_bills_dict = models.JSONField(default=dict)
 

server_py/flatgov/bills/views.py

@@ -125,6 +125,7 @@ def get_qs_related_bill(self):
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context['cosponsors_dict'] = self.get_cosponsors_dict()
+        context['committees_dict'] = self.object.committees_dict
         context['cosponsors'] = self.get_cosponsors()
         context['statements'] = self.get_related_statements()
         context['committees'] = self.get_related_committees()

server_py/flatgov/common/billdata.py

@@ -5,6 +5,7 @@
 import sys, os, argparse, logging, re, json, gzip
 from typing import Dict
 from functools import reduce
+from bills.models import Bill
 
 from common import constants, utils
 
@@ -191,13 +192,56 @@ def loadBillsMeta(billMetaPath = constants.PATH_TO_BILLS_META, zip = True):
 
   return billsMeta
 
-def saveBillsMeta(billsMeta: Dict, metaPath = constants.PATH_TO_BILLS_META, zip = True):
+def saveBillsMeta(billsMeta: Dict, metaPath = constants.PATH_TO_BILLS_META, zip = False):
   with open(metaPath, 'w') as f:
     json.dump(billsMeta, f)
     if zip:
       with gzip.open(metaPath + '.gz', 'wt', encoding="utf-8") as zipfile:
         json.dump(billsMeta, zipfile)
 
+def saveBillsMetaToDb():
+  billsMeta = loadBillsMeta(billMetaPath = constants.PATH_TO_BILLS_META_GO, zip = False)
+  for billnumber, billdata in billsMeta.items():
+    billCongressTypeNumber = billdata.get('bill_congress_type_number','') 
+    if not billCongressTypeNumber:
+      continue
+    billNumberMatch = constants.BILL_NUMBER_REGEX_COMPILED.match(billCongressTypeNumber)
+    [congress, billType, numberOfBill, billVersion, billTypeNumber] = ["" for x in range(5)]
+    if billNumberMatch and billNumberMatch.groups():
+      [congress, billType, numberOfBill, billVersion] = billNumberMatch.groups()
+    else:
+      continue
+    print('Loading: ' + billCongressTypeNumber)
+    metadata = {
+      'type': billType,
+      'congress': int(congress),
+      'number': numberOfBill, 
+    }
+
+    for key, value in metadata.items():
+      if value:
+        billdata[key] = value 
+
+    billdata['cosponsors_dict'] = billdata.get('cosponsors', [])
+    billdata['committees_dict'] = billdata.get('committees', [])
+    keys = billdata.keys()
+    if 'cosponsors' in keys:
+      del billdata['cosponsors']
+    if 'committees' in keys:
+      del billdata['committees']
+
+    # Avoid not null constraint
+    if not billdata.get('related_bills'):
+      billdata['related_bills'] = []
+
+    if not billdata.get('cosponsors_dict'):
+      billdata['cosponsors_dict'] = []
+
+    if not billdata.get('committees_dict'):
+      billdata['committees_dict'] = []
+
+    Bill.objects.update_or_create(bill_congress_type_number=billnumber, defaults=billdata)
+
 def updateBillsMeta(billsMeta= {}):
   def addToBillsMeta(dirName: str, fileName: str):
     billDict = loadJSON(os.path.join(dirName, fileName))
@@ -216,7 +260,9 @@ def addToBillsMeta(dirName: str, fileName: str):
     billsMeta[billCongressTypeNumber]['titles'] = [title.get('title') for title in titles]
     billsMeta[billCongressTypeNumber]['titles_whole_bill'] = [title.get('title') for title in titles if not title.get('is_for_portion')]
     cosponsors = getCosponsors(fileDict=billDict, includeFields=['name', 'bioguide_id'])
+    committees = billDict.get('committees', [])
     billsMeta[billCongressTypeNumber]['cosponsors'] = cosponsors
+    billsMeta[billCongressTypeNumber]['committees'] = committees
 
     # TODO convert bill_id to billnumber
     billsMeta[billCongressTypeNumber]['related_bills'] = billDict.get('related_bills')

server_py/flatgov/common/constants.py

@@ -34,7 +34,8 @@
   with open(PATH_BILL_FULL_JSON, 'r') as f:
     BILL_FULL_MAPPING = json.load(f)
 
-PATH_TO_BILLS_META = settings.PATH_TO_BILLS_META
+PATH_TO_BILLS_META = settings.PATH_TO_BILLS_META 
+PATH_TO_BILLS_META_GO = settings.PATH_TO_BILLS_META_GO 
 PATH_TO_CONGRESSDATA_DIR = settings.CONGRESS_DATA_PATH
 PATH_TO_DATA_DIR = settings.PATH_TO_DATA_DIR
 PATH_TO_CONGRESSDATA_XML_DIR = settings.PATH_TO_CONGRESSDATA_XML_DIR

server_py/flatgov/common/relatedBills.py

@@ -177,11 +177,11 @@ def addSponsors():
 def makeAndSaveRelatedBills(titlesIndex = loadTitlesIndex(), remake = False):
     if not os.path.isdir(PATH_TO_RELATEDBILLS_DIR):
         os.mkdir(PATH_TO_RELATEDBILLS_DIR)
-    logger.info('Adding same titles')
+    logger.info('Adding same titles (in stored json)')
     addSameTitles(titlesIndex=titlesIndex)
-    logger.info('Adding similar titles')
+    logger.info('Adding similar titles (in stored json)')
     addSimilarTitles(noYearTitlesIndex=loadTitlesIndex(titleIndexPath=PATH_TO_NOYEAR_TITLES_INDEX))
-    logger.info('Adding related bills from GPO data')
+    logger.info('Adding related bills from GPO data (in stored json)')
     addGPORelatedBills()
-    logger.info('Adding sponsor info')
+    logger.info('Adding sponsor info (in Bill model of db)')
     addSponsors()

server_py/flatgov/flatgov/dev.py

@@ -40,6 +40,7 @@
 
 
 PATH_TO_BILLS_META = os.path.join(BASE_DIR, 'billsMeta.json')
+PATH_TO_BILLS_META_GO = os.path.join(BASE_DIR, 'billMetaGo.json')
 PATH_TO_CONGRESSDATA_DIR = CONGRESS_DATA_PATH
 PATH_TO_DATA_DIR = os.getenv('PATH_TO_DATA_DIR', os.path.join('/', *"/usr/local/share/xcential/public/data".split('/')))
 PATH_TO_CONGRESSDATA_XML_DIR = os.getenv('PATH_TO_CONGRESSDATA_XML_DIR', os.path.join('/', *"/usr/local/share/xcential/public/data/116/dtd".split('/')))

server_py/flatgov/templates/bills/detail.html

@@ -58,8 +58,9 @@ <h4 class="text-center"><b>Committees and Cosponsors</b></h4>
                 <br>
                 <div class="d-flex justify-content-between">
                     <h5 style="float: left;"><b>Committees of Jurisdiction:</b></h5>
-                    <a href="#" class="link-secondary">House Administration</a>
-                    <a href="#" class="link-secondary">House Oversight</a>
+                    {% for committeeItem in committees_dict %}
+                    <a href="#" class="link-secondary">{{committeeItem.committee}}</a>
+                    {% endfor %}
                 </div>
 
                 <div class="mt-5">