Fix jupytext notebook load error. (#366)

* Fix jupytext notebook load error. * Update jupytext version in all the notebooks * Set notebook interpreter to python3.7 * Update start.sh and README * Remove incompatible notebook deps functools32 and futures * Replace google.datalab with google.cloud for bigquery in notebooks * Using google-cloud-bigquery because datalab in python3.7 yields syntax error (see googledatalab/pydatalab#711) [ci skip]
all-of-us · Dec 27, 2019 · 743c86b · 743c86b
1 parent 712f219
commit 743c86b
Show file tree

Hide file tree

Showing 25 changed files with 90 additions and 103 deletions.
diff --git a/data_steward/analytics/cdr_ops/conformance.py b/data_steward/analytics/cdr_ops/conformance.py
@@ -4,12 +4,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.3
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # # Conformance

diff --git a/data_steward/analytics/cdr_ops/coverage.py b/data_steward/analytics/cdr_ops/coverage.py
@@ -4,12 +4,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.3
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 import warnings

diff --git a/data_steward/analytics/cdr_ops/curation_dashboard.py b/data_steward/analytics/cdr_ops/curation_dashboard.py
@@ -4,12 +4,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.3
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # +

diff --git a/data_steward/analytics/cdr_ops/deid_race.py b/data_steward/analytics/cdr_ops/deid_race.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# + {}
+# +
 from notebooks import bq, render, parameters
 import pandas as pd
 

diff --git a/data_steward/analytics/cdr_ops/ehr_operations.py b/data_steward/analytics/cdr_ops/ehr_operations.py
@@ -4,12 +4,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.3
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # # EHR Operations

diff --git a/data_steward/analytics/cdr_ops/list_tables_with_duplicate_domain_ids.py b/data_steward/analytics/cdr_ops/list_tables_with_duplicate_domain_ids.py
@@ -4,12 +4,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.3
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 import google.datalab.bigquery as bq

diff --git a/data_steward/analytics/table_metrics/Table_Metrics_part_1.py b/data_steward/analytics/table_metrics/Table_Metrics_part_1.py
@@ -1,16 +1,15 @@
 # ---
 # jupyter:
 #   jupytext:
-#     formats: ipynb,py:light
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.1
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # +
@@ -1665,4 +1664,3 @@ def cstr(s, color='black'):
 success_rate
 
 success_rate.to_csv("data\\concept.csv")
-# -
diff --git a/data_steward/analytics/table_metrics/Table_Metrics_part_2.py b/data_steward/analytics/table_metrics/Table_Metrics_part_2.py
@@ -1,16 +1,15 @@
 # ---
 # jupyter:
 #   jupytext:
-#     formats: ipynb,py:light
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.1
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # +

diff --git a/data_steward/analytics/table_metrics/Table_Metrics_part_3.py b/data_steward/analytics/table_metrics/Table_Metrics_part_3.py
@@ -1,16 +1,16 @@
 # ---
 # jupyter:
 #   jupytext:
-#     formats: ipynb,py:light
+#     cell_metadata_json: true
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.1
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # + {"pycharm": {"name": "#%%\n"}}
@@ -222,6 +222,7 @@ def cstr(s, color='black'):
 # + {"pycharm": {"name": "#%%\n"}}
 
 
+
 # + {"pycharm": {"name": "#%%\n"}}
 total_wrong = temporal_df['wrong_date_rows'].sum()
 total_wrong
@@ -300,6 +301,7 @@ def cstr(s, color='black'):
 # + {"pycharm": {"name": "#%%\n"}}
 
 
+
 # + {"pycharm": {"name": "#%%\n"}}
 total_wrong = temporal_df['wrong_date_rows'].sum()
 total_wrong
@@ -1221,7 +1223,10 @@ def cstr(s, color='black'):
 # + {"pycharm": {"name": "#%%\n"}}
 
 
+
 # + {"pycharm": {"name": "#%%\n"}}
 
 
+
 # + {"pycharm": {"name": "#%%\n"}}
+
diff --git a/data_steward/analytics/table_metrics/Table_Metrics_part_4.py b/data_steward/analytics/table_metrics/Table_Metrics_part_4.py
@@ -1,12 +1,11 @@
 # ---
 # jupyter:
 #   jupytext:
-#     formats: ipynb,py:light
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.3'
-#       jupytext_version: 1.0.5
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
 #     display_name: Python 3
 #     language: python
@@ -2329,10 +2328,8 @@ def cstr(s, color='black'):
     40163765, 40165789, 19125668, 40165776, 40165769, 40171412, 19127398, 19013174, 40163759, 974169, 1310756, 40165755,
     19129688, 40163763)
 
-# +
 drugs = antibiotics + ccb + diuretics + opioids + statins + msknaids + oralhypo + painnsaids + vaccine + aceinhibitor
 
-# -
 
 # ## Antibiotics
 

diff --git a/data_steward/analytics/table_metrics/Table_Metrics_part_5.py b/data_steward/analytics/table_metrics/Table_Metrics_part_5.py
@@ -1,12 +1,11 @@
 # ---
 # jupyter:
 #   jupytext:
-#     formats: ipynb,py:light
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.3'
-#       jupytext_version: 1.0.5
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
 #     display_name: Python 3
 #     language: python

diff --git a/data_steward/analytics/table_metrics/Table_Metrics_part_6.py b/data_steward/analytics/table_metrics/Table_Metrics_part_6.py
@@ -1,12 +1,11 @@
 # ---
 # jupyter:
 #   jupytext:
-#     formats: ipynb,py:light
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.3'
-#       jupytext_version: 1.0.5
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
 #     display_name: Python 3
 #     language: python

diff --git a/data_steward/analytics/tools/concept_success_rates_visualizations.py b/data_steward/analytics/tools/concept_success_rates_visualizations.py
@@ -5,12 +5,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.4
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # NOTES:
@@ -414,4 +414,3 @@
 
 img_name = name_of_interest + "_concept_success_line_graph.jpg"
 # plt.savefig(img_name, bbox_extraartist=(lgd,), bbox_inches='tight')
-# -
diff --git a/data_steward/analytics/tools/data_after_death_visualizations.py b/data_steward/analytics/tools/data_after_death_visualizations.py
@@ -5,12 +5,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.4
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # NOTES:
@@ -393,4 +393,3 @@
 
 img_name = name_of_interest + "_data_after_death_line_graph.jpg"
 # plt.savefig(img_name, bbox_extraartist=(lgd,), bbox_inches='tight')
-# -
diff --git a/data_steward/analytics/tools/duplicates_visualizations.py b/data_steward/analytics/tools/duplicates_visualizations.py
@@ -5,12 +5,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.4
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # NOTES:
@@ -387,4 +387,3 @@
 
 img_name = name_of_interest + "_duplicates_line_graph.jpg"
 # plt.savefig(img_name, bbox_extraartist=(lgd,), bbox_inches='tight')
-# -
diff --git a/data_steward/analytics/tools/email_generator.py b/data_steward/analytics/tools/email_generator.py
@@ -297,7 +297,7 @@ def print_error_info(error_dict, starting_msg, percent,
 ###########################################################################
 # No longer defining functions                                            #
 # We are now doing everything that comes BEFORE we print the message      #
-###########################################################################
+# ##########################################################################
 
 
 # 1. Loading the files
@@ -372,7 +372,7 @@ def print_error_info(error_dict, starting_msg, percent,
 # Now we are actually printing out the 'problem' information for the site #
 # This is admittedly clunky but I decided to not implement a standard     #
 # function in order to allow the text feel more 'natural.'                #
-###########################################################################
+# ##########################################################################
 
 
 # same for all of the sites

diff --git a/data_steward/analytics/tools/end_dates_before_start_dates_visualizations.py b/data_steward/analytics/tools/end_dates_before_start_dates_visualizations.py
@@ -5,12 +5,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.4
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # NOTES:
@@ -363,4 +363,3 @@
 
 img_name = name_of_interest + "_end_before_start_data_line_graph.jpg"
 # plt.savefig(img_name, bbox_extraartist=(lgd,), bbox_inches='tight')
-# -
diff --git a/data_steward/analytics/tools/ingredient_usage_visualizations.py b/data_steward/analytics/tools/ingredient_usage_visualizations.py
@@ -5,12 +5,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.4
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # NOTES:
@@ -432,4 +432,3 @@
 
 img_name = name_of_interest + "_drug_integration_line_graph.jpg"
 # plt.savefig(img_name, bbox_extraartist=(lgd,), bbox_inches='tight')
-# -
diff --git a/data_steward/analytics/tools/measurement_integration_visualizations.py b/data_steward/analytics/tools/measurement_integration_visualizations.py
@@ -5,12 +5,12 @@
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: '1.4'
-#       jupytext_version: 1.2.4
+#       format_version: '1.5'
+#       jupytext_version: 1.3.0
 #   kernelspec:
-#     display_name: Python 2
+#     display_name: Python 3
 #     language: python
-#     name: python2
+#     name: python3
 # ---
 
 # NOTES:
@@ -401,4 +401,3 @@
 
 img_name = name_of_interest + "_measurement_integration_line_graph.jpg"
 # plt.savefig(img_name, bbox_extraartist=(lgd,), bbox_inches='tight')
-# -