diff --git a/Dockerfile b/Dockerfile index 6e3ffde..7b06ae6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,4 @@ RUN conda install -y -c conda-forge satpy=0.29.0 netcdf4=1.5.6 h5py=3.2.1 matplo COPY . . -CMD ["python", "oco_vistool.py"] \ No newline at end of file +CMD ["python", "oco_vistool.py"] diff --git a/Encoding.csv b/Encoding.csv index b651649..df669e0 100644 --- a/Encoding.csv +++ b/Encoding.csv @@ -19,4 +19,4 @@ GOES-East_ABI_Band13_Clean_Infrared,16 GOES-West_ABI_Band13_Clean_Infrared,17 Himawari_AHI_Band13_Clean_Infrared,18 GOES-East_ABI_GeoColor,19 -GOES-West_ABI_GeoColor,20 \ No newline at end of file +GOES-West_ABI_GeoColor,20 diff --git a/README.md b/README.md index b57e2b6..c943e4f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # oco_vistool.py -The purpose of this script is to pull different RGB images (depending on the user's choice from Encoding.csv) from Worldview using the NASA GIBS API and overlay various OCO-2 data fields for -case study analysis in support of OCO-2 cloud and aerosol screening validation. +The purpose of this script is to pull different RGB images (depending on the user's choice from Encoding.csv) from Worldview using the NASA GIBS API and overlay various OCO-2/3 data fields for +case study analysis in support of OCO-2/3 cloud and aerosol screening validation. -It can be used as a command line tool to plot any data in the OCO-2 CO2 or SIF Lite file, filtered by warn level, quality flag, and/or footprint number where applicable. +It can be used as a command line tool to plot any data in the OCO-2/3 CO2 or SIF Lite file, filtered by warn level, quality flag, and/or footprint number where applicable. It can also be called as a function from within another Python program for expanded use with other datasets. ## GETTING STARTED @@ -17,24 +17,49 @@ It can also be called as a function from within another Python program for expan and comparing files. 2) Make sure you have the necessary system and Python requirements. - #### Option 1: DIY + #### Option 1: Conda Environment - ##### Python requirements: - Python 2.7 or higher - - For those new to Python, it is easiest to get the Anaconda distribution: - https://www.continuum.io/download and then use the Anaconda utility conda - to fetch and install packages. - - - The conda installs required on top of what comes with Anaconda 5.2 for both Python 2.7 and Python 3.6 are as follows: - + conda install -c conda-forge cartopy - + conda install -c conda-forge ncurses - + conda install -c conda-forge hdf5 - + conda install -c conda-forge future + ##### Miniconda Installation Instructions: + - Linux: https://www.cira.colostate.edu/wiki/miniconda-python-distribution-on-linux/ + - Mac: https://www.cira.colostate.edu/wiki/miniconda-python-distribution-on-mac/ + - Windows: https://www.cira.colostate.edu/wiki/miniconda-python-distribution-on-windows/ + + From the cloned directory, run `conda env create -f environment.yml` to create the conda environment. To activate the environment, use `conda activate oco_vistool` and to deactivate, use `conda deactivate`. More information about working with conda environments can be found at https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html #### Option 2: Docker Container - Dockerfiles and docker-compose files to set up an appropriate Python 2.7 or Python 3.6 container are included in the repo + The Dockerfile included in the repository can be used to create an appropriate Docker image and container environment. + + ##### Building the image + Build the image using the command `docker build -t oco_vistool:1.0 .` + Although the tag (`-t`) is optional, it will assign a name and version to + the resulting image which will make it easily locatable and also provide + a mechanism to have multiple versions available for testing and using + upgrades. Note the `.` assumes you are working in the directory that + contains the Dockerfile. To provide an alternate path, trade the period + out for `-f /path/to/the/Dockerfile` + + ##### Creating a container from the image + Although the default Docker behavior will create a container that runs + as root, it is best practice to declare your own user when you create + the container to avoid messy security issues. For compatability with + the host machine, you can use your user and preferred group ids in the + container, or if compatability is not an issue, you may simply create a new user:group identification for the container. To get your user and group id + on the host (provided that you are working in a linux-like environment), use + `id -u [user name]` and `id -g [group name]`. + + To create and run a container from the image, use + `docker run -it --name oco_vistool -u uid:gid -v /path/to/data/on/host:/path/to/data/in/the/container oco_vistool:1.0`, + adding -v flags before the image name as necessary to provide any data mounts + needed for the code to be able to access the necessary input data. + Again, the `--name` is optional but makes the container easily locatable. + + ##### Entering the container + The `-it` flag in the `docker build` command will start an interactive + session within the container, where you will be dropped in the + `/ECO1280_int_scripts` directory where the conversion scripts are located. + If the container is stopped (you can check by running `docker ps`), + start it by running `docker start oco_vistool` and then enter it + using `docker exec -it oco_vistool /bin/bash`. 3) Test run from the command line `python oco_vistool.py` @@ -77,7 +102,7 @@ It can also be called as a function from within another Python program for expan http://matplotlib.org/examples/color/named_colors.html **oco2_overlay_info** (dict, default: none): - The information for the OCO-2 variable to overlay on the background image. + The information for the OCO-2 variable to overlay on the Background image. **{** **file** (string): This is *required* for a variable overlay. @@ -142,6 +167,8 @@ It can also be called as a function from within another Python program for expan **out_background_name** (string, default example: MODIS_Aqua_CorrectedReflectance_TrueColor_Amazon_20150701.png): The name of the output background file, created if no overlay data is present or make_background_image is enabled. + + ## Other Information GIBS developer documentation: https://wiki.earthdata.nasa.gov/display/GIBS/GIBS+API+for+Developers @@ -156,7 +183,6 @@ The parser used a copy of the wmts capabilities file which was downloaded into t When either GOES or Himawari is chosen as the desired sensor in the config JSON file, there will appear a required key, such as the "files_loc". Two options (as of July 2021) for this key are: "aws" and "local". If the user wants to use the AWS S3 NOAA bucket as the source of background imagery, one would need to provide a Keys.csv file in the code directory. The file must have 2 columns with names: aws_access_key_id,aws_secret_access_key. Then, the user should provide each of the keys under the corresponding columns. To have these keys, one would need to create an AWS account and then generate both keys (or have an account with these keys being active). Full instruction on this process can be found in the "Programmatic access" part of the AWS documentation: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html - Minimum command line call: `python oco_vistool.py` @@ -166,8 +192,8 @@ Minimum function call: from oco_vistool import do_overlay_plot do_overlay_plot([lat_of_upper_lefthand_corner, lon_of_upper_lefthand_corner], - [lat_of_lower_righthand_corner, lon_of_lower_righthand_corner], - date, lat_data, lon_data, variable_data) + [lat_of_lower_righthand_corner, lon_of_lower_righthand_corner], + date, lat_data, lon_data, variable_data) ``` ## A Note about Questions, Suggestions, and Issues diff --git a/annotated_json_format.txt b/annotated_json_format.txt index 72e6aa2..0db454f 100644 --- a/annotated_json_format.txt +++ b/annotated_json_format.txt @@ -53,6 +53,7 @@ out_background_name: set to a specific filename to control the output background image filename, if make_background_image is enabled. Skipping this field is equivalent to setting it to an empty string, which means a default filename is constructed. + Overlay specification: Required Fields diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..bb63eed --- /dev/null +++ b/environment.yml @@ -0,0 +1,11 @@ +name: oco_vistool +channels: + - conda-forge +dependencies: + - python=3 + - netcdf4=1.5.7 + - h5py=2.10.0 + - pandas=1.1.4 + - cartopy=0.18.0 + - owslib=0.24.1 + - pillow=7.2.0 diff --git a/oco_vistool.py b/oco_vistool.py index 0cedfc1..6ba2d22 100644 --- a/oco_vistool.py +++ b/oco_vistool.py @@ -756,6 +756,8 @@ def load_OCO2_L1L2_overlay_data(ovr_d, load_view_geom=False): try: #print(h5[ovr_d['var_name']].attrs['Units'][0]) + #AJM: we probably need a better solution here. What would work for + # any version of python and h5py? dd['data_units'] = h5[ovr_d['var_name']].attrs['Units'][0]#.decode() except KeyError: # probably need a better solution here? @@ -1265,8 +1267,7 @@ def do_overlay_plot( ax2 = plt.subplot(gs[-1, 2:-2]) cb2 = mpl.colorbar.ColorbarBase(ax2, cmap=cmap2, norm=norm2, orientation = 'horizontal', ticks = ticks_list + [bounds_list[0], bounds_list[-1]]) - - + for t in cb2.ax.xaxis.get_ticklabels(): t.set_weight("bold") t.set_fontsize(8) @@ -1367,9 +1368,7 @@ def do_overlay_plot( fig.savefig(out_plot, dpi=150, bbox_inches='tight') print("\nFigure saved at "+out_plot) - ### Static Definitions - code_dir = os.path.dirname(os.path.realpath(__file__)) # dataframe with layers' names and their codes (generated for easier use by us) layers_encoding = pd.read_csv(code_dir + '/Encoding.csv', header = 0) diff --git a/oco_vistool_config.json b/oco_vistool_config.json index aca5929..efd755a 100644 --- a/oco_vistool_config.json +++ b/oco_vistool_config.json @@ -24,4 +24,4 @@ "out_plot_name": "", "out_data_dir": "", "out_data_name": "" -} \ No newline at end of file +} diff --git a/satpy_overlay_plots.py b/satpy_overlay_plots.py index ecb8c19..1bda210 100644 --- a/satpy_overlay_plots.py +++ b/satpy_overlay_plots.py @@ -89,6 +89,7 @@ def get_loc_ABI_files(datetime_utc, data_home, domain, platform, hour_offsets, b # four "?" for the MMSS, and then one more for a character at # the end, I am not sure about (frac second?) stimestamp = dt.strftime('%Y%j%H') + '????' + '?' + #print(data_home) ddir = os.path.join(data_home, y_dir, ymdd_dir, 'abi', 'L1b', 'Rad'+domain) if verbose: