Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updating environment, grib2 writing support, version #38

Merged
merged 19 commits into from
Sep 30, 2021
Merged

Updating environment, grib2 writing support, version #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
994ce0d
Updated hsdata
djgagne Jan 20, 2021
fd8ef44
Added sspf eval script
djgagne Mar 8, 2021
37cd24e
Merge branch 'master' of github.com:djgagne/hagelslag into djgagne
djgagne Mar 8, 2021
1a0237e
Updated travis file
djgagne Mar 8, 2021
497beb1
Merge branch 'master' of github.com:djgagne/hagelslag into djgagne
djgagne Sep 13, 2021
0f62f69
Updated environment file and Grib2 writing to use Grib2io package ins…
djgagne Sep 22, 2021
c312c37
Updated requirements list and version.
djgagne Sep 22, 2021
ba11023
Updated environment.yml to install hagelslag too
djgagne Sep 29, 2021
72add6f
Merge branch 'master' of github.com:djgagne/hagelslag into djgagne
djgagne Sep 29, 2021
b0bb9a6
move workflow to github actions folder
djgagne Sep 29, 2021
d0ca345
Added jasper to environment to help with grib2io install
djgagne Sep 29, 2021
ecd9d03
Trying to install jasper with apt-get command in CI
djgagne Sep 29, 2021
35cb948
update with new package names to install
djgagne Sep 29, 2021
3f5f2bd
removed libjasper
djgagne Sep 29, 2021
0ef91f3
jasper in env
djgagne Sep 29, 2021
19f021f
Added way to install libjasper with ubuntu
djgagne Sep 29, 2021
6f3b7e5
Cleaned up code formatting with auto-format in PyCharm.
djgagne Sep 30, 2021
38a5455
Merge remote-tracking branch 'origin/djgagne' into djgagne
djgagne Sep 30, 2021
b994dfc
Addressing flake8 errors.
djgagne Sep 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions python-package-conda.yml → .github/workflows/python-package-conda.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@ jobs:
run: |
# $CONDA is an environment variable pointing to the root of the miniconda directory
echo $CONDA/bin >> $GITHUB_PATH
- name: Install grib dependencies
run: |
sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu xenial-security main"
sudo apt update
sudo apt install libjasper1 libjasper-dev
ldconfig -p | grep libjasper
- name: Install dependencies
run: |
conda env update --file environment.yml --name base
Expand Down
14 changes: 0 additions & 14 deletions .travis.yml
View file
Open in desktop

This file was deleted.

119 changes: 61 additions & 58 deletions bin/hsdata
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
#!/usr/bin/env python
import argparse, pdb
from multiprocessing import Pool
from hagelslag.util.Config import Config
from hagelslag.processing.TrackProcessing import TrackProcessor
from hagelslag.util.make_proj_grids import read_ncar_map_file
from hagelslag.util.create_sector_grid_data import SectorProcessor
from datetime import timedelta
import pandas as pd
import numpy as np
import argparse
import os
import traceback
from netCDF4 import Dataset, date2num
from datetime import timedelta
from multiprocessing import Pool
from os.path import join, exists

import numpy as np
import pandas as pd
from netCDF4 import Dataset

from hagelslag.processing.TrackProcessing import TrackProcessor
from hagelslag.util.Config import Config


def main():
parser = argparse.ArgumentParser("hsdata - Hagelslag Data Processor")
Expand Down Expand Up @@ -126,13 +126,13 @@ def process_ensemble_member(run_date, member, config):
if config.train:
print("Find obs tracks", run_date, member)
mrms_tracks = track_proc.find_mrms_tracks()

print("Find model tracks", run_date, member)
if patch_radius is None:
model_tracks = track_proc.find_model_tracks()
else:
model_tracks = track_proc.find_model_patch_tracks()

if model_tracks:
print(run_date, member, "Found this many model tracks: {0:d}".format(len(model_tracks)))
print("Extract model attributes", run_date, member)
Expand All @@ -154,62 +154,65 @@ def process_ensemble_member(run_date, member, config):
track_errors = None
else:
track_pairings = track_proc.match_tracks(model_tracks, mrms_tracks,
unique_matches=config.unique_matches,
closest_matches=config.closest_matches)
unique_matches=config.unique_matches,
closest_matches=config.closest_matches)
track_proc.match_size_distributions(model_tracks, mrms_tracks, track_pairings)
track_errors = track_proc.calc_track_errors(model_tracks,
mrms_tracks,
track_pairings)
mrms_tracks,
track_pairings)
print("Output data", run_date, member)
forecast_data = make_forecast_track_data(model_tracks, run_date, member,
config, track_proc.model_grid.proj, mrms_tracks, track_errors)
config, track_proc.model_grid.proj, mrms_tracks,
track_errors)
if patch_radius is not None:
print("Output netCDF", run_date, member)
forecast_track_patches_to_netcdf(model_tracks, patch_radius, run_date, member, config)
if config.json:
print("Output json", run_date, member)
forecast_tracks_to_json(model_tracks, run_date, member, config, track_proc.model_grid.proj,
observed_tracks=mrms_tracks,
track_errors=track_errors)
observed_tracks=mrms_tracks,
track_errors=track_errors)
obs_data = make_obs_track_data(mrms_tracks, member, run_date, config, track_proc.model_grid.proj)
if config.json:
obs_tracks_to_json(mrms_tracks, member, run_date, config, track_proc.model_grid.proj)
print("Output csv", run_date, member)
for table_name, table_data in obs_data.items():
csv_filename = config.csv_path + "{0}_{1}_{2}_{3}.csv".format(table_name,
"obs",
member,
run_date.strftime(
config.run_date_format))
csv_filename = join(config.csv_path, "{0}_{1}_{2}_{3}.csv".format(table_name,
"obs",
member,
run_date.strftime(
config.run_date_format)))
table_data.to_csv(csv_filename,
na_rep="nan",
float_format="%0.5f",
index=False)
na_rep="nan",
float_format="%0.5f",
index=False)
os.chmod(csv_filename, 0o666)
else:
forecast_data = make_forecast_track_data(model_tracks, run_date, member, config,
track_proc.model_grid.proj)
track_proc.model_grid.proj)
if patch_radius is not None:
forecast_track_patches_to_netcdf(model_tracks, patch_radius, run_date, member, config)
if config.json:
forecast_tracks_to_json(model_tracks, run_date, member, config, track_proc.model_grid.proj)
elif len(model_tracks) > 0:
print(run_date, member, "Make Forecast Track Data")
forecast_data = make_forecast_track_data(model_tracks, run_date, member, config, track_proc.model_grid.proj)
forecast_data = make_forecast_track_data(model_tracks, run_date, member, config,
track_proc.model_grid.proj)
if patch_radius is not None:
print(run_date, member, "Track Data to netCDF")
forecast_track_patches_to_netcdf(model_tracks, patch_radius, run_date, member, config)
if config.json:
forecast_tracks_to_json(model_tracks, run_date, member, config, track_proc.model_grid.proj)
else:
print('No {0} {1} modeled tracks found'.format(run_date,member))
print('No {0} {1} modeled tracks found'.format(run_date, member))
forecast_data = {}

for table_name, table_data in forecast_data.items():
csv_filename = config.csv_path + "{0}_{1}_{2}_{3}.csv".format(table_name,
config.ensemble_name,
member,
run_date.strftime(config.run_date_format))
csv_filename = join(config.csv_path, "{0}_{1}_{2}_{3}.csv".format(table_name,
config.ensemble_name,
member,
run_date.strftime(
config.run_date_format)))
print("Output csv file " + csv_filename)
table_data.to_csv(csv_filename,
na_rep="nan",
Expand Down Expand Up @@ -264,7 +267,7 @@ def process_observed_tracks(run_date, member, config):
mrms_tracks = track_proc.find_mrms_tracks()
if len(mrms_tracks) > 0:
obs_data = make_obs_track_data(mrms_tracks, member, run_date, config, track_proc.model_grid.proj)
if config.json:
if config.json:
obs_tracks_to_json(mrms_tracks, member, run_date, config, track_proc.model_grid.proj)
# if not os.access(config.csv_path + run_date.strftime("%Y%m%d"), os.R_OK):
# try:
Expand Down Expand Up @@ -338,10 +341,10 @@ def rematch_ensemble_tracks(run_date, member, config):
obs_data = make_obs_track_data(mrms_tracks, member, run_date, config, track_proc.model_grid.proj,
)
for table_name, table_data in obs_data.items():
table_data.to_csv(config.csv_path + "{0}_{1}_{2}_{3}.csv".format(table_name,
"obs",
member,
run_date.strftime("%Y%m%d")),
table_data.to_csv(join(config.csv_path, "{0}_{1}_{2}_{3}.csv".format(table_name,
"obs",
member,
run_date.strftime("%Y%m%d"))),
na_rep="nan",
float_format="%0.5f",
index=False)
Expand All @@ -350,10 +353,10 @@ def rematch_ensemble_tracks(run_date, member, config):
else:
forecast_data = {}
for table_name, table_data in forecast_data.items():
table_data.to_csv(config.csv_path + "{0}_{1}_{2}_{3}.csv".format(table_name,
config.ensemble_name,
member,
run_date.strftime("%Y%m%d")),
table_data.to_csv(join(config.csv_path, "{0}_{1}_{2}_{3}.csv".format(table_name,
config.ensemble_name,
member,
run_date.strftime("%Y%m%d"))),
na_rep="nan",
float_format="%0.5f",
index=False)
Expand Down Expand Up @@ -483,11 +486,11 @@ def make_forecast_track_data(forecast_tracks, run_date, member, config, proj, ob
else:
if forecast_track.observations is not None:
num_labels = len(forecast_track.observations)
for l in range(num_labels):
for label in range(num_labels):
if config.label_type == "gamma":
hail_label = forecast_track.observations[l].loc[step].values.tolist()
hail_label = forecast_track.observations[label].loc[step].values.tolist()
else:
hail_label = [forecast_track.observations[l].loc[step, "Max_Hail_Size"]]
hail_label = [forecast_track.observations[label].loc[step, "Max_Hail_Size"]]
forecast_data['track_step'].loc[track_step_count] = record + hail_label
track_step_count += 1
else:
Expand Down Expand Up @@ -545,11 +548,11 @@ def forecast_tracks_to_json(forecast_tracks, run_date, member, config, proj, obs
except OSError:
print("directory already created")

json_filename = config.geojson_path + "/".join(full_path) + \
"/{0}_{1}_{2}_model_track_{3:03d}.json".format(ensemble_name,
run_date.strftime("%Y%m%d"),
member,
f)
json_filename = join(config.geojson_path, "/".join(full_path),
"{0}_{1}_{2}_model_track_{3:03d}.json".format(ensemble_name,
run_date.strftime("%Y%m%d"),
member,
f))
json_metadata = dict(id=track_id,
ensemble_name=ensemble_name,
ensemble_member=member,
Expand Down Expand Up @@ -662,7 +665,7 @@ def forecast_track_patches_to_netcdf(forecast_tracks, patch_radius, run_date, me
if hasattr(config, "future_variables"):
for f_variable in config.future_variables:
out_file.variables[f_variable + "_future"][:] = np.vstack([f_track.attributes[f_variable + "-future"]
for f_track in forecast_tracks])
for f_track in forecast_tracks])
if config.train:
for label_column in label_columns:
try:
Expand All @@ -672,20 +675,20 @@ def forecast_track_patches_to_netcdf(forecast_tracks, patch_radius, run_date, me
out_file.variables[label_column][:] = 0

# Save configuration dictionary as global attributes.
for k,v in config.__dict__.items():
for k, v in config.__dict__.items():
# Don't save attributes that are already netCDF varaible names
if k=="dates": continue
if k=="storm_variables": continue
if k=="potential_variables": continue
if k=="tendency_variables": continue
if k == "dates": continue
if k == "storm_variables": continue
if k == "potential_variables": continue
if k == "tendency_variables": continue
v = str(v)
# Don't clobber existing attribute.
if hasattr(out_file, k):
# If it exists already, add "config_" to beginning and recheck.
alt_key = "config_" + k
if hasattr(out_file, alt_key):
# If alternative key exists too, raise attribute error.
raise AttributeError("Can't save "+k+":"+" "+v+" as global attribute. It already exists.")
raise AttributeError("Can't save " + k + ":" + " " + v + " as global attribute. It already exists.")
else:
setattr(out_file, alt_key, v)
else:
Expand Down
Loading