Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update the database schema to handle new columns and line types added to the MET-11.0.0-beta3 output. issue #119 #126

Merged
merged 5 commits into from
Oct 21, 2022
Merged

Update the database schema to handle new columns and line types added to the MET-11.0.0-beta3 output. issue #119 #126

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2c34c4b
Add line_data_seeps and line_data_seeps_mpr
venitahagerty Oct 19, 2022
a999111
Added ecnt/crps_emp_fair, seeps, and seeps_mpr
venitahagerty Oct 20, 2022
f2d2a05
Added new line types to test_tables.py
venitahagerty Oct 20, 2022
88f7c2e
Added default value for ecnt/crpss_emp_fair
venitahagerty Oct 20, 2022
92151e6
Merge branch 'develop' into feature_119_new_seeps_ecnt
venitahagerty Oct 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 77 additions & 1 deletion METdbLoad/sql/mv_mysql.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,7 @@ CREATE TABLE line_data_ecnt
crps_emp DOUBLE,
crpscl_emp DOUBLE,
crpss_emp DOUBLE,
crpss_emp_fair DOUBLE,

CONSTRAINT line_data_ecnt_data_file_id_pk
FOREIGN KEY (data_file_id)
Expand Down Expand Up @@ -2240,6 +2241,82 @@ CREATE TABLE line_data_rps
) ENGINE = MyISAM
CHARACTER SET = latin1;

DROP TABLE IF EXISTS line_data_seeps;
CREATE TABLE line_data_seeps
(
stat_header_id INT UNSIGNED NOT NULL,
data_file_id INT UNSIGNED NOT NULL,
line_num INT UNSIGNED,
fcst_lead INT,
fcst_valid_beg DATETIME,
fcst_valid_end DATETIME,
fcst_init_beg DATETIME,
obs_lead INT UNSIGNED,
obs_valid_beg DATETIME,
obs_valid_end DATETIME,
total INT UNSIGNED,
s12 DOUBLE,
s13 DOUBLE,
s21 DOUBLE,
s23 DOUBLE,
s31 DOUBLE,
s32 DOUBLE,
pf1 DOUBLE,
pf2 DOUBLE,
pf3 DOUBLE,
pv1 DOUBLE,
pv2 DOUBLE,
pv3 DOUBLE,
mean_fcst DOUBLE,
mean_obs DOUBLE,
seeps DOUBLE,

CONSTRAINT line_data_seeps_data_file_id_pk
FOREIGN KEY (data_file_id)
REFERENCES data_file (data_file_id),
CONSTRAINT line_data_seeps_stat_header_id_pk
FOREIGN KEY (stat_header_id)
REFERENCES stat_header (stat_header_id),
INDEX stat_header_id_idx (stat_header_id)
) ENGINE = MyISAM
CHARACTER SET = latin1;

DROP TABLE IF EXISTS line_data_seeps_mpr;
CREATE TABLE line_data_seeps_mpr
(
stat_header_id INT UNSIGNED NOT NULL,
data_file_id INT UNSIGNED NOT NULL,
line_num INT UNSIGNED,
fcst_lead INT,
fcst_valid_beg DATETIME,
fcst_valid_end DATETIME,
fcst_init_beg DATETIME,
obs_lead INT UNSIGNED,
obs_valid_beg DATETIME,
obs_valid_end DATETIME,
obs_sid VARCHAR(32),
obs_lat DOUBLE,
obs_lon DOUBLE,
fcst DOUBLE,
obs DOUBLE,
obs_qc VARCHAR(32),
fcst_cat INT UNSIGNED,
obs_cat INT UNSIGNED,
p1 DOUBLE,
p2 DOUBLE,
t1 DOUBLE,
t2 DOUBLE,
seeps DOUBLE,

CONSTRAINT line_data_seeps_mpr_data_file_id_pk
FOREIGN KEY (data_file_id)
REFERENCES data_file (data_file_id),
CONSTRAINT line_data_seeps_mpr_stat_header_id_pk
FOREIGN KEY (stat_header_id)
REFERENCES stat_header (stat_header_id),
INDEX stat_header_id_idx (stat_header_id)
) ENGINE = MyISAM
CHARACTER SET = latin1;

DROP TABLE IF EXISTS tcst_header;
CREATE TABLE tcst_header
Expand All @@ -2257,7 +2334,6 @@ CREATE TABLE tcst_header
valid_mask VARCHAR(100),
PRIMARY KEY (tcst_header_id)


) ENGINE = MyISAM
CHARACTER SET = latin1;
CREATE INDEX tcst_header_unique_pk ON tcst_header (
Expand Down
3 changes: 2 additions & 1 deletion METdbLoad/tests/test_tables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ def count_rows(query2, query3):
"fho", "grad", "isc", "mctc", "mcts", "mpr", "nbrcnt",
"nbrctc", "nbrcts", "orank", "pct", "perc", "phist",
"pjc", "prc", "pstd", "relp", "rhist", "rps", "sl1l2",
"sal1l2", "vl1l2", "val1l2", "ssidx", "ssvar", "vcnt"]
"sal1l2", "vl1l2", "val1l2", "seeps", "seeps_mpr",
"ssidx", "ssvar", "vcnt"]

for ltype in line_types:

Expand Down
23 changes: 18 additions & 5 deletions METdbLoad/ush/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@
DMAP = "DMAP" # Distance Map
RPS = "RPS" # Ranked Probability Score
SSIDX = "SSIDX" # SKILL SCORE INDEX
SEEPS = "SEEPS" # Stable Equitable Error in Probability Space (SEEPS) score
SEEPS_MPR = "SEEPS_MPR" # SEEPS Matched Pair Data

# VSDB line types
BSS = "BSS" # same as PSTD
Expand All @@ -144,7 +146,7 @@
UC_LINE_TYPES = [FHO, CTC, CTS, MCTC, MCTS, CNT, SL1L2, SAL1L2, VL1L2, VAL1L2,
PCT, PSTD, PJC, PRC, ECLV, MPR, NBRCTC, NBRCTS, NBRCNT, ISC,
RHIST, PHIST, ORANK, SSVAR, GRAD, VCNT, RELP, ECNT, ENSCNT, PERC,
DMAP, RPS, SSIDX]
DMAP, RPS, SSIDX, SEEPS, SEEPS_MPR]

UC_LINE_TYPES_TCST = [TCMPR, PROBRIRW]

Expand All @@ -155,7 +157,7 @@

LINE_TABLES_TCST = ['line_data_' + hname for hname in LC_LINE_TYPES_TCST]

ALPHA_LINE_TYPES = [CTS, NBRCTS, NBRCNT, MCTS, SSVAR, VCNT, DMAP, RPS, CNT,
ALPHA_LINE_TYPES = [CTS, NBRCTS, NBRCNT, MCTS, SSVAR, VCNT, DMAP, RPS, CNT,
PSTD, SSIDX]

COV_THRESH_LINE_TYPES = [NBRCTC, NBRCTS, PCT, PSTD, PJC, PRC]
Expand Down Expand Up @@ -424,7 +426,8 @@
LINE_DATA_FIELDS[ECNT] = TOT_LINE_DATA_FIELDS + \
['n_ens', 'crps', 'crpss', 'ign', 'me', 'rmse', 'spread',
'me_oerr', 'rmse_oerr', 'spread_oerr', 'spread_plus_oerr',
'crpscl', 'crps_emp', 'crpscl_emp', 'crpss_emp']
'crpscl', 'crps_emp', 'crpscl_emp', 'crpss_emp',
'crps_emp_fair']

LINE_DATA_FIELDS[ENSCNT] = ALL_LINE_DATA_FIELDS + \
['rpsf', 'rpsf_ncl', 'rpsf_ncu', 'rpsf_bcl', 'rpsf_bcu',
Expand Down Expand Up @@ -533,7 +536,17 @@

LINE_DATA_FIELDS[SAL1L2] = TOT_LINE_DATA_FIELDS + \
['fabar', 'oabar', 'foabar', 'ffabar', 'ooabar', 'mae']


LINE_DATA_FIELDS[SEEPS] = TOT_LINE_DATA_FIELDS + \
['s12', 's13', 's21', 's23', 's31', 's32',
'pf1', 'pf2', 'pf3', 'pv1', 'pv2', 'pv3',
'mean_fcst', 'mean_obs', 'seeps']

LINE_DATA_FIELDS[SEEPS_MPR] = ALL_LINE_DATA_FIELDS + \
['obs_sid', 'obs_lat', 'obs_lon',
'fcst', 'obs', 'obs_qc', 'fcst_cat', 'obs_cat',
'p1', 'p2', 't1', 't2', 'seeps']

LINE_DATA_FIELDS[SSIDX] = ALL_LINE_DATA_FIELDS + \
[ALPHA, 'fcst_model', 'ref_model', 'n_init', 'n_term',
'v_vld', 'ss_index']
Expand Down Expand Up @@ -571,7 +584,7 @@
'dir_err', 'dir_err_bcl', 'dir_err_bcu',
'dir_abserr', 'dir_abserr_bcl', 'dir_abserr_bcu',
'anom_corr', 'anom_corr_ncl', 'anom_corr_ncu',
'anom_corr_bcl', 'anom_corr_bcu', 'anom_corr_uncntr',
'anom_corr_bcl', 'anom_corr_bcu', 'anom_corr_uncntr',
'anom_corr_uncntr_bcl', 'anom_corr_uncntr_bcu']

COLUMNS[TCMPR] = ['total', 'index_pair', 'level', 'watch_warn', 'initials', 'alat',
Expand Down