Skip to content

Commit 5ce0b60

Browse files
arulalantarulalant
committed
2 parents db23429 + 559fd59 commit 5ce0b60

5 files changed

+98
-69
lines changed

‎bsubScripts/ncumeps_global_tigge/tigge_create_tarball_g2files.py

+31-20
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,16 @@
2222

2323
filesCount = {'ttr': 41, 'lsm': 41, 'orog': 41, '10v': 41, 'tcc': 41, 'gh': 369, 'skt': 41, 'tp': 41, 'msl': 41, 'mx2t6': 40, '2d': 41, '10u': 41, 'mn2t6': 40, 'sshf': 41, 'slhf': 41, 'ssr': 41, '2t': 41, 'sp': 41, 'st': 41, 'q': 328, 'u': 328, 't': 328, 'str': 41, 'v': 328, 'sd': 41}
2424

25+
dirsOrder = [
26+
'gh', 'u', 'v', 'q', 't', '10u', '10v', '2t', 'mx2t6', 'mn2t6', 'skt', 'st',
27+
'2d', 'sp', 'msl', 'tp', 'ttr', 'lsm', 'tcc', 'slhf', 'ssr', 'sshf', 'str', 'sd', 'orog'
28+
]
29+
# merge cmd into single grib2 of each members
30+
catcmd = ['%s/z_tigge_c_dems*%s' % (d,d) for d in dirsOrder]
31+
catcmd = ' '.join(catcmd)
32+
catcmd = 'cat %s ' % catcmd
33+
catcmd += ' > %s'
34+
2535
def createTarBalls(path, today, member):
2636

2737
member = str(member).zfill(3)
@@ -46,8 +56,8 @@ def createTarBalls(path, today, member):
4656
cmd = tigge_check + ' -v -w %s/*' % tgf
4757
tigge_check_val = os.system(cmd) # it should return 0 on pass
4858
if tigge_check_val != 0 :
49-
print "Error : While checking via tigge_check cmd got error!"
50-
sys.exit(0)
59+
print "WARNING : While checking via tigge_check cmd got error!"
60+
#sys.exit(0)
5161
# end of for tgf in os.listdir('.'):
5262

5363
tDay = datetime.datetime.strptime(today, "%Y%m%d")
@@ -60,34 +70,35 @@ def createTarBalls(path, today, member):
6070

6171
tardir = '../../TarFiles/%s' % today
6272
if not os.path.exists(tardir): os.makedirs(tardir)
63-
tarfile = 'ncmrwf_tigge_%s_%s.tar.gz' % (today, member)
73+
mergedg2file = 'ncmrwf_dems_tigge_%s_%s.grib2' % (today, member)
74+
mergedg2filepath = os.path.join(tardir, mergedg2file)
6475
print "currnet path : ", os.getcwd()
65-
# normal "$ tar cvjf fcst_20160223.tar.bz2 *fcst*grb2" cmd takes 6 minutes 43 seconds.
66-
#
67-
# where as in parallel bz2, "$ tar -c *fcst*grb2 | pbzip2 -v -c -f -p32 -m500 > fcst_20160223_parallel.tar.bz2" cmd takes only just 23 seconds alone, with 32 processors and 500MB RAM memory.
68-
#
69-
# create analysis files tar file in parallel # -m500 need to be include for pbzip2
70-
tg_files = ' '.join([' -C %s . ' % os.path.join(inpath, tgf) for tgf in os.listdir('.')])
71-
72-
cmd = "tar -c %s | %s -v -c -f -p32 > %s/%s" % (tg_files, pigz, tardir, tarfile)
73-
74-
print cmd
75-
subprocess.call(cmd, shell=True)
76-
76+
# merge all the params, all the levels, all the time steps, but individual members
77+
# into single grib2 (BIG) file.
78+
catcmd_out = catcmd % mergedg2filepath
79+
subprocess.call(catcmd_out, shell=True)
80+
time.sleep(30)
81+
# Lets compress single BIG grib2 file by using gz compress cmd.
82+
os.chdir(tardir)
83+
gzip_cmd = '%s -9 -p 32 %s' % (pigz, mergedg2file)
84+
print "gzip_cmd = ", gzip_cmd
85+
subprocess.call(gzip_cmd, shell=True)
86+
time.sleep(5)
87+
7788
# remove yesterday directory!!!
78-
y4DayPath = os.path.join(path, '../../%s' % y4Day)
79-
if os.path.exists(y4DayPath):
80-
cmd = "rm -rf %s" % y4DayPath
89+
yDayPath = os.path.join(path, '../../%s' % yDay)
90+
if os.path.exists(yDayPath):
91+
cmd = "rm -rf %s" % yDayPath
8192
print cmd
8293
subprocess.call(cmd, shell=True)
83-
# end of if os.path.exists(y4DayPath):
94+
# end of if os.path.exists(yDayPath):
8495

8596
os.chdir(cdir)
8697
# end of def createTarBalls(path, today, ...):
8798

8899
if __name__ == '__main__':
89100

90-
ftp_server="arulalan@ftp"
101+
ftp_server="prod@ftp"
91102
date = None
92103
member = '000'
93104
outpath = '/gpfs3/home/umeps/EPS/ShortJobs/NCUM_EPS_TIGGE/%s/'

‎bsubScripts/ncumeps_global_tigge/tigge_create_tarball_g2files_put_into_ftp-arulalan.py

+30-20
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,24 @@
1414
## Arulalan.T
1515
## 04-Mar-2016.
1616

17-
import os, subprocess, datetime, getopt, sys, glob
17+
import os, subprocess, datetime, getopt, sys, glob, time
1818

1919
pbzip2 = '/gpfs1/home/Libs/GNU/ZIPUTIL/pbzip2'
2020
pigz = '/gpfs1/home/Libs/GNU/ZIPUTIL/pigz'
2121
tigge_check = '/gpfs2/home/prasanna/SOFTWARE/GNU/grib_api-1.21.0-path/bin/tigge_check'
2222

2323
filesCount = {'ttr': 41, 'lsm': 41, 'orog': 41, '10v': 41, 'tcc': 41, 'gh': 369, 'skt': 41, 'tp': 41, 'msl': 41, 'mx2t6': 40, '2d': 41, '10u': 41, 'mn2t6': 40, 'sshf': 41, 'slhf': 41, 'ssr': 41, '2t': 41, 'sp': 41, 'st': 41, 'q': 328, 'u': 328, 't': 328, 'str': 41, 'v': 328, 'sd': 41}
2424

25+
dirsOrder = [
26+
'gh', 'u', 'v', 'q', 't', '10u', '10v', '2t', 'mx2t6', 'mn2t6', 'skt', 'st',
27+
'2d', 'sp', 'msl', 'tp', 'ttr', 'lsm', 'tcc', 'slhf', 'ssr', 'sshf', 'str', 'sd', 'orog'
28+
]
29+
# merge cmd into single grib2 of each members
30+
catcmd = ['%s/z_tigge_c_dems*%s' % (d,d) for d in dirsOrder]
31+
catcmd = ' '.join(catcmd)
32+
catcmd = 'cat %s ' % catcmd
33+
catcmd += ' > %s'
34+
2535
def createTarBalls(path, today, member):
2636

2737
member = str(member).zfill(3)
@@ -41,13 +51,13 @@ def createTarBalls(path, today, member):
4151

4252
cdir = os.getcwd()
4353
os.chdir(inpath)
44-
54+
4555
for tgf in os.listdir('.'):
46-
cmd = tigge_check + '-v -w %s/*' % tgf
56+
cmd = tigge_check + ' -v -w %s/*' % tgf
4757
tigge_check_val = os.system(cmd) # it should return 0 on pass
4858
if tigge_check_val != 0 :
49-
print "Error : While checking via tigge_check cmd got error!"
50-
sys.exit(0)
59+
print "WARNING : While checking via tigge_check cmd got error!"
60+
#sys.exit(0)
5161
# end of for tgf in os.listdir('.'):
5262

5363
tDay = datetime.datetime.strptime(today, "%Y%m%d")
@@ -60,24 +70,24 @@ def createTarBalls(path, today, member):
6070

6171
tardir = '../../TarFiles/%s' % today
6272
if not os.path.exists(tardir): os.makedirs(tardir)
63-
tarfile = 'ncmrwf_tigge_%s_%s.tar.gz' % (today, member)
73+
mergedg2file = 'ncmrwf_dems_tigge_%s_%s.grib2' % (today, member)
74+
mergedg2filepath = os.path.join(tardir, mergedg2file)
6475
print "currnet path : ", os.getcwd()
65-
# normal "$ tar cvjf fcst_20160223.tar.bz2 *fcst*grb2" cmd takes 6 minutes 43 seconds.
66-
#
67-
# where as in parallel bz2, "$ tar -c *fcst*grb2 | pbzip2 -v -c -f -p32 -m500 > fcst_20160223_parallel.tar.bz2" cmd takes only just 23 seconds alone, with 32 processors and 500MB RAM memory.
68-
#
69-
# create analysis files tar file in parallel # -m500 need to be include for pbzip2
70-
tg_files = ' '.join([' -C %s . ' % os.path.join(inpath, tgf) for tgf in os.listdir('.')])
71-
72-
cmd = "tar -c %s | %s -v -c -f -p32 > %s/%s" % (tg_files, pigz, tardir, tarfile)
73-
74-
print cmd
75-
subprocess.call(cmd, shell=True)
76-
76+
# merge all the params, all the levels, all the time steps, but individual members
77+
# into single grib2 (BIG) file.
78+
catcmd_out = catcmd % mergedg2filepath
79+
subprocess.call(catcmd_out, shell=True)
80+
time.sleep(30)
81+
# Lets compress single BIG grib2 file by using gz compress cmd.
82+
os.chdir(tardir)
83+
gzip_cmd = '%s -9 -p 32 %s' % (pigz, mergedg2file)
84+
print "gzip_cmd = ", gzip_cmd
85+
subprocess.call(gzip_cmd, shell=True)
86+
time.sleep(5)
87+
7788
if member == '000':
7889
tarpath = os.path.abspath(tardir)
79-
80-
90+
8191
# do scp the tar files to ftp_server
8292
cmd = 'ssh ncmlogin3 "rsync --update --ignore-existing -razt %s %s:/data/ftp/pub/outgoing/arulalan/NCUM_TIGGE/"' % (tarpath, ftp_server)
8393
print cmd

‎bsubScripts/ncumeps_global_tigge/tigge_create_tarball_g2files_put_into_ftp.py

+33-22
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,23 @@
1414
## Arulalan.T
1515
## 04-Mar-2016.
1616

17-
import os, subprocess, datetime, getopt, sys, glob
17+
import os, subprocess, datetime, getopt, sys, glob, time
1818

1919
pbzip2 = '/gpfs1/home/Libs/GNU/ZIPUTIL/pbzip2'
2020
pigz = '/gpfs1/home/Libs/GNU/ZIPUTIL/pigz'
2121
tigge_check = '/gpfs2/home/prasanna/SOFTWARE/GNU/grib_api-1.21.0-path/bin/tigge_check'
2222

2323
filesCount = {'ttr': 41, 'lsm': 41, 'orog': 41, '10v': 41, 'tcc': 41, 'gh': 369, 'skt': 41, 'tp': 41, 'msl': 41, 'mx2t6': 40, '2d': 41, '10u': 41, 'mn2t6': 40, 'sshf': 41, 'slhf': 41, 'ssr': 41, '2t': 41, 'sp': 41, 'st': 41, 'q': 328, 'u': 328, 't': 328, 'str': 41, 'v': 328, 'sd': 41}
2424

25+
dirsOrder = [
26+
'gh', 'u', 'v', 'q', 't', '10u', '10v', '2t', 'mx2t6', 'mn2t6', 'skt', 'st',
27+
'2d', 'sp', 'msl', 'tp', 'ttr', 'lsm', 'tcc', 'slhf', 'ssr', 'sshf', 'str', 'sd', 'orog'
28+
]
29+
# merge cmd into single grib2 of each members
30+
catcmd = ['%s/z_tigge_c_dems*%s' % (d,d) for d in dirsOrder]
31+
catcmd = ' '.join(catcmd)
32+
catcmd = 'cat %s ' % catcmd
33+
catcmd += ' > %s'
2534

2635
def createTarBalls(path, today, member):
2736

@@ -47,8 +56,8 @@ def createTarBalls(path, today, member):
4756
cmd = tigge_check + ' -v -w %s/*' % tgf
4857
tigge_check_val = os.system(cmd) # it should return 0 on pass
4958
if tigge_check_val != 0 :
50-
print "Error : While checking via tigge_check cmd got error!"
51-
sys.exit(0)
59+
print "WARNING : While checking via tigge_check cmd got error!"
60+
#sys.exit(0)
5261
# end of for tgf in os.listdir('.'):
5362

5463
tDay = datetime.datetime.strptime(today, "%Y%m%d")
@@ -61,23 +70,26 @@ def createTarBalls(path, today, member):
6170

6271
tardir = '../../TarFiles/%s' % today
6372
if not os.path.exists(tardir): os.makedirs(tardir)
64-
tarfile = 'ncmrwf_tigge_%s_%s.tar.gz' % (today, member)
73+
mergedg2file = 'ncmrwf_dems_tigge_%s_%s.grib2' % (today, member)
74+
mergedg2filepath = os.path.join(tardir, mergedg2file)
6575
print "currnet path : ", os.getcwd()
66-
# normal "$ tar cvjf fcst_20160223.tar.bz2 *fcst*grb2" cmd takes 6 minutes 43 seconds.
67-
#
68-
# where as in parallel bz2, "$ tar -c *fcst*grb2 | pbzip2 -v -c -f -p32 -m500 > fcst_20160223_parallel.tar.bz2" cmd takes only just 23 seconds alone, with 32 processors and 500MB RAM memory.
69-
#
70-
# create analysis files tar file in parallel # -m500 need to be include for pbzip2
71-
tg_files = ' '.join([' -C %s . ' % os.path.join(inpath, tgf) for tgf in os.listdir('.')])
72-
cmd = "tar -c %s | %s -v -c -f -p32 > %s/%s" % (tg_files, pigz, tardir, tarfile)
7376

74-
print cmd
75-
subprocess.call(cmd, shell=True)
77+
# merge all the params, all the levels, all the time steps, but individual members
78+
# into single grib2 (BIG) file.
79+
catcmd_out = catcmd % mergedg2filepath
80+
subprocess.call(catcmd_out, shell=True)
81+
time.sleep(30)
82+
83+
# Lets compress single BIG grib2 file by using gz compress cmd.
84+
os.chdir(tardir)
85+
gzip_cmd = '%s -9 -p 32 %s' % (pigz, mergedg2file)
86+
print "gzip_cmd = ", gzip_cmd
87+
subprocess.call(gzip_cmd, shell=True)
88+
time.sleep(5)
7689

7790
if member == '000':
7891
tarpath = os.path.abspath(tardir)
79-
80-
92+
8193
# do scp the tar files to ftp_server
8294
cmd = 'ssh ncmlogin3 "rsync --update --ignore-existing -razt %s %s:/data/ftp/pub/outgoing/NCUM_TIGGE/"' % (tarpath, ftp_server)
8395
print cmd
@@ -89,8 +101,7 @@ def createTarBalls(path, today, member):
89101
try:
90102
subprocess.call(cmd, shell=True)
91103
except Exception as e:
92-
print "past 11th day tar ball has been removed from ftp_server, already", e
93-
104+
print "past 11th day tar ball has been removed from ftp_server, already", e
94105

95106
# remove yesterday directory!!!
96107
y4DayPath = os.path.join(path, '../../%s' % y4Day)
@@ -101,11 +112,11 @@ def createTarBalls(path, today, member):
101112
# end of if os.path.exists(y4DayPath):
102113

103114
# remove past 11th day tar file!!!
104-
y11DayPath = os.path.join(path, '../../TarFiles/%s' % y11Day)
105-
if os.path.exists(y11DayPath):
106-
cmd = "rm -rf %s" % y11DayPath
107-
print cmd
108-
subprocess.call(cmd, shell=True)
115+
# y11DayPath = os.path.join(path, '../../TarFiles/%s' % y11Day)
116+
# if os.path.exists(y11DayPath):
117+
# cmd = "rm -rf %s" % y11DayPath
118+
# print cmd
119+
# subprocess.call(cmd, shell=True)
109120
# end of if os.path.exists(y11DayPath):
110121
# end of if member == '000':
111122
os.chdir(cdir)

‎g2utils/um2grb2.py

+1-4
Original file line numberDiff line numberDiff line change
@@ -870,10 +870,7 @@ def getVarInOutFilesDetails(inDataPath, fname, hr):
870870
('convective_rainfall_amount', 'm01s05i201'),]
871871
# all vars
872872
varNamesSTASH = varNamesSTASH1 + varNamesSTASH2
873-
if _requiredPressureLevels_ and not set(_requiredPressureLevels_).issubset([925., 960., 975., 980., 985., 990., 995., 1000.]):
874-
# same stash available in pd file also. so remove only incase of chosen pressure level
875-
# not applicable to this pe file.
876-
varNamesSTASH.remove(('geopotential_height', 'm01s16i202')) # 8 pressure levels
873+
877874
# the cube contains Instantaneous data at every 1-hours.
878875
if __anl_step_hour__ == 1:
879876
fcstHours = numpy.arange(0,6,1)

‎g2utils/um2grb2tigge.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ def getTiggeFileName(cube):
406406
prefix = 'z_tigge_c'
407407
cccc = 'dems' # DEMS Delhi Meteorological Station centre code (WMO Standard)
408408
mmmm = 'glob'
409-
vvvv = 'test' # TIGGE TEST
409+
vvvv = 'prod' # TIGGE production mode
410410
cstash = None
411411

412412
# get the cube time and make it as yyyymmddhhmmss
@@ -1006,7 +1006,7 @@ def save_tigge_tweaked_messages(cubeList):
10061006
# https://software.ecmwf.int/wiki/display/TIGGE/Rules+for+data+encoding+and+exchange
10071007
# 4 for TIGGE-NCMRWF Operational data
10081008
# 5 for TIGGE-NCMRWF Test data
1009-
gribapi.grib_set_long(grib_message, "productionStatusOfProcessedData", 5)
1009+
gribapi.grib_set_long(grib_message, "productionStatusOfProcessedData", 4) # Operational mode
10101010

10111011
if cube.coords("realization"):
10121012
# ensembles tweak
@@ -1205,7 +1205,7 @@ def makeTotalCummulativeVars(arg):
12051205
lname = 'time_integrated_' + sname
12061206
rstash = False
12071207

1208-
fname = 'z_tigge_c_dems_' +_current_date_+ '000000_glob_test_' # TIGGE TEST
1208+
fname = 'z_tigge_c_dems_' +_current_date_+ '000000_glob_prod_' # TIGGE prod
12091209
fname += umfcstype+ '_sl_%s_' + ens + '_0000_' + svar + '.nc'
12101210
infiles = [os.path.join(*[_opPath_, ens, svar, fname % str(t).zfill(4)])
12111211
for t in range(6, __end_long_fcst_hour__+1, 6)]

0 commit comments

Comments
 (0)