Ticket #322: romsaf_profile.sh

File romsaf_profile.sh, 16.3 KB (added by Stig Syndergaard, 5 years ago)

ROM SAF operational processing chain as of March 2019 - used for reprocessing, offline and ICDR generation

Line 
1#!/bin/bash
2#
3# Note: (system) returns
4# 222, reserved by timeout.
5# 223, if we failed in retreiving NWP data.
6# 224, qc fails critical - which should be debugged.
7# 225, if bgf file is missing, but dis is there - this should be debugged.
8# 238, missing/bad input arguments.
9#
10[ -f ./gpac_env_vars.sh ] || { echo "Environment file './gpac_env_vars.sh' does not exist"; exit 238; }
11
12# set -o verbose
13. ./gpac_env_vars.sh
14
15
16occ_abs_path=${GPAC_WRKDIR}/occ_${GPAC_PID}.nc
17ben_abs_path=${GPAC_WRKDIR}/ben_${GPAC_PID}.nc
18bet_abs_path=${GPAC_WRKDIR}/bet_${GPAC_PID}.nc
19atm_abs_path=${GPAC_WRKDIR}/atm_${GPAC_PID}.nc
20wet_abs_path=${GPAC_WRKDIR}/wet_${GPAC_PID}.nc
21wfm_abs_path=${GPAC_WRKDIR}/wfm_${GPAC_PID}.nc
22bgo_abs_path=${GPAC_WRKDIR}/bgo_${GPAC_PID}.nc
23bgn_abs_path=${GPAC_WRKDIR}/bgn_${GPAC_PID}.nc
24bgf_abs_path=${GPAC_WRKDIR}/bgf_${GPAC_PID}.nc
25bga_abs_path=${GPAC_WRKDIR}/bga_${GPAC_PID}.nc
26bgr_abs_path=${GPAC_WRKDIR}/bgr_${GPAC_PID}.nc # temporary file
27bge_abs_path=${GPAC_WRKDIR}/bge_${GPAC_PID}.nc
28dfm_abs_path=${GPAC_WRKDIR}/dfm_${GPAC_PID}.nc # temporary file
29cor_abs_path=${GPAC_WRKDIR}/cor_${GPAC_PID}.nc
30att_abs_path=${GPAC_WRKDIR}/att_${GPAC_PID}.nc # temporary file
31ate_abs_path=${GPAC_WRKDIR}/ate_${GPAC_PID}.nc
32cvs_abs_path=${GPAC_WRKDIR}/cvs_${GPAC_PID}.nc
33grh_abs_path=${GPAC_WRKDIR}/grh_${GPAC_PID}.nc
34tph_abs_path=${GPAC_WRKDIR}/tph_${GPAC_PID}.nc # temporary file
35dis_abs_path=${GPAC_WRKDIR}/dis_${GPAC_PID}.nc
36bfr_abs_path=${GPAC_WRKDIR}/bfr_${GPAC_PID}.bin
37pnc_abs_path=${GPAC_WRKDIR}/pnc_${GPAC_PID}.nc
38ppc_abs_path=${GPAC_WRKDIR}/ppc_${GPAC_PID}.nc
39qcr_abs_path=${GPAC_WRKDIR}/qcr_${GPAC_PID}.json # perhaps specify this in gpac_env_vars.sh
40nwp_retriever_settings_ei=${NWP_RETRIEVER_SETTINGS_EI}
41nwp_retriever_settings_od=${NWP_RETRIEVER_SETTINGS_OD}
42qcr_settings=${QCR_SETTINGS}
43# set +o verbose
44
45#
46# Wrapper (to print version number and handle error).
47#
48cmd_wrapper() {
49 echo ""
50 echo "+++ Running: $*"
51 # Get version
52 echo "$(eval $1 -v)"
53 # Run command
54 eval $*
55 status=$?
56 if [ $status -ne 0 ]; then
57 echo "$1 failed, status: $status"
58 fi
59 return $status
60}
61
62
63#--- 1. Convert input file formats
64if [ -f "${GPAC_ATMPHS_ABS_PATH}" ]; then
65 # ATMPHS file from ucar.
66 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ucar2ropp \
67 ${GPAC_ATMPHS_ABS_PATH} \
68 -o ${occ_abs_path} \
69 -u \
70 -d \
71 --no-ranchk
72elif [ -f "${GPAC_NC4_ABS_PATH}" ]; then
73 # NC4 file from eumetsat.
74 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/eum2ropp \
75 ${GPAC_NC4_ABS_PATH} \
76 -o ${bet_abs_path} \
77 -r thinned \
78 -e \
79 -u \
80 -d \
81 --no-ranchk
82 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/eum2ropp \
83 ${GPAC_NC4_ABS_PATH} \
84 -o ${ben_abs_path} \
85 -r high_resolution \
86 -e \
87 -u \
88 -d \
89 --no-ranchk
90 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/eum2ropp \
91 ${GPAC_NC4_ABS_PATH} \
92 -o ${occ_abs_path} \
93 -x \
94 -l cl+ol \
95 -c \
96 -u \
97 -d \
98 --no-ranchk
99else
100 echo "No valid input file are defined"
101 exit 238
102fi
103
104# If occ file is not produced - no need to continue, just exit!
105[ -f ${occ_abs_path} ] || { echo "***ERROR: occ file not produced!"; exit 1; }
106
107#--- 2. Process to bangle, refrac, and dry temp
108
109#----2.1 Generate observation profiles
110if [ -f "${GPAC_GPSBIT_ABS_PATH}" ]; then
111 navopt="-navfile ${GPAC_GPSBIT_ABS_PATH}"
112else
113 navopt=""
114fi
115if [ ! -f "${GPAC_ATTRIBUTE_CFG}" ]; then
116 GPAC_ATTRIBUTE_CFG=$(trn_attributes_path ${occ_abs_path})
117fi
118cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_pp_occ_tool ${occ_abs_path} \
119 -o ${atm_abs_path} \
120 -c ${GPAC_MISSION_CFG} \
121 -a ${GPAC_ATTRIBUTE_CFG} \
122 ${navopt} \
123 -egm96 ${GEOPOT_COEF} \
124 -cegm96 ${GEOPOT_CORR} \
125 -mfile ${GPAC_ROPP_ROOT}/data/BAROCLIM_coeff.nc \
126 -msisfile ${GPAC_ROPP_ROOT}/data/MSIS_coeff.nc \
127 -d \
128 ${OPTATM} \
129 --no-ranchk
130
131status=$?
132
133# If atm file is not produced or processing error - no need to continue, just exit!
134# Continue on processing warning.
135if [ ! -f ${atm_abs_path} ] || [ $status -ne 0 ]; then
136 if [ ! -f ${atm_abs_path} ] || [ $status -ne 1 ]; then
137 echo "***ERROR: atm file not produced / bad return code! Setting file(s) to non nominal"
138 gpac_qc set ${occ_abs_path} ${atm_abs_path} --pcd 113 --or
139 exit 2
140 fi
141 echo "***WARNING: return code = 1! Continuing."
142fi
143
144# set status to OK, for now
145declare -i overall_status
146overall_status=0
147
148#--- 2.2 Thinning of observed refractivity: atm -> att
149# Also removes Vlist
150cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp2ropp ${atm_abs_path} \
151 -p ${GPAC_ROPP_ROOT}/data/ropp_thin_eum-247.dat \
152 -o ${att_abs_path} \
153 -b ${GPAC_ATTRIBUTE_CFG} \
154 -s \
155 --no-ranchk
156
157#--- 2.3 Tropopause height and tropopause parameters
158
159
160cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_apps_tph_tool ${atm_abs_path} \
161 -o ${tph_abs_path} \
162 -a ${GPAC_ATTRIBUTE_CFG} \
163 -b \
164 -n \
165 -y \
166 --no-ranchk
167
168# Move tph file to atm file, if it has been produced.
169if [ -f ${tph_abs_path} ]; then
170 mv ${tph_abs_path} ${atm_abs_path}
171else
172 echo "***ERROR: ${tph_abs_path} not produced."
173 overall_status=3
174fi
175
176
177#--- 3. Generate background profiles
178
179#--- 3.1 Find the GRIB files by reading info from atm files.
180# Returns a string with the grib file path(s), lat, and lon.
181echo "Get EI GRIB files, based on atm file ..."
182GRIB_FILE_STR=$(time get_nwp ${nwp_retriever_settings_ei} ${atm_abs_path})
183status=$?
184[ $status -eq 0 ] || { echo "***ERROR, get_nwp failed, status $status"; exit 223; }
185
186#--- 3.2 Dissect the string.
187FC_FILE_LEFT=$(echo $GRIB_FILE_STR | cut -d' ' -f 1)
188FC_FILE_RIGHT=$(echo $GRIB_FILE_STR | cut -d' ' -f 2)
189AN_FC_FILE_LEFT=$(echo $GRIB_FILE_STR | cut -d' ' -f 3)
190AN_FC_FILE_RIGHT=$(echo $GRIB_FILE_STR | cut -d' ' -f 4)
191AN_FILE_LEFT=$(echo $GRIB_FILE_STR | cut -d' ' -f 5)
192AN_FILE_RIGHT=$(echo $GRIB_FILE_STR | cut -d' ' -f 6)
193LAT=$(echo $GRIB_FILE_STR | cut -d' ' -f 7)
194LON=$(echo $GRIB_FILE_STR | cut -d' ' -f 8)
195
196#--- 3.3 Diagnostics.
197echo "USING EI FORECAST GRIB FILES : $FC_FILE_LEFT $FC_FILE_RIGHT"
198echo "USING EI OROGRAPHY FROM : $AN_FC_FILE_LEFT"
199echo "USING EI ANALYSIS GRIB FILES : $AN_FILE_LEFT $AN_FILE_RIGHT"
200echo "POSITION: LAT: $LAT LON $LON"
201
202#--- 3.4 Extract two background profiles from GRIB files: bgf and bga
203cmd_wrapper ${GPAC_ROPP_ROOT}/bin/grib2bgrasc $FC_FILE_LEFT \
204 -g $FC_FILE_RIGHT \
205 -z $AN_FC_FILE_LEFT \
206 -y ${atm_abs_path} \
207 -x ${bgf_abs_path} \
208 --no-ranchk
209
210cmd_wrapper ${GPAC_ROPP_ROOT}/bin/grib2bgrasc $AN_FILE_LEFT \
211 -g $AN_FILE_RIGHT \
212 -y ${atm_abs_path} \
213 -x ${bga_abs_path} \
214 --no-ranchk
215
216if [ ! -f ${bga_abs_path} ]; then
217 echo "***ERROR: bga file not produced!"
218 overall_status=4
219fi
220
221#--- 3.5 Get OD nwp file
222
223echo "Get OD GRIB files, based on atm file ..."
224GRIB_FILE_STR=$(time get_nwp ${nwp_retriever_settings_od} ${atm_abs_path})
225status=$?
226[ $status -eq 0 ] || { echo "***ERROR, get_nwp failed, status $status"; exit 223; }
227#--- 3.xx Dissect the string.
228FC_FILE_LEFT=$(echo $GRIB_FILE_STR | cut -d' ' -f 1)
229AN_FC_FILE_LEFT=$(echo $GRIB_FILE_STR | cut -d' ' -f 2)
230AN_FILE_LEFT=$(echo $GRIB_FILE_STR | cut -d' ' -f 3)
231LAT=$(echo $GRIB_FILE_STR | cut -d' ' -f 4)
232LON=$(echo $GRIB_FILE_STR | cut -d' ' -f 5)
233
234#--- 3.6 Diagnostics.
235echo "USING OD FORECAST GRIB FILE : $FC_FILE_LEFT"
236echo "USING OD OROGRAPHY FROM : $AN_FC_FILE_LEFT"
237echo "USING OD ANALYSIS GRIB FILE : $AN_FILE_LEFT"
238echo "POSITION: LAT: $LAT LON $LON"
239
240#--- 3.7 Extract background profiles from GRIB files: bgo and bgn
241
242cmd_wrapper ${GPAC_ROPP_ROOT}/bin/grib2bgrasc $FC_FILE_LEFT \
243 -z $AN_FC_FILE_LEFT \
244 -y ${atm_abs_path} \
245 -x ${bgo_abs_path} \
246 --no-ranchk
247
248cmd_wrapper ${GPAC_ROPP_ROOT}/bin/grib2bgrasc $AN_FILE_LEFT \
249 -y ${atm_abs_path} \
250 -x ${bgn_abs_path} \
251 --no-ranchk
252
253if [ ! -f ${bgn_abs_path} ]; then
254 echo "***ERROR: bgn file not produced!"
255 overall_status=5
256fi
257
258#--- 3.8 Forward model to bangle, refrac, and dry temp
259cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_fm_bg2ro_1d ${bgf_abs_path} \
260 -o ${dfm_abs_path} \
261 -a ${GPAC_ROPP_ROOT}/etc/nwp_ei_fc_attrb.cf \
262 -l ${atm_abs_path} \
263 -f \
264 -d \
265 -new_op \
266 -comp \
267 --no-ranchk
268
269if [ -f ${dfm_abs_path} ]; then
270 mv ${dfm_abs_path} ${bgf_abs_path}
271else
272 echo "***ERROR: ${dfm_abs_path} does not exist!"
273 overall_status=7
274fi
275
276cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_fm_bg2ro_1d ${bga_abs_path} \
277 -o ${dfm_abs_path} \
278 -a ${GPAC_ROPP_ROOT}/etc/nwp_ei_an_attrb.cf \
279 -l ${atm_abs_path} \
280 -f \
281 -d \
282 -new_op \
283 -comp \
284 --no-ranchk
285
286if [ -f ${dfm_abs_path} ]; then
287 mv ${dfm_abs_path} ${bga_abs_path}
288else
289 echo "***ERROR: ${dfm_abs_path} does not exist!"
290 overall_status=8
291fi
292
293cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_fm_bg2ro_1d ${bgo_abs_path} \
294 -o ${dfm_abs_path} \
295 -a ${GPAC_ROPP_ROOT}/etc/nwp_od_fc_attrb.cf \
296 -l ${atm_abs_path} \
297 -f \
298 -d \
299 -new_op \
300 -comp \
301 --no-ranchk
302
303if [ -f ${dfm_abs_path} ]; then
304 mv ${dfm_abs_path} ${bgo_abs_path}
305else
306 echo "***ERROR: ${dfm_abs_path} does not exist!"
307 overall_status=9
308fi
309
310cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_fm_bg2ro_1d ${bgn_abs_path} \
311 -o ${dfm_abs_path} \
312 -a ${GPAC_ROPP_ROOT}/etc/nwp_od_an_attrb.cf \
313 -l ${atm_abs_path} \
314 -f \
315 -d \
316 -new_op \
317 -comp \
318 --no-ranchk
319
320if [ -f ${dfm_abs_path} ]; then
321 mv ${dfm_abs_path} ${bgn_abs_path}
322else
323 echo "***ERROR: ${dfm_abs_path} does not exist!"
324 overall_status=9
325fi
326
327#--- 3.9 Tropopause height and tropopause parameters
328if [ -f ${bgf_abs_path} ]; then
329 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_apps_tph_tool ${bgf_abs_path} \
330 -o ${tph_abs_path} \
331 -b \
332 -n \
333 -y \
334 -t \
335 --no-ranchk
336 if [ -f ${tph_abs_path} ]; then
337 mv ${tph_abs_path} ${bgf_abs_path}
338 else
339 echo "***ERROR: tph file not produced!"
340 overall_status=10
341 fi
342fi
343
344if [ -f ${bga_abs_path} ]; then
345 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_apps_tph_tool ${bga_abs_path} \
346 -o ${tph_abs_path} \
347 -b \
348 -n \
349 -y \
350 -t \
351 --no-ranchk
352 if [ -f ${tph_abs_path} ]; then
353 mv ${tph_abs_path} ${bga_abs_path}
354 else
355 echo "***ERROR: tph file not produced!"
356 overall_status=11
357 fi
358fi
359
360if [ -f ${bgo_abs_path} ]; then
361 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_apps_tph_tool ${bgo_abs_path} \
362 -o ${tph_abs_path} \
363 -b \
364 -n \
365 -y \
366 -t \
367 --no-ranchk
368 if [ -f ${tph_abs_path} ]; then
369 mv ${tph_abs_path} ${bgo_abs_path}
370 else
371 echo "***ERROR: tph file not produced!"
372 overall_status=12
373 fi
374fi
375
376if [ -f ${bgn_abs_path} ]; then
377 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_apps_tph_tool ${bgn_abs_path} \
378 -o ${tph_abs_path} \
379 -b \
380 -n \
381 -y \
382 -t \
383 --no-ranchk
384 if [ -f ${tph_abs_path} ]; then
385 mv ${tph_abs_path} ${bgn_abs_path}
386 else
387 echo "***ERROR: tph file not produced!"
388 overall_status=12
389 fi
390fi
391
392#--- 4. 1D-Var processing
393
394#--- 4.1 Copy bgf -> bgr (for the 1D-Var processing)
395if [ -f ${bgf_abs_path} ]; then
396 cp ${bgf_abs_path} ${bgr_abs_path}
397else
398 echo "***ERROR: bgf file not produced!"
399 overall_status=6
400fi
401
402if [ -f ${att_abs_path} ]; then
403 #--- 4.2 Add observation errors: att -> ate
404 cmd_wrapper ${GPAC_1DVAR_ROOT}/bin/1DV_add_obs_error.x _${GPAC_PID} \
405 -a ${GPAC_ATTRIBUTE_CFG} \
406 -attdir ${GPAC_WRKDIR} \
407 -bgrdir ${GPAC_WRKDIR} \
408 -outdir ${GPAC_WRKDIR} \
409 -Omod TP \
410 -datadir ${GPAC_1DVAR_ROOT}/etc/gras_1dvar \
411 -o _${GPAC_PID}
412 if [ ! -f ${ate_abs_path} ]; then
413 echo "***ERROR: ${ate_abs_path} not produced!"
414 overall_status=13
415 fi
416
417 #--- 4.3 Get background error correlation filename, based on number of nwp levels
418 GPAC_BMATRIX=$(trn_bgr_error_cor_path ${bgf_abs_path})
419 status=$?
420 echo "***INFO: USING GPAC_BMATRIX: ${GPAC_BMATRIX}"
421 [ $status -eq 0 ] || { echo "***ERROR: Could not find background error correlation filename";}
422
423 if [ $status -eq 0 ] && [ -f ${bgr_abs_path} ]; then
424 #--- 4.4 1D-Var processing
425 # TODO: config should be defined somewhere else like the config file for ropp_pp_occ_tool
426
427 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_1dvar_add_bgr_error ${bgr_abs_path} \
428 -c ${GPAC_BMATRIX}
429
430 cmd_wrapper ${GPAC_1DVAR_ROOT}/bin/1DV_refrac.x \
431 -y ${ate_abs_path} \
432 -b ${bgr_abs_path} \
433 -o ${wet_abs_path} \
434 -wfm ${wfm_abs_path} \
435 --obs-corr ${cor_abs_path} \
436 -c ${GPAC_1DVAR_ROOT}/etc/gras_1dvar/1DV_config.cf \
437 -a ${GPAC_ATTRIBUTE_CFG} \
438 --bg-corr ${GPAC_BMATRIX} \
439 -cvS ${cvs_abs_path} \
440 -grH ${grh_abs_path} \
441 -J_qc 5.0 \
442 -comp
443
444 if [ -f ${wfm_abs_path} ]; then
445 #--- 4.5 Change name of background file: bgr -> bge
446 mv ${bgr_abs_path} ${bge_abs_path}
447 #--- 4.6 Tropopause height and tropopause parameters
448 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp_apps_tph_tool ${wfm_abs_path} \
449 -o ${tph_abs_path} \
450 -a ${GPAC_ATTRIBUTE_CFG} \
451 -t \
452 --no-ranchk
453 mv ${tph_abs_path} ${wfm_abs_path}
454 else
455 echo "***ERROR: wfm file not produced!"
456 overall_status=14
457 fi
458 else
459 echo "***INFO: Skipping 1dvar processing."
460 fi
461else
462 echo "***ERROR: ${att_abs_path} not produced! Skipping 1dvar"
463 overall_status=15
464fi
465
466#--- 5. QC handling
467# TODO: handle errors from qc. Handle archiving of qc-file.
468
469#--- 5.1 First perform QC methods defined in config
470gpac_qc qc \
471 --bgr ${bgf_abs_path} \
472 --levels 1b 2a \
473 -o ${qcr_abs_path} \
474 ${qcr_settings} \
475 ${atm_abs_path}
476
477status=$?
478[ $status -eq 0 ] || { echo "***ERROR: gpac_qc level 1b, 2a failed!"; exit 224; }
479
480gpac_qc qc \
481 --levels 2b \
482 -o ${qcr_abs_path} \
483 ${qcr_settings} \
484 ${wet_abs_path}
485
486status=$?
487[ $status -eq 0 ] || { echo "***ERROR: gpac_qc level 2b failed!"; exit 224; }
488
489#--- 5.2 Now determine the total pcd value from QC and obs files
490#--- The additional files we want to set pcd value in are added in --add-files
491gpac_qc flag \
492 --qcr ${qcr_abs_path} \
493 --obs-files ${occ_abs_path} ${atm_abs_path} ${wet_abs_path} \
494 --add-files ${att_abs_path} ${ate_abs_path} ${wfm_abs_path} \
495 --qc-rules
496
497status=$?
498[ $status -eq 0 ] || { echo "***ERROR: gpac_qc set pcd failed!"; exit 224; }
499
500#--- 6. Generate dis and BUFR files
501
502#--- 6.1 The dis file
503if [ -f ${att_abs_path} ]; then
504 if [ -f ${wet_abs_path} ]; then
505 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp2ropp ${wet_abs_path} \
506 -o ${dis_abs_path} \
507 -c ${att_abs_path} \
508 -b ${GPAC_ATTRIBUTE_CFG} \
509 -s \
510 --no-ranchk
511 else
512 cp ${att_abs_path} ${dis_abs_path}
513 fi
514
515 #--- 6.2 Store dis as BUFR
516 cmd_wrapper ${GPAC_ROPP_ROOT}/bin/ropp2bufr ${dis_abs_path} \
517 -o ${bfr_abs_path}
518
519 if [ ! -f ${bfr_abs_path} ]; then
520 echo "***ERROR: bfr file not produced"
521 overall_status=16
522 fi
523else
524 echo "***INFO: Skipping dis and bfr - att file not produced"
525fi
526
527# Can we skip pnc generation if some previous files not produced?
528#--- 7. Generate pnc files
529cmd_wrapper ${GPAC_CLIM_ROOT}/bin/rom_profs_ropp2pnc.x \
530 -occfile ${occ_abs_path} \
531 -atmfile ${atm_abs_path} \
532 -wetfile ${wfm_abs_path} \
533 -bgrfile ${bgf_abs_path} \
534 -bgefile ${bge_abs_path} \
535 -pncfile ${pnc_abs_path} \
536 -ppcfile ${ppc_abs_path}
537
538if [ ! -f ${pnc_abs_path} ]; then
539 echo "***ERROR: pnc file not produced"
540 overall_status=17
541fi
542if [ ! -f ${ppc_abs_path} ]; then
543 echo "***ERROR: ppc file not produced"
544 overall_status=18
545fi
546
547echo "***INFO: Final status: $overall_status"
548
549# return final status
550exit $overall_status