point_forecast

#!/bin/bash
set +H
declare -A slf=([PATH]="$(dirname $(readlink -e $0))" [NAME]="${0##*/}")

source /opt/scripts/functions/strings.inc

doShowUsage () {
cat <<EOF
Usage:
 ${slf:-${0##*/}} [-f ALL_POINTS_FILE] [-s CSV_SOURCE_PATH] [-d XLS_DEST_PATH ] [-H PREDICT_HOURS ] [-D] [-x] [-n] 'longitude1,latitude1 longitude2,latitude2 ... longitudeN,latitudeN' date hour
 Where:
  -s CSV_SOURCE_PATH: specify the path to the day-folders containing CSV generated from Grib's
  -d XLS_DEST_PATH: specify a base path to the folder, where resulting/output XLS-files will be placed   
  -H PREDICT_HOURS: How many hours of prediction to include in XLS
  -x BASH debug mode. Script will run very slow and with extra-verbose output.
  -R For developers only! Recompile hashes.
  -Z For developers only! Output hashes and exit immediately.
  -n Delete source CSV after conversion to XLS 
  -f FILE: If you want to save report for all points in one file
EOF
 return 0
}

(( $# )) || { doShowUsage; exit 1; }

ncep_pth='/store/GRIB/cooked/GFS4/NCEP'
order_pred_hours=120
flReCreateMP2LBL=0
flDontSaveCSV=0
flDontUnpackCSV=0
mode='standart'
while getopts 's: d: H: m: f: Z: Rhxn' k; do
 case $k in
  h) doShowUsage; exit 0 ;;
  s) ncep_pth="$OPTARG" ;;
  d) dst_pth="$OPTARG" ;;
  H) order_pred_hours="$OPTARG" ;;
  x) set -x ;;
  R) flReCompileHashes=1 ;;
  m) mode="$OPTARG" ;;
  Z) 
   flPrintHashesAndExit=1
   flReCompileHashes=1
   pthStoreHashes="$OPTARG"
  ;;
  n) flDontSaveCSV=1 ;;
  f) fileWhere2Save="$OPTARG" ;;
  \?) echo "Unknown key: $k" >&2; doShowUsage; exit 1 ;;
 esac
done
shift $((OPTIND-1))

source $(dirname $(readlink -e "$0"))/${0##*/}.inc || {
 echo "Cant source my include file: \"$(dirname $(readlink -e "$0"))/${0##*/}.inc\", check pathes!" >&2
 exit 1
}

if [[ $flPrintHashesAndExit ]]; then
 [[ $pthStoreHashes == '-' ]] || exec 3<&1 1>"$pthStoreHashes"
 for a in MP2LBL LBL2MP LBLOrder; do declare -p $a; done
 [[ $pthStoreHashes == '-' ]] || exec 1<&3
 exit 0
fi

for k in points bdate bhours; do
 eval "$k='$1'"; shift
done

[[ $bhours =~ ^[0-9]+$ ]] || { doShowUsage; exit 1; }

bdate=${bdate//-/}
if ! [[ $bdate =~ ^(20[0-9]{2})(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[01])$ ]]; then 
 if [[ $bdate =~ ^(0[1-9]|[1-2][0-9]|3[01])(0[1-9]|1[0-2])(20[0-9]{2})$ ]]; then
  bdate="${BASH_REMATCH[3]}${BASH_REMATCH[2]}${BASH_REMATCH[1]}"
 else
  echo 'Wrong date format!'; exit 1
 fi
fi

bhours=$(( (bhours/6)*6 ))
zz_bhours=$(printf '%02g' $bhours)

src_pth="${ncep_pth}/${bdate}"
if [[ ! -d $src_pth ]]; then
 src_pth="${ncep_pth}/${bdate:0:4}-${bdate:4:2}-${bdate:6:2}"
 [[ -d $src_pth ]] || exit 112
fi

[[ $dst_pth ]] && { 
 mkdir -p "$dst_pth"
 cd "$dst_pth"
}

outCSVs=''
csvFC="$fileWhere2Save"
for lola in $points; do
 [[ $fileWhere2Save ]] || {
  csvFC=${lola//./}
  csvFC="${bdate}_${zz_bhours}00_${csvFC/,/-}.csv" 
 }
 lon=${lola%,*}; lat=${lola#*,}
 lon="${lon%.*}.$(rpad ${lon#*.} '000000')"
 lat="${lat%.*}.$(rpad ${lat#*.} '000000')"
 
 exec 3<&1 1>>"$csvFC"

 header="time,$(eval echo {0..${order_pred_hours}..3})"
 echo "${header// /,}" 
    
 for lbl in ${LBLOrder[@]}; do
  echo -n "$lbl"
  
  mp=${LBL2MP["$lbl"]} 
  max_pred_hours=$(cd ${src_pth}; ls gfs_4_${bdate}_${zz_bhours}00_*_${mp}.csv*  | cut -d_ -f5 | sort -rn | head -1)
  if [[ $max_pred_hours =~ ^0*([0-9]+)$ ]]; then
   max_pred_hours=${BASH_REMATCH[1]}
  else
   eval "printf ',$UNDEF%.0s' {0..${order_pred_hours}..3}"; echo
   continue
  fi
  pred_hours=$order_pred_hours
  (( max_pred_hours<order_pred_hours )) && pred_hours=$max_pred_hours
  start_pred_hour=0
  if [[ ${NotInRA["$mp"]} ]]; then
   start_pred_hour=3
   echo -n ",$UNDEF"
  fi
  csv_row=''
  for si in $(seq -f '%03g' ${start_pred_hour} 3 ${pred_hours}); do
   csvSource="${src_pth}/gfs_4_${bdate}_${zz_bhours}00_${si}_${mp}.csv"
   [[ ! -f "$csvSource" && -f "${csvSource}.gz" ]] && \
    gzip -d "$csvSource.gz" 2>/dev/null
    
   if [[ -f "$csvSource" ]]; then    
    csv_row+=$(sed -nr "s%^${lon},\s*${lat},\s*([^,]+),?\s*$%,\1%"'; T lbl; p; q; :lbl; $s%.*%,'$UNDEF'%p' "$csvSource")
   else
    csv_row+=",$UNDEF"
   fi
  done
  echo -n "$csv_row"
  (( max_pred_hours<order_pred_hours )) && \
   eval "printf ',$UNDEF%.0s' {$((pred_hours+3))..${order_pred_hours}..3}"
  echo
 done
 exec 1<&3
 outCSVs+="$csvFC "
done

if [[ $fileWhere2Save ]]; then
 csv2xls ${fileWhere2Save%.*}.xls $fileWhere2Save
else
 { for csvFC in ${outCSVs% }; do  
    echo "${csvFC%.csv}.xls $csvFC"
   done; } | parallel eval 'csv2xls'
fi