-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathfunctions_download_dwd.sh
87 lines (87 loc) · 3.48 KB
/
functions_download_dwd.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
#Given a variable name and year-month-day-run as environmental variables download and merges the variable
################################################
# listurls() {
# filename="$1"
# url="$2"
# wget -t 2 --spider -r -nH -np -nv -nd --reject "index.html" --cut-dirs=3 \
# -A $filename.bz2 $url 2>&1\
# | grep -Eo '(http|https)://(.*).bz2'
# }
# export -f listurls
listurls() {
filename="$1"
url="$2"
wget -qO- $url | grep -Eoi '<a [^>]+>' | \
grep -Eo 'href="[^\"]+"' | \
grep -Eo $filename | \
xargs -I {} echo "$url"{}
}
export -f listurls
#
get_and_extract_one() {
url="$1"
file=`basename $url | sed 's/\.bz2//g'`
if [ ! -f "$file" ]; then
wget -t 2 -q -O - "$url" | bzip2 -dc > "$file"
fi
}
export -f get_and_extract_one
##############################################
download_merge_2d_variable_icon_eu()
{
filename="icon-eu_europe_regular-lat-lon_single-level_${year}${month}${day}${run}_*_${1}.grib2"
filename_grep="icon-eu_europe_regular-lat-lon_single-level_${year}${month}${day}${run}_(.*)_${1}.grib2.bz2"
url="https://opendata.dwd.de/weather/nwp/icon-eu/grib/${run}/${1,,}/"
echo "folder: ${url}"
echo "files: ${filename}"
#
if [ ! -f "${1}_${year}${month}${day}${run}_eur.nc" ]; then
listurls $filename_grep $url | parallel -j 10 get_and_extract_one {}
find ${filename} -empty -type f -delete # Remove empty files
sleep 1
cdo -f nc copy -mergetime ${filename} ${1}_${year}${month}${day}${run}_eur.nc
rm ${filename}
fi
}
export -f download_merge_2d_variable_icon_eu
################################################
download_merge_3d_variable_icon_eu()
{
filename="icon-eu_europe_regular-lat-lon_pressure-level_${year}${month}${day}${run}_*_${1}.grib2"
filename_grep="icon-eu_europe_regular-lat-lon_pressure-level_${year}${month}${day}${run}_(.*)_(1000|850|500|300|50)_${1}.grib2.bz2"
url="https://opendata.dwd.de/weather/nwp/icon-eu/grib/${run}/${1,,}/"
echo "folder: ${url}"
echo "files: ${filename}"
if [ ! -f "${1}_${year}${month}${day}${run}_eur.nc" ]; then
listurls $filename_grep $url | parallel -j 10 get_and_extract_one {}
find ${filename} -empty -type f -delete # Remove empty files
cdo merge ${filename} ${1}_${year}${month}${day}${run}_eur.grib2
rm ${filename}
cdo -f nc copy ${1}_${year}${month}${day}${run}_eur.grib2 ${1}_${year}${month}${day}${run}_eur.nc
rm ${1}_${year}${month}${day}${run}_eur.grib2
fi
}
export -f download_merge_3d_variable_icon_eu
################################################
download_invariant_icon_eu()
{
filename="icon-eu_europe_regular-lat-lon_time-invariant_${year}${month}${day}${run}_HSURF.grib2"
wget -r -nH -np -nv -nd --reject "index.html*" --cut-dirs=3 -A "${filename}.bz2" "https://opendata.dwd.de/weather/nwp/icon-eu/grib/${run}/hsurf/"
bzip2 -d ${filename}.bz2
cdo -f nc copy ${filename} HSURF_${year}${month}${day}${run}_eur.nc
rm ${filename}
}
export -f download_invariant_icon_eu
################################################
download_merge_soil_variable_icon_eu()
{
filename="icon-eu_europe_regular-lat-lon_soil-level_${year}${month}${day}${run}_*_3_${1}.grib2"
filename_grep="icon-eu_europe_regular-lat-lon_soil-level_${year}${month}${day}${run}_(.*)_3_${1}.grib2.bz2"
url="https://opendata.dwd.de/weather/nwp/icon-eu/grib/${run}/${1,,}/"
if [ ! -f "${1}_${year}${month}${day}${run}_eur.nc" ]; then
listurls $filename_grep $url | parallel -j 10 get_and_extract_one {}
cdo -f nc copy -mergetime ${filename} ${1}_${year}${month}${day}${run}_eur.nc
rm ${filename}
fi
}
export -f download_merge_soil_variable_icon_eu