diff --git a/.gitignore b/.gitignore index 3ed3e8b76854f6dc0cb8f79e1661c956572a7584..cafa800b40cfb7157c94ee50969e873bfa644429 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ -.*.sw* -.*.log* +*.pyc +.idea/* +doc/build/* +*.err +*.out +*.nc +.coverage +htmlcov \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..9009fcce0a1d3b740c2178b8c76c25d5e2651015 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "earthdiagnostics/cmor_tables/cmip6"] + path = earthdiagnostics/cmor_tables/cmip6 + url = https://github.com/jvegasbsc/cmip6-cmor-tables.git +[submodule "earthdiagnostics/cmor_tables/primavera"] + path = earthdiagnostics/cmor_tables/primavera + url = https://github.com/jonseddon/cmip6-cmor-tables.git diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000000000000000000000000000000000000..11b4c2eec83862f710ff1381f0c310d7660d08ec --- /dev/null +++ b/CHANGES @@ -0,0 +1,5 @@ +3.0.0 + Complete rewrite in Python + Update CDFTools from version 2.1 to 3.0 + Adoption of CMOR standard + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..fdd60f8f4758922148447187aae13cca9236e00f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +graft earthdiagnostics +include diags.conf +include README +include VERSION \ No newline at end of file diff --git a/README b/README index 805fa9b23f4afc511dc2e2c4d85fae0adddd9715..f247d009952abfbc0a536addd6730abcd6f5f8a6 100644 --- a/README +++ b/README @@ -1,101 +1,12 @@ -To use the ocean diagnostic tools available here and to be able to create new diagnostics, -you need first to add the following line to your .bashrc: -PATH=/cfu/software/cdftools2.1/:$PATH +This tool is a set of diagnostics used at BSC-ES department for NEMO and EC-EARTH models postprocessing. +They are based on CDO, NCO and CDFTOOLS 3.0. For CDFTOOLS, a custom build is required. -#Launching instructions: +FULL DOCUMENTATION AND HOW TOs +============================== -./ocean_pp.bash $path_to_your_file/config_file-ocean_pp.bash -- You can copy and modify the config_file to any directory if you indicate the good path when launching ocean_pp.bash -- ocean_pp.bash, README and example of config_file-ocean_pp.bash can be downloaded from git repository: git clone https://dev.cfu.local/cfutools.git +Check the Earth Diagnostics documentation in PDF format in EarthDiagnostics.pdf available also in this folder. -# +CONTACT +======= -####################### DIAGNOSTICS LIST ####################################### -'siasiesiv' : sea ice area + sea ice extent + sea ice volume -'moc' : Atlantic + Pacific + Indian + Indo-Pacific + Global - meridional overturning streamfunction -'max_moc' : Yearly maximum of the Atlantic meridional overturning - streamfunction between 38N-50N and 500-3000m and - at 40N. - Beware that the max_moc diagnostics can not be computed - if you don't process complete years, as it is a diagnostic - computed from complete years. -'area_moc' : Averaged Atlantic meridional overturning streamfunction - between 40N-55N and 1km-2km and between 30N-40N and 1km-2km -'convection' : indices of convective activity in the Labrador, extended Irminger, - GIN and Wedell seas computed as the maximum in mixed layer depth - based on the potential density criteria -'stc' : Subtropical cell strength, computed as the averaged Atlantic - and Pacific overturning streamfunction in 0-25N/S, 0-200m -'ext_raw_ice' : raw ice varibales to be extracted from input files - (called "ice" before february 2013 update)) -'ext_raw_oce' : raw ocean varibales to be extracted from input files - (called "sstsssmld" before february 2013 update)) -'heat_sal_mxl' : mixed layer heat and salt content (potential density criteria) -'psi' : barotropic streamfunction -'gyres' : indices of the barotropic gyres computed as the maximum absolute - barotropic streamfunction in the North Atlantic and North Pacific - subpolar and subtropical gyres and in the South Atlantic, South - Pacific and Indian subtropical gyres and indice of the Antarctic - Circumpolar Current strength -'usalc' : upper layer salt content 0-300m -'temp_lev' : temperature, vertical mean between dif levels (2D) -'sal_lev' : salinity, vertical mean between dif levels (2D) -'lmsalc' : middle + lower layer salt content 300m-bottom -'ohc' : global total ocean heat content -'ohc_specified_layer' : Pointwise ocean heat content in a specified ocean thickness (at present 0-300 m) -'uohc' : global upper (0-350m) ocean heat content -'mohc' : global middle (350-800m) ocean heat content -'lohc' : global lower (800m-bottom) ocean heat content -'xohc' : global mixed layer ocean heat content -'TSec_ave190-220E' : meridional temperature section averaged between 190E and 220E -'SSec_ave190-220E' : meridional salinity section averaged between 190E and 220E (added in february 2013 update) -'vert_Tsections' : zonal temperature sections along 45S, 0N, 45N, and - meridional temperature sections along 30W, 80E, 180E -'vert_Ssections' : zonal salinity sections along 45S, 0N, 45N, and - meridional salinity sections along 30W, 80E, 180E (added in february 2013 update) -'3dtemp' : 3-dimensional temperature interpolated horizontally onto the - atmospheric grid -'3dsal' : 3-dimensional salinity interpolated horizontally onto the - atmospheric grid (added in february 2013 update) -'NAtlohc' : North Atlantic (10-65N) total ocean heat content -'xNAtlohc' : North Atlantic (10-65N) mixed layer ocean heat content -'uNAtlohc' : North Atlantic (10-65N) upper (0-350m) ocean heat content -'mNAtlohc' : North Atlantic (10-65N) middle (350-800m) ocean heat content -'lNAtlohc' : North Atlantic (10-65N) lower (800m-bottom) ocean heat content -'NPacohc' : North Pacific (10-70N) ocean heat content -'xNPacohc' : North Pacific (10-70N) mixed layer ocean heat content -'uNPacohc' : North Pacific (10-70N) upper (0-350m) ocean heat content -'mNPacohc' : North Pacific (10-70N) middle (350-800m) ocean heat content -'lNPacohc' : North Pacific (10-70N) lower (800m-bottom) ocean heat content -'TAtlohc' : Tropical Atlantic (30S-30N) ocean heat content -'xTAtlohc' : Tropical Atlantic (30S-30N) mixed layer ocean heat content -'uTAtlohc' : Tropical Atlantic (30S-30N) upper (0-350m) ocean heat content -'mTAtlohc' : Tropical Atlantic (30S-30N) middle (350-800m) ocean heat - content -'lTAtlohc' : Tropical Atlantic (30S-30N) lower (800m-bottom) ocean heat - content -'TPacohc' : Tropical Pacific (30S-30N) ocean heat content -'xTPacohc' : Tropical Pacific (30S-30N) mixed layer ocean heat content -'uTPacohc' : Tropical Pacific (30S-30N) upper (0-350m) ocean heat content -'mTPacohc' : Tropical Pacific (30S-30N) middle (350-800m) ocean heat content -'lTPacohc' : Tropical Pacific (30S-30N) lower (800m-bottom) ocean heat - content -'TIndohc' : Tropical Indian (30S-30N) ocean heat content -'xTIndohc' : Tropical Indian (30S-30N) mixed layer ocean heat content -'uTIndohc' : Tropical Indian (30S-30N) upper (0-350m) ocean heat content -'mTIndohc' : Tropical Indian (30S-30N) middle (350-800m) ocean heat content -'lTIndohc' : Tropical Indian (30S-30N) lower (800m-bottom) ocean heat - content -'Antaohc' : Antarctic (90-60S) ocean heat content -'xAntaohc' : Antarctic (90-60S) mixed layer ocean heat content -'uAntaohc' : Antarctic (90-60S) upper (0-350m) ocean heat content -'mAntaohc' : Antarctic (90-60S) middle (350-800m) ocean heat content -'lAntaohc' : Antarctic (90-60S) lower (800m-bottom) ocean heat content -'Arctohc' : Arctic (65-90N) ocean heat content -'xArctohc' : Arctic (65-90N) mixed layer ocean heat content -'uArctohc' : Arctic (65-90N) upper (0-350m) ocean heat content -'mArctohc' : Arctic (65-90N) middle (350-800m) ocean heat content -'lArctohc' : Arctic (65-90N) lower (800m-bottom) ocean heat content -'temp_lev' : vertical mean of ocean temp (weighted) between level1 and level2 (in numbers, not in meters), specified in the config file -'sal_lev' : vertical mean of ocean sal (weighted) between level1 and level2 (in numbers, not in meters), specified in the config file +For any doubts or suggestions, contact javier.vegas@bsc.es \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000000000000000000000000000000000000..c4dd8a8adc11e0beee6be4c8c5bab7389b9c2c55 --- /dev/null +++ b/VERSION @@ -0,0 +1,2 @@ +3.0.0rc1 + diff --git a/bin/earthdiags b/bin/earthdiags new file mode 100644 index 0000000000000000000000000000000000000000..6ac2607276ec2558b7179ce10e942048f989218c --- /dev/null +++ b/bin/earthdiags @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# coding=utf-8 + +"""Script for launching Earth Diagnostics""" +import os +import sys + +scriptdir = os.path.abspath(os.path.dirname(sys.argv[0])) +assert sys.path[0] == scriptdir +sys.path[0] = os.path.normpath(os.path.join(scriptdir, os.pardir)) + +# noinspection PyUnresolvedReferences,PyPep8 +from earthdiagnostics.earthdiags import EarthDiags + + +# noinspection PyProtectedMember +def main(): + """ + Entry point for the Earth Diagnostics + """ + if not EarthDiags.parse_args(): + os._exit(1) + os._exit(0) + +if __name__ == "__main__": + main() diff --git a/common_ocean_post.txt b/common_ocean_post.txt deleted file mode 100644 index 0f468dc3a66c0d5bd44bd7ff249791ac996a3b43..0000000000000000000000000000000000000000 --- a/common_ocean_post.txt +++ /dev/null @@ -1,1345 +0,0 @@ -############################################################################### -# This file gathers a set of bash functions that rely on cdftools to # -# # -# reduce_mmo # -# get_diagsMMO # -# get_nemovar # -# get_glorys # -# clean_diagsMMO # -# vertmeansal # -# heat_sal_mxl # -# ohc_specified_layer # -# moc # -# convection # -# psi # -# gyres # -# area_moc # -# max_moc # -# siasiesiv # -# ohc # -# cutsection # -# interp3d # -# setminmax # -# concat # -# gather_memb # -# vertmeanvar # -# # -# Those functions would never have seen the day without Hui Du, # -# usually referred to as Super-Hui. # -# # -# He made a crucial work to develop what ended up below in the functions # -# that computes the sea ice extent, sea ice area, ocean heat content and # -# meridional overturning streamfunction. # -# Especially, he developped new options from the cdftools sources to be # -# able to compute the heat content in different basins. # -# # -#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -# You want to make available a new diagnostic ? @ -# @ -# 1) Get an MMO tar files from any experiment on esnas @ -# 2) Write a bash function that works on a grid_T or grid_U or grid_V or @ -# grid_W file or a combination from this MMO file @ -# --> You can test your function by defining the CON_FILES and NEMOVERSION @ -# variables and by sourcing the current file, the meshmasks will be @ -# available after sourcing, remember to source again after any @ -# modification of your function @ -# --> Your function should work on input files of any resolution @ -# ORCA1/ORCA025/ORCA2 @ -# --> Your function should work on input files of any time length @ -# --> The output file should contain a proper time axis that you can copy @ -# from your input file @ -# --> The output file should be at most a 3d field including the time @ -# dimension @ -# 3) Write a short description of your function, and add its name to the @ -# list above @ -# 4) Go the the ocean_pp.sh script to add a call to your function @ -# @ -# Any doubt ---> vguemas@ic3.cat @ -#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -# Link constant file for co # -############################################################################### - -cp -f ${CON_FILES}/mesh_mask_nemo.${NEMOVERSION}.nc mesh_hgr.nc -cp -f ${CON_FILES}/mesh_mask_nemo.${NEMOVERSION}.nc mesh_zgr.nc -cp -f ${CON_FILES}/mesh_mask_nemo.${NEMOVERSION}.nc mask.nc -cp -f ${CON_FILES}/new_maskglo.${NEMOVERSION}.nc new_maskglo.nc - -if [ -e ${CON_FILES}/mask.regions.${NEMOVERSION}.nc ] ; then -cp ${CON_FILES}/mask.regions.${NEMOVERSION}.nc mask_regions.nc -fi - -if [[ ! -f mask.nc ]] ; then - echo "No configuration files for cdftools" - exit -fi - -############################################################################### -# Reduced number of variables in diag files to save disk space # -# # -# $1 : input grid_T file name # -# $2 : input icemod file name # -# $3 : suffix of output files with nc extension # -# # -# Created in February 2012 Author : vguemas@ic3.cat # -# May 2014 : Compatibility with PA changes to oce output - Virginie # -############################################################################### - -function reduce_mmo { -ncks -O -v sosstsst,sosaline,somixhgt,somxl010,sossheig $1 oce_${3} -typeset var lstvars=`cdo showvar $2` -if [[ ${lstvars/ileadfra} != ${lstvars} ]] ; then - ncks -O -v isnowthi,iicethic,ileadfra,iicetemp $2 ice_${3} -else - ncks -O -v isnowthi,iicethic,iiceconc,iicetemp $2 ice_${3} -fi -ncks -O -v votemper $1 t3d_${3} -} -############################################################################### -# Copy diags or MMO files from esnas # -# # -# $1 : starting date # -# $2 : expid # -# $3 : member # -# $4 : starting leadtime # -# $5 : end leadtime # -# $6 : chunk length in month # -# $7 : nemo/ecearth # -# $8 : diags/MMO # -# $9 : storage frequency (daily/monthly) # -# $10 : list of files extracted from the original tarballs -# # -# Created in May 2012 Author : vguemas@ic3.cat # -# Option 10: June 2013 isabel.andreu-burillo@ic3.cat -############################################################################### - -function get_diagsMMO { -typeset var yyyy0=`echo $1|cut -c1-4` -typeset var mm0=`echo $1|cut -c5-6` -if [ -z "${10}" ] ; then - typeset var lstypes="grid_T grid_U grid_V grid_W icemod" -else - typeset var lstypes=${10} -fi - -typeset var jt -typeset var year1 -typeset var year2 -typeset var mon1 -typeset var mon2 -for jt in $(seq $4 $6 $5) ; do - year1=$(($yyyy0+(10#$mm0+$jt-2)/12)) - mon1=$(((10#$mm0+$jt-2)%12+1)) - year2=$(($yyyy0+(10#$mm0+$jt+$6-3)/12)) - mon2=$(((10#$mm0+$jt+$6-3)%12+1)) - cp /esnas/exp/$7/$2/$1/fc$3/outputs/$8_$2_$1_fc$3_${year1}$(printf "%02d" $mon1)01-${year2}$(printf "%02d" $mon2)*.tar . - if [[ "$8" == "MMO" ]]; then - for filetype in $lstypes; do - tar --wildcards -xvf $8_$2_$1_fc$3_${year1}$(printf "%02d" $mon1)01-${year2}$(printf "%02d" $mon2)*.tar "*${freqkeep}*${filetype}*" - done - else - tar --wildcards -xvf $8_$2_$1_fc$3_${year1}$(printf "%02d" $mon1)01-${year2}$(printf "%02d" $mon2)*.tar - fi - rm -f $8_$2_$1_fc$3_${year1}$(printf "%02d" $mon1)01-${year2}$(printf "%02d" $mon2)*.tar - if [[ `ls *.gz` != '' ]] ; then gunzip -f *.gz ; fi -done - -typeset var listroots -case $8 in - 'diags' ) - listroots="t3d heat_sal_mxl ice moc psi sal_0-300m sal_300-5400m" - if [[ `ls sstsimld*` != '' ]] ; then - listroots=$listroots" sstsimld" - elif [[ `ls sstsssmld*` != '' ]] ; then - listroots=$listroots" sstsssmld" - else - listroots=$listroots" oce" - fi - ;; - 'MMO' ) listroots=$lstypes ;; -esac -case $9 in - 'daily') freqexcl1='1m' ; freqexcl2='MM' ;; - 'monthly' ) freqexcl1='1d' ; freqexcl2='DD' ;; - *) freqexcl1='1d' ; freqexcl2='DD' ;; -esac - -function concat_startdate { - typeset var root - typeset var lstfiles - for root in ${listroots[@]} ; do - if [[ `ls *${root}*` != '' ]] && [[ `ls *${root}*` != ${root}_1m.nc ]] && [[ `ls *${root}* | grep -v "${freqexcl1}" | grep -v "${freqexcl2}"` != '' ]] ; then - if [[ "$8" == "MMO" ]] ; then - lstfiles=`ls *${root}* | grep -v "${root}_$2_$1_fc" | grep -v "${freqexcl1}" | grep -v "${freqexcl2}" | grep -v "km" ` - else - lstfiles=`ls ${root}* | grep -v "${root}_$2_$1_fc" | grep -v "${freqexcl1}" | grep -v "${freqexcl2}" | grep -v "km" ` - fi - if [[ ! -z ${lstfiles} ]] ; then - file1=`echo "${lstfiles}" | tail -n 1` - cdo_version=`cdo -V &> ff ; grep Climate ff | cut -d \ -f 5` - rm ff - #test on cdo version: if >1.5.6, remove valid_min/max attributes to avoid values out of that range to be replaced by NaN - if [[ "$cdo_version" = "`echo -e "$cdo_version\n1.5.6" | sort -V | head -n1`" ]] ; then - if [[ $root == 'grid_T' || $root == 't3d' ]] ; then - for file in $lstfiles ; do - ncatted -O -a valid_max,votemper,d,, $file $file - ncatted -O -a valid_min,votemper,d,, $file $file - done - fi - if [[ $root == 'heat_sal_mxl' ]] ; then - for file in $lstfiles ; do - ncatted -O -a valid_max,somxlheatc,d,, $file $file - ncatted -O -a valid_min,somxlheatc,d,, $file $file - done - fi - fi - - - outfile=${root}_$2_$1_fc$3_$(($yyyy0+(10#$mm0+$4-2)/12))$(printf "%02d" $(((10#$mm0+$4-2)%12+1)))_${year2}$(printf "%02d" $mon2).nc - typeset var lstvars=`cdo showvar $file1` - if [[ ${lstvars/iicenflx} != ${lstvars} ]] ; then for file in $lstfiles ; do ncks -O -x -v iicenflx $file $file ; done ; fi - cdo mergetime $lstfiles ${outfile} - timevar=`ncdump -h ${outfile} | grep UNLIMITED | awk '{print $1}'` - if [[ $timevar == 'time_counter' ]] ; then ncrename -v time_counter,time -d time_counter,time ${outfile} ; fi - if [[ $root == 'moc' ]] ; then - lstdims=`ncdump -h ${outfile} | awk /dimensions:/,/variables:/ | grep -v dimensions: | grep -v variables: | awk '{print $1}'` - if [[ ${lstdims/gsize} != ${lstdims} ]] ; then - ncrename -d gsize,y ${outfile} - fi - lenx=`ncdump -h ${outfile} | grep 'x =' | head -n 1 | awk '{print $3}'` - if [[ $lenx > 1 ]] ; then - if [[ ${lstvars/nav_lon} != ${lstvars} ]] ; then - ncks -O -x -v nav_lon,nav_lat ${outfile} ${outfile} - fi - ncrename -d x,y ${outfile} - fi - ncks -A -v nav_lon,nav_lat `echo $lstfiles | awk '{print $1}' ` ${outfile} - fi - rm -f $lstfiles - if [[ $root == 'sstsimld' || $root == 'sstsssmld' ]] ; then mv ${outfile} oce_$2_$1_fc$3_$(($yyyy0+(10#$mm0+$4-2)/12))$(printf "%02d" $(((10#$mm0+$4-2)%12+1)))_${year2}$(printf "%02d" $mon2).nc ; fi - fi - fi - done -} - -concat_startdate $1 $2 $3 $4 $5 $6 $7 $8 - -# These lines aim at concatenating the daily means as well and computing the monthly means from these daily means -if [[ $9 == 'monthly' ]] ; then - freqexcl1='1m' ; freqexcl2='MM' - for root in ${listroots[@]} ; do - outfile=${root}_$2_$1_fc$3_$(($yyyy0+(10#$mm0+$4-2)/12))$(printf "%02d" $(((10#$mm0+$4-2)%12+1)))_${year2}$(printf "%02d" $mon2).nc - if [[ -e $outfile ]] ; then - mv $outfile ${root}_1m.nc - fi - done - concat_startdate $1 $2 $3 $4 $5 $6 $7 $8 - for root in ${listroots[@]} ; do - outfile=${root}_$2_$1_fc$3_$(($yyyy0+(10#$mm0+$4-2)/12))$(printf "%02d" $(((10#$mm0+$4-2)%12+1)))_${year2}$(printf "%02d" $mon2).nc - if [[ -e $outfile ]] ; then - cdo monmean $outfile ${root}_daily2monthly.nc - rm -f $outfile - if [[ -e ${root}_1m.nc ]] ; then - mv ${root}_1m.nc $outfile - ncks -A ${root}_daily2monthly.nc $outfile - rm -f ${root}_daily2monthly.nc - else - mv ${root}_daily2monthly.nc $outfile - fi - else - if [[ -e ${root}_1m.nc ]] ; then - mv ${root}_1m.nc $outfile - fi - fi - done -fi - -rm -f *${freqexcl1}* *${freqexcl2}* - -} -############################################################################### -# Copy NEMOVAR files from esnas # -# # -# $1 : expid # -# $2 : member # -# $3 : start year # -# $4 : end year # -# $5 : start month # -# $6 : end month # -# $7 : list of files extracted from the original tarballs -# # -# Created in May 2012 Author : vguemas@ic3.cat # -# Modified: June 2013 isabel.andreu-burillo@ic3.cat # -############################################################################### - -function get_nemovar { - -if [ -z "$5" ] ; then - typeset var moni=9 -else - typeset var moni=$5 -fi - -if [ -z "$5" ] ; then - typeset var monf=8 -else - typeset var monf=$6 -fi - -typeset var path -typeset var yearf -case $1 in - 'nemovar_s4') path=/esnas/exp/ECMWF/NEMOVAR_S4/outputs/fc$2/s4 ;; - 'nemovar_combine') path=/esnas/exp/ECMWF/NEMOVAR_COMBINE/outputs/opa0/fa9p_1m ;; -esac -typeset var year -typeset var mon -for year in $(seq $3 $4) ; do - case $year in - $3) mona=${moni} ;; - *) mona=1 ;; - esac - case $year in - $4) monb=${monf} ;; - *) monb=12 ;; - esac - for mon in $(seq $mona $monb); do - cp ${path}_fc$2_${year}$(printf "%02d" $mon)*.gz . - done -done -gunzip -f *.gz - -typeset var listroots=${7} -typeset var root -typeset var lstfiles -typeset var ntimes -typeset var jt -for root in ${listroots[@]} ; do - lstfiles=`ls *fc${2}*${root}* | grep -v "${root}_$1_195709_fc$2_${3}09_${4}$(printf "%02d" $monb).nc"` - ncrcat -O -x -v vorbiasp $lstfiles tmp_${root}.nc - cdo settaxis,${3}-$(printf "%02d" $moni)-15,12:00,1mon tmp_${root}.nc ${root}_$1_19570901_fc$2_${3}$(printf "%02d" $moni)_${4}$(printf "%02d" $monf).nc - rm -f $lstfiles tmp_${root}.nc -done -} -############################################################################### -# Copy GLORYS files from esnas # -# # -# $1 : start year # -# $2 : end year # -# $3 : start month # -# $4 : end month # -# # -# Created in June 2013 Author : vguemas@ic3.cat # -############################################################################### - -function get_glorys { -typeset var path=/esnas/exp/MERCATOR/GLORYS2V1/outputs/ORCA1L46 #ORCA025L75_glorys -typeset var lstfiles="" -for year in $(seq $1 $2) ; do - cp ${path}/vosaline_${year}.nc . - cp ${path}/votemper_${year}.nc . - ncks -A vosaline_${year}.nc votemper_${year}.nc - rm -f vosaline_${year}.nc - lstfiles=${lstfiles}" "votemper_${year}.nc -done -cdo cat ${lstfiles} tmp.nc -cdo settaxis,${1}-01-15,12:00,1mon tmp.nc tmp2.nc -cdo seldate,${1}-$(printf "%02d" $3)-00,${2}-$(printf "%02d" $4)-31 tmp2.nc grid_T_glorys2v1_19930101_fc0_${1}$(printf "%02d" $3)_${2}$(printf "%02d" $4).nc -rm -f ${lstfiles} tmp.nc tmp2.nc -ncks -O -x -v nav_lon,nav_lat,x_2,y_2 grid_T_glorys2v1_19930101_fc0_${1}$(printf "%02d" $3)_${2}$(printf "%02d" $4).nc grid_T_glorys2v1_19930101_fc0_${1}$(printf "%02d" $3)_${2}$(printf "%02d" $4).nc -ncks -A -v nav_lon,nav_lat mesh_hgr.nc grid_T_glorys2v1_19930101_fc0_${1}$(printf "%02d" $3)_${2}$(printf "%02d" $4).nc -} -############################################################################### -# Clean diags or MMO files after postprocessing # -# # -# $1 : starting date # -# $2 : expid # -# $3 : member # -# $4 : starting leadtime # -# $5 : end leadtime # -# $6 : diags/MMO # -# $7 : list of files extracted from the original tarballs -# # -# Created in May 2012 Author : vguemas@ic3.cat # -# Modified: June 2013 isabel.andreu-burillo@ic3.cat # -############################################################################### - -function clean_diagsMMO { -typeset var yyyy0=`echo $1|cut -c1-4` -typeset var mm0=`echo $1|cut -c5-6` -typeset var year1=$(($yyyy0+(10#$mm0+$4-2)/12)) -typeset var year2=$(($yyyy0+(10#$mm0+$5-2)/12)) -typeset var mon1=$(((10#$mm0+$4-2)%12+1)) -typeset var mon2=$(((10#$mm0+$5-2)%12+1)) - -typeset var listroots - case $6 in - 'diags' ) listroots="t3d" ;; - 'MMO' ) - if [ -z "${7}" ] ; then - listroots="grid_T grid_U grid_V grid_W icemod" - else - listroots=${7} - fi - ;; - esac -typeset var root -typeset var lstfiles -for root in ${listroots[@]} ; do - rm -f ${root}_$2_$1_fc$3_${year1}$(printf "%02d" $mon1)_${year2}$(printf "%02d" $mon2).nc -done -} -############################################################################### -# Vertically averaged salt content # -# # -# $1 : input grid_T file name # -# $2 : upper depth of the layer (in meters) # -# $3 : lower depth of the layer (in meters) # -# $4 : output file name (=> 2D) # -# # -# Created in February 2012 Author : vguemas@ic3.cat # -############################################################################### - -function vertmeansal { -cdo_version=`cdo -V &> ff ; grep Climate ff | cut -d \ -f 5` -rm ff -typeset var ntime=`cdo ntime $1` -typeset var list="" -typeset var jt -for jt in $(seq 1 $ntime); do - ncks -O -d time,$((jt-1)) $1 intvertmeansal.nc - - #test on cdo version: if >1.5.6, remove valid_min/max attributes to avoid values out of that range to be replaced by NaN - if [[ "$cdo_version" > "`echo -e "$cdo_version\n1.5.6" | sort -V | head -n1`" ]] ; then - ncatted -O -a valid_max,deptht,d,, intvertmeansal.nc - ncatted -O -a valid_min,deptht,d,, intvertmeansal.nc - fi - cdfvertmean intvertmeansal.nc vosaline T $2 $3 - ncrename -O -v sovertmean,vertmeansal -d time_counter,time -v time_counter,time vertmean.nc - mv vertmean.nc outputvertmeansal_$jt.nc - list=$list" "outputvertmeansal_$jt.nc - rm -f intvertmeansal.nc -# #test on cdo version: if >1.5.6, remove valid_min/max attributes to avoid values out of that range to be replaced by NaN -# if [[ "$cdo_version" = "`echo -e "$cdo_version\n1.5.6" | sort -V | head -n1`" ]] ; then -# ncatted -O -a valid_max,vertmeansal,d,, outputvertmeansal_$jt.nc outputvertmeansal_$jt.nc -# ncatted -O -a valid_min,vertmeansal,d,, outputvertmeansal_$jt.nc outputvertmeansal_$jt.nc -# fi -done -cdo cat $list $4 -ncks -A -v time $1 $4 -rm -f $list -setminmax $4 vertmeansal -} -############################################################################### -# Compute mixed layer heat and salt content # -# # -# $1 : input grid_T file name # -# $2 : output file name (=> 2D x-y ) # -# # -# Created in February 2012 Author : vguemas@ic3.cat # -################################################################################ - -function heat_sal_mxl { -typeset var ntime=`cdo ntime $1` -typeset var list="" -typeset var jt -typeset var lstvars=`cdo showvar $1` -for jt in $(seq 1 $ntime); do - ncks -O -d time,$((jt-1)) $1 intheat_sal_mxl.nc - if [[ ${lstvars/somxl010} == ${lstvars} ]] ; then - cdfmxl intheat_sal_mxl.nc mxl.nc - ncrename -d time_counter,time mxl.nc - ncks -A mxl.nc intheat_sal_mxl.nc - rm -f mxl.nc - fi - cdfmxlheatc intheat_sal_mxl.nc - if [[ $lstvars != ${lstvars/vosaline} ]] ; then - cdfmxlsaltc intheat_sal_mxl.nc - ncks -A mxlsaltc.nc mxlheatc.nc - rm -f mxlsaltc.nc - fi - mv mxlheatc.nc outputintheat_sal_mxl_$jt.nc - timevar=`ncdump -h outputintheat_sal_mxl_$jt.nc | grep UNLIMITED | awk '{print $1}'` - if [[ $timevar == 'time_counter' ]] ; then ncrename -v time_counter,time -d time_counter,time outputintheat_sal_mxl_$jt.nc ; fi - list=$list" "outputintheat_sal_mxl_$jt.nc - rm -f intheat_sal_mxl.nc -done -cdo cat $list $2 -ncks -A -v time $1 $2 -rm -f $list -setminmax $2 somxlheatc -if [[ $lstvars != ${lstvars/vosaline} ]] ; then setminmax $2 somxlsaltc ; fi -} -############################################################################### -# Pointwise Ocean Heat Content in a specified ocean thickness # -# (J/m-2) -# # -# $1 : input grid_T file name # -# $2 : upper depth of the layer (in meters) # -# $3 : lower depth of the layer (in meters) # -# $4 : output file name (=> 2D x-y ) # -# # -# Created in June 2012 Author : isabel.andreu-burillo@ic3.cat # -# May 2014 - Virginie Guemas - Way around the bc that does not work on moore # -############################################################################### - -function ohc_specified_layer { -typeset var ntime=`cdo ntime $1` -typeset var list="" -typeset var jt -ncap2 -v -O -s "heatc_sl=tmask*e3t" mesh_zgr.nc e3t_file.nc -ncrename -d t,time -d z,deptht e3t_file.nc -for jt in $(seq 1 $ntime); do - cdo seltimestep,$jt $1 intohc_slayer.nc - ncks -O -v votemper intohc_slayer.nc intmeantem.nc - ncrename -v votemper,heatc_sl intmeantem.nc #to be commented - cdo mul intmeantem.nc e3t_file.nc heatc_sl_out.nc -#? ncks -A -m -v nav_lon,nav_lat $1 heatc_sl_out.nc - # extract the data between the two given depths --> heatc_sl_top.nc - ncks -O -d deptht,$2,$3 heatc_sl_out.nc heatc_sl_top.nc - #perform the integration of ohc down to that level (main contribution) - ncap2 -O -s 'heatc_sl=heatc_sl.total($deptht)' heatc_sl_top.nc heatc_sl_top.nc - # now extract a few levels below, to compute the residual ohc - # lower_bnd=`echo "$3 + 200.0" | bc` -> does not work on new moore - # strip out the .* from $3: - stripped=`echo ${3/.*}` - # addition with float returned: - lower_bnd=`echo $(printf "%f" $(( $stripped + 200)))` - ncks -O -d deptht,$3,$lower_bnd heatc_sl_out.nc heatc_sl_bottom.nc - # obtain the weight for the extra level containing the 300 m - # deptht in the gridT files is positive - # weight = (300.0 - depth_top)/(depth_bottom - depth_top) - # and add the thickness down to 300 m in the next layer - ncpdq -a '-deptht' heatc_sl_top.nc heatc_sl_top_invert.nc - ncks -O -d deptht,0,0,1 heatc_sl_top_invert.nc level_above.nc - ncks -O -d deptht,0,0,1 heatc_sl_bottom.nc level_below.nc - ## Here, add the residual contribution, before adding it to the main contribution - ncrename -v deptht,layerthcknss level_below.nc - ncrename -v deptht,layerthcknss level_above.nc - ncbo -A --op_typ=sub -v layerthcknss level_below.nc level_above.nc depth_diff_lay.nc - ncrename -v layerthcknss,heatc_sl depth_diff_lay.nc - ncap2 -s "heatc_sl=($3 - layerthcknss)" level_above.nc depth_diff_sublay.nc - ncbo --op_typ=/ -v heatc_sl depth_diff_sublay.nc depth_diff_lay.nc factor.nc - ncrename -v heatc_sl,factor factor.nc #to be commented - ncks -A -v factor factor.nc level_below.nc - rm -f depth_diff_sublay.nc depth_diff_lay.nc - ncap2 -O -s "heatc_sl=(factor * heatc_sl)" level_below.nc level_below.nc - ncwa -O -a deptht level_below.nc level_below.nc - ncbo --op_typ=+ -v heatc_sl heatc_sl_top.nc level_below.nc total_heatc_sl.nc - ncap2 -s "heatc_sl=1020.0*4000*heatc_sl" total_heatc_sl.nc heatc_sl_$jt.nc - list=$list" "heatc_sl_$jt.nc - rm -f depth_diff_lay.nc depth_diff_sublay.nc - rm -f heatc_sl_out.nc heatc_sl_top.nc heatc_sl_top_invert.nc heatc_sl_bottom.nc - rm -f level_above.nc level_below.nc - rm -f intohc_slayer.nc intmeantem.nc vertmean.nc total_heatc_sl.nc - rm -f factor.nc -done -cdo cat $list $4 -ncks -A -v time $1 $4 -rm -f $list -rm -f e3t_file.nc -setminmax $4 heatc_sl -} -############################################################################### -# Compute the MOC for oceanic basins # -# # -# $1 : input grid_V file name # -# $2 : output file name (=> 2D, depth-y) # -# # -# Created in March 2012 Author : vguemas@ic3.cat # -############################################################################### - -function moc { -typeset var ntime=`cdo ntime $1` -typeset var list="" -typeset var jt -for jt in $(seq 1 $ntime); do - cdo seltimestep,$jt $1 intmoc.nc - cdfmoc intmoc.nc - ncwa -O -a x moc.nc outmoc_$jt.nc - ncks -O -x -v nav_lon,nav_lat outmoc_$jt.nc outmoc_$jt.nc - timevar=`ncdump -h outmoc_$jt.nc | grep UNLIMITED | awk '{print $1}'` - if [[ $timevar == 'time_counter' ]] ; then ncrename -v time_counter,time -d time_counter,time outmoc_$jt.nc ; fi - list=$list" "outmoc_$jt.nc - rm -f intmoc.nc moc.nc -done -cdo cat $list $2 -lstdims=`ncdump -h $2 | awk /dimensions:/,/variables:/ | grep -v dimensions: | grep -v variables: | awk '{print $1}'` -if [[ ${lstdims/gsize} != ${lstdims} ]] ; then - ncrename -d gsize,y $2 -fi -ncks -A -v nav_lon,nav_lat $1 $2 -ncks -A -v time $1 $2 -rm -f $list -} -############################################################################### -# # -# Compute the intensity of convection in the four main convection sites # -# # -# $1 : input oce file name containing somxl010 # -# $2 : input grid # -# $3 : output file name (=> index) # -# # -# Created in October 2013 Author : vguemas@ic3.cat # -############################################################################### - -function convection { -case $2 in - 'Ec2.3_O1L42'|'Ec3.0_O1L46'|'N3.2_O1L42'|'N3.3_O1L46'|'nemovar_O1L42') - A1=225;A2=245;A3=215;A4=255; - B1=245;B2=290;B3=215;B4=245; - C1=260;C2=310;C3=245;C4=291; - D1=225;D2=280;D3=1;D4=50;; - - 'Ec3.0_O25L46'|'Ec3.0_O25L75'|'glorys2v1_O25L75') - stop"Option convection not available yet for this configuration" - ;; -esac - -cdo fldmax -selindexbox,${A1},${A2},${A3},${A4} $1 Labrador.nc -ncrename -v somxl010,Labrador Labrador.nc -ncks -O -v Labrador Labrador.nc convection.nc -rm -f Labrador.nc - -cdo fldmax -selindexbox,${B1},${B2},${B3},${B4} $1 Irminger.nc -ncrename -v somxl010,Irminger Irminger.nc -ncks -A -v Irminger Irminger.nc convection.nc -rm -f Irminger.nc - -cdo fldmax -selindexbox,${C1},${C2},${C3},${C4} $1 GIN.nc -ncrename -v somxl010,GIN GIN.nc -ncks -A -v GIN GIN.nc convection.nc -rm -f GIN.nc - -cdo fldmax -selindexbox,${D1},${D2},${D3},${D4} $1 Wedell.nc -ncrename -v somxl010,Wedell Wedell.nc -ncks -A -v Wedell Wedell.nc convection.nc -rm -f Wedell.nc - -mv convection.nc $3 -} -############################################################################### -# # -# Compute the barotropic stream function # -# # -# $1 : input grid_U file name # -# $2 : input grid_V file name # -# $3 : output file name without nc extension (=> 2D x-y) # -# # -# Created in March 2012 Author : vguemas@ic3.cat # -############################################################################### - -function psi { -typeset var ntime=`cdo ntime $1` -typeset var list="" -typeset var jt -for jt in $(seq 1 $ntime); do - cdo seltimestep,$jt $1 intU.nc - cdo seltimestep,$jt $2 intV.nc - cdfpsi intU.nc intV.nc - mv psi.nc psi_U.nc - cdfpsi intU.nc intV.nc V - mv psi.nc psi_V.nc - ncea psi_U.nc psi_V.nc psi_${jt}.nc - timevar=`ncdump -h psi_$jt.nc | grep UNLIMITED | awk '{print $1}'` - if [[ $timevar == 'time_counter' ]] ; then ncrename -v time_counter,time -d time_counter,time psi_$jt.nc ; fi - list=$list" "psi_$jt.nc - rm -f intU.nc intV.nc psi_U.nc psi_V.nc -done -cdo cat $list ${3} -ncks -A -v time $1 ${3} -rm -f $list -} -############################################################################### -# # -# Compute the intensity of the subtropical and subpolar gyres # -# # -# $1 : input psi file name # -# $2 : input grid # -# $3 : output file name ( => index ) # -# # -# Created in October 2013 Author : vguemas@ic3.cat # -############################################################################### - -function gyres { -case $2 in - 'Ec2.3_O1L42'|'Ec3.0_O1L46'|'N3.2_O1L42'|'N3.3_O1L46'|'nemovar_O1L42') - A1=230;A2=275;A3=215;A4=245; - B1=70;B2=145;B3=195;B4=235; - C1=45;C2=175;C3=165;C4=220; - D1=195;D2=275;D3=175;D4=225; - E1=70;E2=205;E3=120;E4=145; - F1=235;F2=300;F3=120;F4=145; - G1=320;G2=30;G3=110;G4=180; - H1=1;H2=361;H3=1;H4=65;; - - 'Ec3.0_O25L46'|'Ec3.0_O25L75'|'glorys2v1_O25L75') - stop"Option gyres not available yet for this configuration" - ;; -esac - -cdo fldmin -selindexbox,${A1},${A2},${A3},${A4} $1 subpolar_NAtl.nc -ncrename -v sobarstf,subpolNAtl subpolar_NAtl.nc -cdo mulc,-1 subpolar_NAtl.nc gyres.nc -rm -f subpolar_NAtl.nc - -cdo fldmin -selindexbox,${B1},${B2},${B3},${B4} $1 subpolar_NPac.nc -ncrename -v sobarstf,subpolNPac subpolar_NPac.nc -cdo mulc,-1 subpolar_NPac.nc tmp.nc -ncks -A tmp.nc gyres.nc -rm -f subpolar_NPac.nc tmp.nc - -cdo fldmax -selindexbox,${C1},${C2},${C3},${C4} $1 subtrop_NPac.nc -ncrename -v sobarstf,subtropNPac subtrop_NPac.nc -ncks -A subtrop_NPac.nc gyres.nc -rm -f subtrop_NPac.nc - -cdo fldmax -selindexbox,${E1},${E2},${E3},${E4} $1 subtrop_SPac.nc -ncrename -v sobarstf,subtropSPac subtrop_SPac.nc -ncks -A subtrop_SPac.nc gyres.nc -rm -f subtrop_SPac.nc - -cdo fldmax -selindexbox,${D1},${D2},${D3},${D4} $1 subtrop_NAtl.nc -ncrename -v sobarstf,subtropNAtl subtrop_NAtl.nc -ncks -A subtrop_NAtl.nc gyres.nc -rm -f subtrop_NAtl.nc - -cdo fldmax -selindexbox,${F1},${F2},${F3},${F4} $1 subtrop_SAtl.nc -ncrename -v sobarstf,subtropSAtl subtrop_SAtl.nc -ncks -A subtrop_SAtl.nc gyres.nc -rm -f subtrop_SAtl.nc - -cdo fldmax -selindexbox,${G1},${G2},${G3},${G4} $1 subtrop_Ind.nc -ncrename -v sobarstf,subtropInd subtrop_Ind.nc -ncks -A subtrop_Ind.nc gyres.nc -rm -f subtrop_Ind.nc - -cdo fldmax -selindexbox,${H1},${H2},${H3},${H4} $1 ACC.nc -ncrename -v sobarstf,ACC ACC.nc -ncks -A ACC.nc gyres.nc -rm -f ACC.nc - -mv gyres.nc $3 - -} -############################################################################### -# # -# Compute an Atlantic MOC index by averaging the meridional overturning # -# in a latitude band between 1km and 2km # -# or any other index averaging the meridional overturning in # -# a given basin and a given domain # -# # -# $1 : input moc file name # -# $2 : latitude min # -# $3 : latitude max # -# $4 : output file name ( => index ) # -# $5 : depth min (default : 1km) # -# $6 : depth max (default : 2km) # -# $7 : basin (default : zomsfatl) # -# # -# Created in March 2012 Author : vguemas@ic3.cat # -############################################################################### - -function area_moc { -if [ -z "$5" ] ; then - typeset var depmin=-1000.0 -else - typeset var depmin=-$5 -fi -if [ -z "$6" ] ; then - typeset var depmax=-2000.0 -else - typeset var depmax=-$6 -fi -if [ -z "$7" ] ; then - typeset var basin=zomsfatl -else - typeset var basin=$7 -fi -lstdims=`ncdump -h $1 | awk /dimensions:/,/variables:/ | grep -v dimensions: | grep -v variables: | awk '{print $1}'` -if [[ ${lstdims/x} != ${lstdims} ]] ; then - ncwa -O -a x $1 tmpmoc.nc -else - cp $1 tmpmoc.nc -fi -ncrename -O -d y,lat -v nav_lat,lat tmpmoc.nc tmpmoc.nc -ncks -O -v $basin,time,depthw,lat tmpmoc.nc tmpmoc.nc -ncks -O -d lat,$2,$3 -d depthw,${depmax},${depmin} tmpmoc.nc area_moc.nc -cdo vertmean area_moc.nc area_ave_moc.nc -ncap -O -s "coslat[lat]=cos(lat[lat]*3.141592657/180.0)" area_ave_moc.nc area_ave_moc2.nc -ncwa -w coslat -a lat area_ave_moc2.nc area_ave_moc3.nc -ncks -O -v $basin,time area_ave_moc3.nc $4 -rm -f tmpmoc.nc area_moc.nc area_ave_moc2.nc area_ave_moc3.nc -if [[ $4 != area_ave_moc.nc ]] ; then - rm -f area_ave_moc.nc -fi -} -############################################################################### -# # -# Compute an Atlantic MOC index by finding the maximum of the annual # -# mean meridional overturning in a latitude / depth region # -# # -# $1 : input moc file name # -# $2 : latitude min # -# $3 : latitude max # -# $4 : depth mean # -# $5 : depth max # -# $6 : output file name ( => index ) # -# # -# Created in March 2012 Author : vguemas@ic3.cat # -############################################################################### - -function max_moc { -if [ ! -f $6 ] ; then - ncecat -h $1 tmpmoc1.nc - lstdims=`ncdump -h tmpmoc1.nc | awk /dimensions:/,/variables:/ | grep -v dimensions: | grep -v variables: | awk '{print $1}'` - if [[ ${lstdims/x} != ${lstdims} ]] ; then - ncwa -O -a x tmpmoc1.nc tmpmoc1.nc - fi - ncrename -d record,x tmpmoc1.nc - ncpdq -O -h -a time,x tmpmoc1.nc tmpmoc1.nc - ncpdq -O -h -a depthw,x tmpmoc1.nc tmpmoc1.nc - ncpdq -O -h -a y,x tmpmoc1.nc tmpmoc1.nc - cdo yearmean tmpmoc1.nc tmpmoc.nc - typeset var ntime=`cdo ntime tmpmoc.nc` - typeset var list="" - for jt in $(seq 1 $ntime) ; do - cdo seltimestep,$jt tmpmoc.nc tmpmoc2.nc - cdfmaxmoc tmpmoc2.nc atl $2 $3 $4 $5 - mv maxmoc.nc maxmoc_$jt.nc - timevar=`ncdump -h maxmoc_$jt.nc | grep UNLIMITED | awk '{print $1}'` - if [[ $timevar == 'time_counter' ]] ; then ncrename -v time_counter,time -d time_counter,time maxmoc_$jt.nc ; fi - list=${list}" "maxmoc_$jt.nc - rm -f tmpmoc2.nc - done - cdo cat $list $6 - ncks -A -v time tmpmoc.nc $6 - rm -f $list tmpmoc.nc tmpmoc1.nc -fi -} -############################################################################### -# # -# Compute the sea ice extent (1000km2), area (1000km2), volume (km3) # -# and mean thickness (m) in both hemispheres or a specified region. # -# # -# $1 : input ice file name # -# $2 : output file name ( => index ) # -# $3 : region of interest (if empty default is global) # -# # -# Created in April 2012 Author : vguemas@ic3.cat # -# Modified in June 2014 Author : neven.fuckar@ic3.cat # -# # -# Computation of the properties in various selected regions according to # -# mask.regions.${NEMOVERSION}.nc (mask_regions.nc) is based on modification # -# of mask.regions.ORCA1.noverticalinfo.Matt.nc from Matthieu Chevallier. # -# # -############################################################################### - -function siasiesiv { -cp ${CON_FILES}/ice_template.nc toto_N.nc -cp ${CON_FILES}/ice_template.nc toto_S.nc -case ${NEMOVERSION} in - 'Ec3.0_O1L46'|'Ec3.0_O25L46'|'Ec3.0_O25L75') for var in `cdo showvar $1 | head -n 1` -do -[[ $var = "ice_pres" || $var = "iiceconc" ]] && ncrename -v $var,ileadfra $1 -done;; -#'Ec3.0_O1L46'|'Ec3.0_O25L46') ncrename -v ice_pres,ileadfra $1 ;; -#'Ec3.0_O1L46'|'Ec3.0_O25L46') ncrename -v iiceconc,ileadfra $1 ;; -esac - -typeset var ntime=`cdo ntime $1` -typeset var list1="" -typeset var list2="" -typeset var jt - -if [ ! -z "$3" ] ; then - mv mask.nc mask_tmp.nc - mv mask_regions.nc mask.nc - ncrename -h -v $3,tmask mask.nc -fi - -for jt in $(seq 1 $ntime) ; do - cdo seltimestep,$jt $1 tmpice.nc - cdficediags tmpice.nc>ice.txt - for d in N S;do - ncdump toto_${d}.nc > ice_template.cdl - sia=`grep ${d}Area ice.txt |awk '{print $4}'` - sie=`grep ${d}Exnsidc ice.txt|awk '{print $4}'` - siv=`grep ${d}Volume ice.txt|awk '{print $4}'` - sed -e "s/sia =.*/sia = $sia ;/" ice_template.cdl > ice_template2.cdl - sed -e "s/sie =.*/sie = $sie ;/" ice_template2.cdl > ice_template3.cdl - sed -e "s/siv =.*/siv = $siv ;/" ice_template3.cdl > ice_template.cdl - ncgen -o ice_${d}_${jt}.nc ice_template.cdl - rm -f ice_template.cdl ice_template2.cdl ice_template3.cdl - done - list1=$list1" "ice_N_${jt}.nc - list2=$list2" "ice_S_${jt}.nc - rm -f ice.txt tmpice.nc icediags.nc -done -cdo cat $list1 ice_N_${2} -cdo cat $list2 ice_S_${2} -ncks -A -v time $1 ice_N_${2} -ncks -A -v time $1 ice_S_${2} -rm -f $list1 $list2 toto_N.nc toto_S.nc - -for d in N S;do - ncatted -O -a units,sia,m,c,1000km2 ice_${d}_${2} - ncatted -O -a units,sie,m,c,1000km2 ice_${d}_${2} - - ncks -O -v siv ice_${d}_${2} siv_${d}_${2}1 - ncks -O -v sia ice_${d}_${2} sia_${d}_${2}1 - ncrename -h -v sia,siv sia_${d}_${2}1 - ncbo -O --op_typ=dvd siv_${d}_${2}1 sia_${d}_${2}1 sit_${d}_${2} - ncatted -O -a standard_name,siv,m,c,Mean_sea_ice_thickness sit_${d}_${2} - ncatted -O -a long_name,siv,m,c,"Mean sea ice thickness" sit_${d}_${2} - ncatted -O -a units,siv,m,c,m sit_${d}_${2} - cdo ltc,100 sit_${d}_${2} sit_${d}_${2}1 - cdo ifthenelse sit_${d}_${2}1 sit_${d}_${2} sit_${d}_${2}1 sit_${d}_${2}2 - ncrename -h -v siv,sit sit_${d}_${2}2 - ncks -A sit_${d}_${2}2 ice_${d}_${2} - - rm siv_${d}_${2}1 sia_${d}_${2}1 sit_${d}_${2} sit_${d}_${2}1 sit_${d}_${2}2 -done - -setminmax ice_N_${2} sia sie siv sit -setminmax ice_S_${2} sia sie siv sit - -if [ ! -z "$3" ] ; then - ncrename -h -v tmask,$3 mask.nc - mv mask.nc mask_regions.nc - mv mask_tmp.nc mask.nc -fi - -} -############################################################################### -# # -# Compute the total ocean heat extent # -# # -# $1 : input temperature file name # -# $2 : output file name ( => 2D x-y ) # -# $3 : basin (NAtl, NPac, TAtl, TPac, TInd, Anta, Arct, Glob) Default : Glob # -# $4 = 0 if $3 = Glob # -# $4 : mixed layer (1=only, 0=included, -1=without) Default : 0 # -# $5 : upper level of the layer (optional) Default : top # -# $6 : lower level of the layer (optional) Default : bottom # -# # -# Created in May 2012 Author : vguemas@ic3.cat # -############################################################################### -module load CDO/1.5.3-foss-2015a -function ohc { -cp ${CON_FILES}/depth.${NEMOVERSION}.txt depth.txt -# -# Input arguments -# -if [ -z "$3" ] ; then - typeset var basin='Glob' -else - typeset var basin=$3 -fi -if [ -z "$4" ] ; then - typeset var mxl=0 -else - typeset var mxl=$4 -fi -if [ -z "$5" ] ; then - typeset var up=1 -else - typeset var up=$5 -fi -if [ -z "$6" ] ; then - typeset var down=`cat depth.txt | wc -l` -else - typeset var down=$6 -fi - -if [[ ${up} -eq 1 ]] ; then - typeset var depmin=0 -else - typeset var depmin=`cat depth.txt |head -n ${up} |tail -n 1 | awk '{print $2}' | awk '{printf "%.0f",$1}'` -fi -typeset var depmax=`cat depth.txt |head -n ${down} |tail -n 1 | awk '{print $2}' | awk '{printf "%.0f",$1}'` - -cp ${CON_FILES}/heatc_template.nc template_heatc.nc -ncdump template_heatc.nc > template_heatc.cdl -# -# Define some parameters -# -typeset var para -typeset var output -typeset var nlev=`cat depth.txt | wc -l` -if [[ ! -z "$depmin" && ! -z "$depmax" ]] ; then - if [[ $depmin != 0 || ${down} != ${nlev} && ${down} != 0 ]] ; then - output=${depmin}-${depmax}'_' - fi -fi - -case $basin in - 'NAtl') para="atl $mxl 0 0 10 65"; output='NAtl_10N65N_'${output} ;; - 'TAtl') para="atl $mxl 0 0 -30 30" ; output='TAtl_30S30N_'${output} ;; - 'NPac') para="pac $mxl 0 0 10 70" ; output='NPac_10N70N_'${output} ;; - 'TPac') para="pac $mxl 0 0 -30 30" ; output='TPac_30S30N_'${output} ;; - 'Arct') para="atl $mxl 0 0 65 90" ; output='Arc_65N90N_'${output} ;; - 'Anta') para="all $mxl 0 0 -90 -60" ; output='Ant_90S60S_'${output} ;; - 'TInd') para="ind $mxl 0 0 -30 30" ; output='TInd_30S30N_'${output} ;; - 'Glob') para="all $mxl 0 0 0 0" ;; -esac - -case $mxl in - 1) output='mxl_'${output} ;; - -1) output='nonmxl_'${output} ;; -esac -# -# Compute ohc -# -typeset var lstvars=`cdo showvar $1` -typeset var ntime=`cdo ntime $1` -typeset var list="" -typeset var jt -for jt in $(seq 1 $ntime) ; do - cdo seltimestep,$jt $1 tmpohc.nc - lstdims=`ncdump -h tmpohc.nc | awk /dimensions:/,/variables:/ | grep -v dimensions: | grep -v variables: | awk '{print $1}'` - if [[ ${lstdims/x_2} != ${lstdims} ]] ; then - if [[ ${lstdims/x} != ${lstdims} ]] ; then - ncwa -O -a x tmpohc.nc tmpohc.nc - fi - ncrename -d x_2,x tmpohc.nc - fi - if [[ ${lstvars/somxl010} != ${lstvars} ]] ; then - ncks -O -v somxl010 tmpohc.nc mxl.nc - else - cdfmxl tmpohc.nc mxl.nc - fi - cdfheatc-cfu tmpohc.nc $para $up $down > tmp.log -echo $para -echo $up -echo $down - cat tmp.log - thc=`cat tmp.log | grep "Total Heat content :" | awk '{print $5}'`; - uhc=`cat tmp.log | grep "Total Heat content/volume" | awk '{print $5}'`; - sed -e "s/thc =.*/thc = $thc ;/" template_heatc.cdl > template_heatc2.cdl - sed -e "s/uhc =.*/uhc = $uhc ;/" template_heatc2.cdl > template_heatc.cdl - ncgen -o heatc_${jt}.nc template_heatc.cdl - rm -f template_heatc2.cdl tmpohc.nc mxl.nc tmp.log - list=$list" "heatc_${jt}.nc -done -cdo cat $list ${output}$2 -ncks -h -A -v time $1 ${output}$2 -rm -f $list template_heatc.nc template_heatc.cdl depth.txt -setminmax ${output}$2 thc uhc -} -############################################################################### -# # -# Cut a meridional or zonal section # -# # -# # -# $1 : input file # -# $2 : input var # -# $3 : Z/M (zonal / meridional section) # -# $4 : lat/lon # -# $5 : output file ( => 2D ) # -# # -# Created in September 2012 Author : vguemas@ic3.cat # -# # -############################################################################### - -function cutsection { - typeset var ntime=`cdo ntime $1` - typeset var nx=`ncdump -h $1|grep 'x = '|head -n 1|cut -f3 -d" "` - typeset var ny=`ncdump -h $1|grep 'y = '|head -n 1|cut -f3 -d" "` - typeset var nz=`ncdump -h $1|grep 'depth'|head -n 1|cut -f3 -d" "` -cat>section.R<max(lon)) {exactpos=exactpos-360} - } - # Collect the indexes defining the section - listi=array(dim=switch('$3','Z'=$nx-2,'M'=$ny-1)) - listj=array(dim=switch('$3','Z'=$nx-2,'M'=$ny-1)) - for (jpt in 1:length(listi)) { - vect=switch('$3','Z'=lat[jpt,],'M'=lon[,jpt+1]) - if (min(abs(vect-exactpos))<(2*360./$nx)) { - pos=sort(abs(vect-exactpos),index.return=T)\$ix[1] - listi[jpt]=switch('$3','Z'=jpt+1,'M'=pos) - listj[jpt]=switch('$3','Z'=pos,'M'=jpt) - } - } - listi=listi[is.na(listi)==F] - listj=listj[is.na(listj)==F] - print(listi) - print(listj) - # Select variable at those indexes - fnc1=open.ncdf('$1') - varout=array(dim=c(length(listi),$nz,$ntime)) - for (jt in 1:$ntime) { - varin=get.var.ncdf(fnc1,'$2',start=c(1,1,1,jt),count=c($nx,$ny,$nz,1)) - varin[which(mask<0.5)]=1e20 - for (jpt in 1:length(listi)) { - varout[jpt,,jt]=varin[listi[jpt],listj[jpt],] - } - } - close.ncdf(fnc1) - # Write the output - wtime=dim.def.ncdf("time","",seq(1,$ntime),unlim=TRUE) - dimout=array(dim=length(listi)) - for (jpt in 1:length(listi)) { - dimout[jpt]=switch('$3','Z'=lon[listi[jpt],listj[jpt]],'M'=lat[listi[jpt],listj[jpt]]) - } - wsec=switch('$3','Z'=dim.def.ncdf("lon","",dimout),'M'=dim.def.ncdf("lat","",dimout)) - wdep=dim.def.ncdf("deptht","",depth) - wvar=var.def.ncdf("$2","",list(wsec,wdep,wtime),1e20) - fnc2=create.ncdf('$5',wvar) - put.var.ncdf(fnc2,wvar,varout) - close.ncdf(fnc2) -EOF1 -R CMD BATCH section.R -ncks -h -A -v time $1 $5 -} -############################################################################### -# # -# 3-dimensional conservative interpolation to the regular atmospheric grid # -# # -# $1 : input file # -# $2 : input var # -# $3 : output file ( => 3D ) # -# # -# Created in November 2012 Author : vguemas@ic3.cat # -# # -############################################################################### - -function interp3d { - typeset var nz=`ncdump -h $1|grep 'deptht'|head -n 1|cut -f3 -d" "` - [[ ! -f scrip_use ]] && ln -sf /shared/earth/software/scripts/interpolation/scrip_use scrip_use - for lev in $(seq 1 $nz) ; do - ncks -O -d deptht,$((lev-1)) -v $2 $1 tmp_${lev}.nc - ncwa -O -h -a deptht tmp_${lev}.nc tmp_${lev}.nc - [[ ! -f rmp_${NEMOVERSION}_to_regular_lev${lev}.nc ]] && ln -sf /esnas/autosubmit/con_files/weigths/${NEMOVERSION}/rmp_${NEMOVERSION}_to_*_lev${lev}.nc rmp_${NEMOVERSION}_to_regular_lev${lev}.nc - cat > scrip_use_in < 2D field ) # -# # -# Created in February 2012 Author : vguemas@ic3.cat # -# Modified (more generic, -# i.e. for any input var) in December 2014 # -# Author : eleftheria.exarchou@ic3.cat # -############################################################################### - -function vertmeanvar { - typeset var ntime=`cdo ntime $1` - typeset var list="" - typeset var jt -for jt in $(seq 1 $ntime); do - ncks -O -d time,$((jt-1)) $1 a1 - # The oras4 data do not have gdepth data in their mask, but only gdept_0, so: - if grep -q nemovar_s4 $1 ; then - l1=$(( $3 % 6 )) - l2=$(( $4 % 6 )) - ll1=$(( $3 / 6 + 1 )) - ll2=$(( $4 / 6 + 1 )) - lev1=`echo $(cdo output -selvar,gdept_0 mesh_zgr.nc | sed -n ${ll1}p | awk '{ print $'$l1' }')` - lev2=`echo $(cdo output -selvar,gdept_0 mesh_zgr.nc | sed -n ${ll2}p | awk '{ print $'$l2' }')` - else - l1=$(($3+1)) - l2=$(($4+1)) - lev1=`echo $(cdo info -seltimestep,1 -selvar,gdept mesh_zgr.nc | sed -n ${l1}p | awk '{ print $10 }')` - lev2=`echo $(cdo info -seltimestep,1 -selvar,gdept mesh_zgr.nc | sed -n ${l2}p | awk '{ print $10 }')` - fi - cdfvertmean a1 $2 T $lev1 $lev2 - rm -f a1 - ncrename -O -v sovertmean,vertmean -d time_counter,time -v time_counter,time vertmean.nc - mv vertmean.nc outputvertmean_$jt.nc - list=$list" "outputvertmean_$jt.nc - rm -f intvertmean.nc a? -# #test on cdo version: if >1.5.6, remove valid_min/max attributes to avoid values out of that range to be replaced by NaN - ncatted -O -a valid_max,vertmean,d,, outputvertmean_$jt.nc outputvertmean_$jt.nc - ncatted -O -a valid_min,vertmean,d,, outputvertmean_$jt.nc outputvertmean_$jt.nc - ncatted -O -a standard_name,time,a,c,time outputvertmean_$jt.nc outputvertmean_$jt.nc - ncatted -O -a units,time,o,c,'seconds since 1993-05-01 00:00:00' outputvertmean_$jt.nc outputvertmean_$jt.nc - ncatted -O -a long_name,time,a,c,'Time axis' outputvertmean_$jt.nc outputvertmean_$jt.nc -done -cdo cat $list $5 -ncks -A -v time $1 $5 -rm -f $list -setminmax $5 vertmean -ncrename -v vertmean,$2 $5 -#typeset var level=`echo $(cdo info -selvar,$2 -setctomiss,0 $5 | sed -n 2p | awk '{ print $7 }')` -#typeset var lev=`echo $(cdo info -seltimestep,1 -selvar,$2 -setctomiss,0 $1 | grep $level | awk '{ print $1 }')` -lev=$3 -echo $lev -cp $5 tmp_${lev}.nc -## Here we interpolate horizontally onto a regular grid - [[ ! -f rmp_${NEMOVERSION}_to_regular_lev${lev}.nc ]] && ln -sf /esnas/autosubmit/con_files/weigths/${NEMOVERSION}/rmp_${NEMOVERSION}_to_*_lev${lev}.nc rmp_${NEMOVERSION}_to_regular_lev${lev}.nc - [[ ! -f scrip_use ]] && ln -sf /shared/earth/software/scripts/interpolation/scrip_use scrip_use - cat > scrip_use_in <' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/EarthDiagnostics.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/EarthDiagnostics.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/EarthDiagnostics" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/EarthDiagnostics" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/doc/source/codedoc/earthdiagnostics.rst b/doc/source/codedoc/earthdiagnostics.rst new file mode 100644 index 0000000000000000000000000000000000000000..0bd7d0319695a2f2fb0c2e18fcdcd72d70a1258a --- /dev/null +++ b/doc/source/codedoc/earthdiagnostics.rst @@ -0,0 +1,87 @@ +earthdiagnostics +================ + +earthdiagnostics.box +-------------------- +.. automodule:: earthdiagnostics.box + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.cdftools +------------------------- +.. automodule:: earthdiagnostics.cdftools + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.cmorizer +------------------------- +.. automodule:: earthdiagnostics.cmorizer + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.cmormanager +---------------------------- +.. automodule:: earthdiagnostics.cmormanager + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.config +----------------------- +.. automodule:: earthdiagnostics.config + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.constants +-------------------------- +.. automodule:: earthdiagnostics.constants + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.datamanager +---------------------------- +.. automodule:: earthdiagnostics.datamanager + :show-inheritance: + :inherited-members: + :members: + + +earthdiagnostics.diagnostic +--------------------------- +.. automodule:: earthdiagnostics.diagnostic + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.earthdiags +--------------------------- +.. automodule:: earthdiagnostics.earthdiags + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.parser +----------------------- +.. automodule:: earthdiagnostics.parser + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.utils +---------------------- +.. automodule:: earthdiagnostics.utils + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.variable +------------------------- +.. automodule:: earthdiagnostics.variable + :show-inheritance: + :inherited-members: + :members: diff --git a/doc/source/codedoc/general.rst b/doc/source/codedoc/general.rst new file mode 100644 index 0000000000000000000000000000000000000000..215b700a42fca267dd66a98f2db0e6710db8a987 --- /dev/null +++ b/doc/source/codedoc/general.rst @@ -0,0 +1,63 @@ +earthdiagnostics.general +======================== + +earthdiagnostics.general.attribute +---------------------------------- +.. automodule:: earthdiagnostics.general.attribute + :show-inheritance: + :members: + +earthdiagnostics.general.dailymean +---------------------------------- +.. automodule:: earthdiagnostics.general.dailymean + :show-inheritance: + :members: + +earthdiagnostics.general.module +------------------------------- +.. automodule:: earthdiagnostics.general.module + :show-inheritance: + :members: + +earthdiagnostics.general.monthlymean +------------------------------------ +.. automodule:: earthdiagnostics.general.monthlymean + :show-inheritance: + :members: + +earthdiagnostics.general.relink +------------------------------- +.. automodule:: earthdiagnostics.general.relink + :show-inheritance: + :members: + +earthdiagnostics.general.relinkall +---------------------------------- +.. automodule:: earthdiagnostics.general.relinkall + :show-inheritance: + :members: + + +earthdiagnostics.general.rewrite +-------------------------------- +.. automodule:: earthdiagnostics.general.rewrite + :show-inheritance: + :members: + +earthdiagnostics.general.scale +------------------------------ +.. automodule:: earthdiagnostics.general.scale + :show-inheritance: + :members: + +earthdiagnostics.general.simplify_dimensions +-------------------------------------------- +.. automodule:: earthdiagnostics.general.simplify_dimensions + :show-inheritance: + :members: + +earthdiagnostics.general.yearlymean +----------------------------------- +.. automodule:: earthdiagnostics.general.yearlymean + :show-inheritance: + :members: diff --git a/doc/source/codedoc/main.rst b/doc/source/codedoc/main.rst new file mode 100644 index 0000000000000000000000000000000000000000..1229db7286ebd88f6d3970ba426babb62094a365 --- /dev/null +++ b/doc/source/codedoc/main.rst @@ -0,0 +1,11 @@ +******************** +Module documentation +******************** + +.. toctree:: + :titlesonly: + + earthdiagnostics + general + ocean + statistics \ No newline at end of file diff --git a/doc/source/codedoc/ocean.rst b/doc/source/codedoc/ocean.rst new file mode 100644 index 0000000000000000000000000000000000000000..52830bb9ccdc0e114c4506ed1f9d836aeda99834 --- /dev/null +++ b/doc/source/codedoc/ocean.rst @@ -0,0 +1,128 @@ +earthdiagnostics.ocean +====================== + +earthdiagnostics.ocean.areamoc +------------------------------ +.. automodule:: earthdiagnostics.ocean.areamoc + :show-inheritance: + :members: + +earthdiagnostics.ocean.averagesection +------------------------------------- +.. automodule:: earthdiagnostics.ocean.averagesection + :show-inheritance: + :members: + +earthdiagnostics.ocean.convectionsites +-------------------------------------- +.. automodule:: earthdiagnostics.ocean.convectionsites + :show-inheritance: + :members: + +earthdiagnostics.ocean.cutsection +--------------------------------- +.. automodule:: earthdiagnostics.ocean.cutsection + :show-inheritance: + :members: + +earthdiagnostics.ocean.gyres +---------------------------- +.. automodule:: earthdiagnostics.ocean.gyres + :show-inheritance: + :members: + +earthdiagnostics.ocean.heatcontent +---------------------------------- +.. automodule:: earthdiagnostics.ocean.heatcontent + :show-inheritance: + :members: + +earthdiagnostics.ocean.heatcontentlayer +--------------------------------------- +.. automodule:: earthdiagnostics.ocean.heatcontentlayer + :show-inheritance: + :members: + +earthdiagnostics.ocean.interpolate +---------------------------------- +.. automodule:: earthdiagnostics.ocean.interpolate + :show-inheritance: + :members: + +earthdiagnostics.ocean.interpolatecdo +------------------------------------- +.. automodule:: earthdiagnostics.ocean.interpolatecdo + :show-inheritance: + :members: + +earthdiagnostics.ocean.maskland +------------------------------- +.. automodule:: earthdiagnostics.ocean.maskland + :show-inheritance: + :members: + +earthdiagnostics.ocean.maxmoc +----------------------------- +.. automodule:: earthdiagnostics.ocean.maxmoc + :show-inheritance: + :members: + +earthdiagnostics.ocean.mixedlayerheatcontent +-------------------------------------------- +.. automodule:: earthdiagnostics.ocean.mixedlayerheatcontent + :show-inheritance: + :members: + +earthdiagnostics.ocean.mixedlayersaltcontent +-------------------------------------------- +.. automodule:: earthdiagnostics.ocean.mixedlayersaltcontent + :show-inheritance: + :members: + +earthdiagnostics.ocean.moc +-------------------------- +.. automodule:: earthdiagnostics.ocean.moc + :show-inheritance: + :members: + +earthdiagnostics.ocean.mxl +-------------------------- +.. automodule:: earthdiagnostics.ocean.mxl + :show-inheritance: + :members: + +earthdiagnostics.ocean.psi +-------------------------- +.. automodule:: earthdiagnostics.ocean.psi + :show-inheritance: + :members: + +earthdiagnostics.ocean.rotation +------------------------------- +.. automodule:: earthdiagnostics.ocean.rotation + :show-inheritance: + :members: + +earthdiagnostics.ocean.siasiesiv +-------------------------------- +.. automodule:: earthdiagnostics.ocean.siasiesiv + :show-inheritance: + :members: + +earthdiagnostics.ocean.verticalgradient +--------------------------------------- +.. automodule:: earthdiagnostics.ocean.verticalgradient + :show-inheritance: + :members: + +earthdiagnostics.ocean.verticalmean +----------------------------------- +.. automodule:: earthdiagnostics.ocean.verticalmean + :show-inheritance: + :members: + +earthdiagnostics.ocean.verticalmeanmeters +----------------------------------------- +.. automodule:: earthdiagnostics.ocean.verticalmeanmeters + :show-inheritance: + :members: diff --git a/doc/source/codedoc/statistics.rst b/doc/source/codedoc/statistics.rst new file mode 100644 index 0000000000000000000000000000000000000000..e78a54c917af432d1c52d625c7963a8704e3989b --- /dev/null +++ b/doc/source/codedoc/statistics.rst @@ -0,0 +1,15 @@ +earthdiagnostics.statistics +=========================== + +earthdiagnostics.statistics.climatologicalpercentile +---------------------------------------------------- +.. automodule:: earthdiagnostics.statistics.climatologicalpercentile + :show-inheritance: + :members: + +earthdiagnostics.statistics.monthlypercentile +--------------------------------------------- +.. automodule:: earthdiagnostics.statistics.monthlypercentile + :show-inheritance: + :members: + diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..1564091e49dd182f563cdb87a57d27bcd3c17141 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,343 @@ +# -*- coding: utf-8 -*- +# +# Earth Diagnostics documentation build configuration file, created by +# sphinx-quickstart on Fri May 13 12:40:01 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('../..')) +print os.path.abspath('../..') + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.pngmath', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Earth Diagnostics' +copyright = u'2016, BSC-CNS Earth Sciences Department' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents.source ~/vi +# +# The short X.Y version. +version = '3.0b' +# The full version, including alpha/beta/rc tags. +release = '3.0.0b55' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'EarthDiagnosticsd' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'EarthDiagnostics.tex', u'Earth Diagnostics Documentation', + u'BSC-CNS Earth Sciences Department', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'earthdiagnostics', u'Earth Diagnostics Documentation', + [u'BSC-CNS Earth Sciences Department'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'EarthDiagnostics', u'Earth Diagnostics Documentation', + u'BSC-CNS Earth Sciences Department', 'EarthDiagnostics', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = u'Earth Diagnostics' +epub_author = u'BSC-CNS Earth Sciences Department' +epub_publisher = u'BSC-CNS Earth Sciences Department' +epub_copyright = u'2016, BSC-CNS Earth Sciences Department' + +# The basename for the epub file. It defaults to the project name. +#epub_basename = u'Earth Diagnostics' + +# The HTML theme for the epub output. Since the default themes are not optimized +# for small screen space, using the same theme for HTML and epub output is +# usually not wise. This defaults to 'epub', a theme designed to save visual +# space. +#epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or en if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +#epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files shat should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +#epub_tocscope = 'default' + +# Fix unsupported image types using the PIL. +#epub_fix_images = False + +# Scale large images. +#epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#epub_show_urls = 'inline' + +# If false, no index is generated. +#epub_use_index = True + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/config_file.rst b/doc/source/config_file.rst new file mode 100644 index 0000000000000000000000000000000000000000..efe2c79bb060589f84c17434f970265fbaaf9be5 --- /dev/null +++ b/doc/source/config_file.rst @@ -0,0 +1,287 @@ +Configuration file options +========================== + +This section contains the list and explanation about all the options that are available on the configuration file. Use +it as a reference while preparing your configuration file. Each subsection will refer to the matching section from the +config file. Those subsections explanation may be divided itself for the shake of clarity but this further divisions +have nothing to do with the config file syntax itself. + +DIAGNOSTICS +----------- + +This section contains the general configuration for the diagnostics. The explanation has been divided in two subsections: the first +one will cover all the mandatory options that you must specify in every configuration, while the second will cover all +the optional configurations. + +Mandatory configurations +~~~~~~~~~~~~~~~~~~~~~~~~ + +* SCRATCH_DIR: + Temporary folder for the calculations. Final results will never be stored here. + +* DATA_DIR: + ':' separated list of folders to look for data in. It will look for file in the path $DATA_FOLDER/$EXPID and + $DATA_FOLDER/$DATA_TYPE/$MODEL/$EXPID + +* CON_FILES: + Folder containing mask and mesh files for the dataset. + +* FREQUENCY: + Default data frequency to be used by the diagnostics. Some diagnostics can override this configuration or even + ignore it completely. + +* DIAGS: + List of diagnostic to run. No specific order is needed: data dependencies will be enforced. + + +Optional configurations +~~~~~~~~~~~~~~~~~~~~~~~ + +* SCRATCH_MASKS + Common scratch folder for the ocean masks. This is useful to avoid replicating them for each run at the fat nodes. + By default is '/scratch/Earth/ocean_masks' + +* RESTORE_MESHES + By default, Earth Diagnostics only copies the mask files if they are not present in the scratch folder. If this + option is set to true, Earth Diagnostics will copy them regardless of existence. Default is False. + +* DATA_ADAPTOR + This is used to choose the mechanism for storing and retrieving data. Options are CMOR (for our own experiments) or + THREDDS (for anything else). Default value is CMOR + +* DATA_TYPE + Type of the dataset to use. It can be exp, obs or recon. Default is exp. + +* DATA_CONVENTION + Convention to use for file paths and names and variable naming among other things. Can be SPECS, PREFACE, + PRIMAVERA or CMIP6. Default is SPECS. + +* CDFTOOLS_PATH + Path to the folder containing CDFTOOLS executables. By default is empty, so CDFTOOLS binaries must be added to the + system path. + +* MAX_CORES + Maximum number of cores to use. By default the diagnostics will use all cores available to them. It is not + necessary when launching through a scheduler, as Earthdiagnostics can detect how many cores the scheduler has + allocated to it. + +* AUTO_CLEAN + If True, EarthDiagnostics removes the temporary folder just after finsihing. If RAM_DISK is set to True, this value + is ignored and always Default is True + +* RAM_DISK + If set to True, the temporary files is created at the /dev/shm partition. This partition is not mounted from a disk. + Instead, all files are created in the RAM memory, so hopefully this will improve performance at the cost of a much + higher RAM consumption. Default is False. + +* MESH_MASK + Custom file to use instead of the corresponding mesh mask file. + +* NEW_MASK_GLO + Custom file to use instead of the corresponding new mask glo file + +* MASK_REGIONS + Custom file to use instead of the corresponding 2D regions file + +* MASK_REGIONS_3D + Custom file to use instead of the corresponding 3D regions file + + +EXPERIMENT +---------- + +This sections contains options related to the experiment's definition or configuration. + +* MODEL + Name of the model used for the experiment. + +* MODEL_VERSION + Model version. Used to get the correct mask and mesh files + +* ATMOS_TIMESTEP + Time between outputs from the atmosphere. This is not the model simulation timestep! Default is 6. + +* OCEAN_TIMESTEP + Time between outputs from the ocean. This is not the model simulation timestep! Default is 6. + +* ATMOS_GRID + Atmospheric grid definition. Will be used as a default target for interpolation diagnostics. + +* INSTITUTE + Institute that made the experiment, observation or reconstruction + +* EXPID + Unique identifier for the experiment + +* NAME + Experiment's name. By default it is the EXPID. + +* STARTDATES + Startdates to run as a space separated list + +* MEMBER + Members to run as a space separated list. You can just provide the number or also add the prefix + +* MEMBER_DIGITS + Number of minimum digits to compose the member name. By default it is 1. For example, for member 1 member name + will be fc1 if MEMBER_DIGITS is 1 or fc01 if MEMBER_DIGITS is 2 + +* MEMBER_PREFIX + Prefix to use for the member names. By default is 'fc' + +* MEMBER_COUNT_START + Number corresponding to the first member. For example, if your first member is 'fc1', it should be 1. + If it is 'fc0', it should be 0. By default is 0 + +* CHUNK_SIZE + Length of the chunks in months + +* CHUNKS + Number of chunks to run + +* CHUNK_LIST + List of chunks to run. If empty, all diagnostics will be applied to all chunks + +* CALENDAR + Calendar to use for date calculation. All calendars supported by Autosubmit are available. Default is 'standard' + +CMOR +---- + +In this section, you can control how will work the cmorization process. All options belonging to this section are optional. + +Cmorization options +~~~~~~~~~~~~~~~~~~~ + +This options control when and which varibales will be cmorized. + +* FORCE + If True, launches the cmorization, regardless of existence of the extracted files or the package containing the + online-cmorized ones. If False, only the non-present chunks will be cmorized. Default value is False + +* FORCE_UNTAR + Unpacks the online-cmorized files regardless of exstience of extracted files. If 'FORCE is True, this parameter has + no effect. If False, only the non-present chunks will be unpacked. Default value is False. + +* FILTER_FILES + Only cmorize original files containing any of the given strings. This is a space separated list. Default is the + empty string. + +* OCEAN_FILES + Boolean flag to activate or no NEMO files cmorization. Default is True. + +* ATMOSPHERE_FILES + Boolean flag to activate or no IFS files cmorization. Default is True. + +* USE_GRIB + Boolean flag to activate or no GRIB files cmorization for the atmosphere. If activated and no GRIB files are present, + it will cmorize using the MMA files instead (as if it was set to False). Default is True. + +* CHUNKS + Space separated list of chunks to be cmorized. If not provided, all chunks are cmorized + +* VARIABLE_LIST + Space separated list of variables to cmorize. Variables must be specified as domain:var_name. If no one is specified, + all the variables will be cmorized + +Grib variables extraction +************************* + +These three options ares used to configure the variables to be CMORized from the grib atmospheric files. +They must be specified using the IFS code in a list separated by comma. + +You can also specify the levels to extract using one of the the following syntaxes: + +* VARIABLE_CODE +* VARIABLE_CODE:LEVEL, +* VARIABLE_CODE:LEVEL_1-LEVEL_2-...-LEVEL_N +* VARIABLE_CODE:MIN_LEVEL:MAX_LEVEL:STEP + +Some examples to clarify it further: +* Variable with code 129 at level 30000: 129:30000 +* Variable with code 129 at levels 30000, 40000 and 60000: 129:30000-40000-60000 +* Variable with code 129 at levels between 30000 and 600000 with 10000 intervals: + 129:30000:60000:10000 equivalent to 129:30000-40000-50000-60000 + +* ATMOS_HOURLY_VARS + Configuration of variables to be extracted in an hourly basis + +* ATMOS_DAILY_VARS + Configuration of variables to be extracted in a daily basis + +* ATMOS_MONTHLY_VARS + Configuration of variables to be extracted in a monthly basis + +Metadata options +~~~~~~~~~~~~~~~~ +All the options in this subsection will serve just to add the given values to the homonymous attributes in the +cmorized files. + +* ASSOCIATED_EXPERIMENT + Default value is 'to be filled' + +* ASSOCIATED_MODEL + Default value is 'to be filled' + +* INITIALIZATION_DESCRIPTION + Default value is 'to be filled' + +* INITIALIZATION_METHOD + Default value is '1' + +* PHYSICS_DESCRIPTION + Default value is 'to be filled' + +* PHYSICS_VERSION + Default value is '1' + +* SOURCE + Default value is 'to be filled' + +* VERSION + Dataset version to use (not present in all conventions) + +* DEFAULT_OCEAN_GRID + Name of the default ocean grid for those conventions that require it (CMIP6 and PRIMAVERA). Default is gn. + +* DEFAULT_ATMOS_GRID + Name of the default atmos grid for those conventions that require it (CMIP6 and PRIMAVERA). Default is gr. + +* ACTIVITY + Name of the activity. Default is CMIP + +THREDDS +------- + +For now, there is only one option for the THREDDS server configuration. + +* SERVER_URL + THREDDS server URL + + +ALIAS +----- + +This config file section is different from all the others because it does not contain a set of configurations. Instead, +in this section the user can define a set of aliases to be able to launch its most used configurations with ease. To do +this, the user must add an option with named after the desired alias and assign to it the configuration or configurations to launch +when this ALIAS is invoked. See the next example: + +.. code-block:: ini + + ALIAS_NAME = diag,opt1,opt2 diag,opt1new,opt2 + +In this case, the user has defined a new alias 'ALIAS' that can be used launch two times the diagnostic 'diag', +the first with the options 'opt1' and 'opt2' and the second replacing 'opt1' with 'opt1new'. + +In this example, configuring the DIAGS as + +.. code-block:: ini + + DIAGS = ALIAS_NAME + +will be identical to + +.. code-block:: ini + + DIAGS = diag,opt1,opt2 diag,opt1new,opt2 diff --git a/doc/source/developers.rst b/doc/source/developers.rst new file mode 100644 index 0000000000000000000000000000000000000000..0a259ae202cfeb7a4d8c3b0c87827c5baa59a722 --- /dev/null +++ b/doc/source/developers.rst @@ -0,0 +1,60 @@ +Developer's guide +================= + +The tool provides a set of useful diagnostics, but a lot more can be required at anytime. +If you miss something and are able to develop it, you are more than welcome to collaborate. Even if you can not develop +it, please let us know what do you want. + +The first step is to go to the GitLab page for the project ( https://earth.bsc.es/gitlab/es/ocean_diagnostics/ ) +and open a new issue. Be sure that the title is self-explicative and give a detailed description of what you want. +Please, be very explicit about what you want to avoid misunderstandings. + +.. hint:: + + If reading your description, you think that you are taking the developers as stupids, you are doing it perfectly. + +Don't forget to add the relevant tags. At this stage you will have to choose between 'enhancement', if you are proposing +an improvement on a currently available feature, or 'new feature' in any the other case. + +Now, if you are thinking on developing it yourself, please refer to the BSC-ES Git strategy ( wiki_link_when_available ) +If you have any doubts, or just want help to start the development, contact javier.vegas@bsc.es. + + +Developing a diagnostic +----------------------- + +For new diagnostics development, we have some advice to give: + + * Do not worry about performance at first, just create a version that works. Developers can help you to optimize it + later. + + * There is nothing wrong with doing some common preparations in the generate_jobs of the diagnostic. + + * Parallelization is achieved by running multiple diagnostics at a time. You don't need to implement it at + diagnostic level + + * Use the smallest time frame for your diagnostic: if you can work at chunk level, do not ask for full year data. + + * Prefer NCO over CDO, you will have less problems when versions change. + + * Ask for help as soon as you get stuck. + + * Use always the methods in Utils instead of writing your own code. + + * Use meaningful variable names. If you are using short names just to write less, please switch to an editor with + autocompletion! + + * Do not modify the mesh and mask files, another diagnostic can be using them at the same time. + + + + + + + + + + + + + diff --git a/doc/source/diagnostic_list.rst b/doc/source/diagnostic_list.rst new file mode 100644 index 0000000000000000000000000000000000000000..b26b6a26cf6fd53f77763ceed62ed1df4705fbd5 --- /dev/null +++ b/doc/source/diagnostic_list.rst @@ -0,0 +1,734 @@ +# coding=utf-8 + +Diagnostic list +=============== + +In this section you have a list of the available diagnostics, with a small description of each one and a link to +the full documentation. To see what options are available for each diagnostic, see generate_jobs documentation. + +Remember that diagnostics are specified separated by spaces while options are given separated by commas: + +.. code-block:: ini + + DIAGS = diag1 diag2,option1,option2 diag3 + +General +------- + +The diagnostics from this section are of general use and can be used with any variable you may have. Most of them are +meant to help you to solve usual issues that you may have with the data: incorrect metadata, scaled up or down variables, +links missing. This section also contains the diagnostic used to calculate the monthly means. + +att +~~~~ + +Writes a global attributte to all the netCDF files for a given variable. +See :class:`~earthdiagnostics.general.attribute.Attribute` + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Attributte name: + Attributte to write + +4. Attribute value: + Atrribute's new value. Replace ',' with '&;' and ' ' with '&.' to avoid parsing errors when processing the diags + +5. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + +dailymean +~~~~~~~~~ +Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.general.dailymean.DailyMean` + +.. warning:: + + This diagnostic does not use the frequency configuration from the config file. You must specify the original + frequency when calling it. + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Original frequency: + Original frequency to use + +4. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + +module +~~~~~~ +Calculates the module for two given variables and stores the result in a third. +See :class:`~earthdiagnostics.general.module.Module` + +Options: +******** + +1. Domain: + Variables domain + +2. Variable U: + Variable U name + +3. Variable V: + Variable V name + +4. Variable Module: + Variable module name + +5. Grid = '': + Variable grids. Only required in case that you want to use interpolated data. + +monmean +~~~~~~~ +Calculates the monthly mean for a given variable. See :class:`~earthdiagnostics.general.monthlymean.MonthlyMean` + +.. warning:: + + This diagnostic does not use the frequency configuration from the config file. You must specify the original + frequency when calling it. Otherwise, it will always try to use daily data. + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Original frequency = daily: + Original frequency to use + +4. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + +relink +~~~~~~ + +Regenerates the links created in the monthly_mean, daily_mean, etc folders for a given varible. +See :class:`~earthdiagnostics.general.relink.Relink` + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Move old = + True: If True, any data founded in the target directory will be moved to another folder + (called FOLDER_NAME_old) instead of deleted. + +4. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + + +relinkall +~~~~~~~~~ + +Regenerates the links created in the monthly_mean, daily_mean, etc folders for all variables +See :class:`~earthdiagnostics.general.relinkall.RelinkAll` + +Options: +******** + +This diagnostic has no options + +rewrite: +~~~~~~~~ + +Just rewrites the CMOR output of a given variable. Useful to correct metadata or variable units. +See :class:`~earthdiagnostics.general.rewrite.Rewrite` + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + + +scale +~~~~~ + +Scales a given variable using a given scale factor and offset (NEW_VALUE = OLD_VALUE * scale + offset). Useful to +correct errors on the data. + See :class:`~earthdiagnostics.general.scale.Scale` + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Scale value: + Scale factor for the variable + +4. Offset value: + Value to add to the original value after scaling + +5. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + +6. Min limit = NaN: + If there is any value below this threshold, scale will not be applied + +7. Max limit = NaN: + If there is any value above this threshold, scale will not be applied + +8. Frequencies = [*Default_frequency*]: + List of frequencies ('-' separated) to apply the scale on. Default is the frequency defined globally for all the + diagnostics + +simdim +~~~~~~ + +Convert i j files to lon lat when there is no interpolation required, +i.e. lon is constant over i and lat is constat over j + + + See :class:`~earthdiagnostics.general.simplify_dimensions.SimplifyDimensions` + +Options: +******** + +1. Domain: + Variable domain + +2. Variable: + Variable name + +5. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + +yearlymean +~~~~~~~~~~ +Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.general.yearlymean.YearlyMean` + +.. warning:: + + This diagnostic does not use the frequency configuration from the config file. You must specify the original + frequency when calling it. + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Original frequency: + Original frequency to use + +4. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + +Ocean +----- + +The diagnostics from this section are meant to be used with NEMO variables. Some of them will compute new variables +while others just calculate means or sections for variables in the ORCA grid. The interpolation diagnostics are also +included here as they are usually used with variables in the ORCA grid. + +areamoc +~~~~~~~ + +Compute an Atlantic MOC index by averaging the meridional overturning +in a latitude band between 1km and 2km or any other index averaging the meridional overturning in +a given basin and a given domain. See :class:`~earthdiagnostics.ocean.areamoc.AreaMoc` + +.. warning:: + The MOC for the given basin must be calculated previously. Usually, it will suffice to call the 'moc' diagnostic + earlier in the DIAGS list. + +Options: +******** + +1. Min latitude: + Minimum latitude to compute + +2. Max latitude: + Maximum latitude to compute + +3. Min depth: + Minimum depth (in levels) + +4. Max depth: + Maximum depth (in levels) + +5. Basin = 'Global': + Basin to calculate the diagnostic on. + + +averagesection +~~~~~~~~~~~~~~ + +Compute an average of a given zone. The variable MUST be in a regular grid +See :class:`~earthdiagnostics.ocean.averagesection.AverageSection` + +Options: +******** + +1. Variable: + Variable to average + +2. Min longitude: + Minimum longitude to compute + +3. Max longitude: + Maximum longitude to compute + +4. Min latitude: + Minimum latitude to compute + +5. Max latitude: + Maximum latitude to compute + +6. Domain = ocean: + Variable domain + +convectionsites +~~~~~~~~~~~~~~~ + +Compute the intensity of convection in the four main convection sites. +See :class:`~earthdiagnostics.ocean.convectionsites.ConvectionSites` + +Options: +******** + +This diagnostic has no options + +cutsection +~~~~~~~~~~ + +Cuts a meridional or zonal section. See :class:`~earthdiagnostics.ocean.cutsection.CutSection` + +Options: +******** + +1. Variable: + Variable to cut the section on + +2. Zonal: + If True, calculates a zonal section. If False, it will be a meridional one + +3. Value: + Reference value for the section + +4. Domain = ocean: + Variable's domain + + +gyres +~~~~~ + +Compute the intensity of the subtropical and subpolar gyres. See :class:`~earthdiagnostics.ocean.gyres.Gyres` + +Options: +******** + +This diagnostic has no options + +heatcontent +~~~~~~~~~~~ + +Compute the total and mean ocean heat content. See :class:`~earthdiagnostics.ocean.heatcontent.HeatContent` + +Options: +******** + +1. Basin + Basin to calculate the heat content one + +2. Mixed layer: + If 1, reduces the compuation to the mixed layer. If -1, excludes the mixed layer from the computations. + If 0, no effect. + +3. Min depth: + Minimum depth for the calculation in levels. If 0, whole depth is used + +4. Max depth: + Maximum depth for the calculation in levels + +heatcontentlayer +~~~~~~~~~~~~~~~~ + +Point-wise Ocean Heat Content in a specified ocean thickness. +See :class:`~earthdiagnostics.ocean.heatcontentlayer.HeatContentLayer` + +Options: +******** + +3. Min depth: + Minimum depth for the calculation in meteres + +4. Max depth: + Maximum depth for the calculation in meters + +5. Basin = 'Global': + Basin to calculate the heat content on. + +interpolate +~~~~~~~~~~~ + +3-dimensional conservative interpolation to the regular atmospheric grid. +It can also be used for 2D (i,j) variables. See :class:`~earthdiagnostics.ocean.interpolate.Interpolate` + +.. warning:: + This interpolation requires the pre-generated weights that can be found in '/esnas/autosubmit/con_files/weights'. + Make sure that they are available for your configuration. + +Options: +******** + +1. Target grid: + New grid for the data + +2. Variable: + Variable to interpolate + +3. Domain = ocean: + Variable's domain + +4. Invert latitude: + If True, inverts the latitude in the output file. + +5. Original grid = '': + Source grid to choose. By default this is the original data, but sometimes you will want to use another + (for example, the 'rotated' one produced by the rotation diagnostic) + +interpolateCDO +~~~~~~~~~~~~~~ + +Bilinear interpolation to a given grid using CDO. See :class:`~earthdiagnostics.ocean.interpolatecdo.InterpolateCDO` + +.. warning:: + This interpolation is non-conservative, so treat its output with care. It has the advantage that does not require the + pre-generated weights so it can be used when the 'interp' diagnostic is not available. + +Options: +******** + +1. Variable: + variable to interpolate + +2. Target grid: + Variable domain + +3. Domain = ocean: + Variable's domain + +4. Mask oceans = True: + If True, replaces the values in the ocean by NaN. You must only set it to false if, for some reason, you are + interpolating an atmospheric or land variable that is stored in the NEMO grid (yes, this can happen, i.e. with tas). + +5. Original grid = '': + Source grid to choose. By default this is the original data, but sometimes you will want to use another + (for example, the 'rotated' one produced by the rotation diagnostic) + +maskland +~~~~~~~~ + +Replaces all values excluded by the mask by NaN. See :class:`~earthdiagnostics.ocean.maskland.MaskLand` + +Options: +******** + +1. Domain: + Variable to mask domain + +2. Variable: + variable to mask + +3. Cell point = T: + Cell point where variable is stored. Options: T, U, V, W, F + +4. Original grid = '': + Source grid to choose. By default this is the original data, but sometimes you will want to use another + (for example, the 'rotated' one produced by the rotation diagnostic) + +maxmoc +~~~~~~ + +Compute an Atlantic MOC index by finding the maximum of the annual mean meridional overturning in a +latitude / depth region. Output from this diagnostic will be always in yearly frequency. +See :class:`~earthdiagnostics.ocean.maxmoc.MaxMoc` + +.. warning:: + The MOC for the given basin must be calculated previously. Usually, it will suffice to call the 'moc' diagnostic + earlier in the DIAGS list. + +.. warning:: + This diagnostic can only be computed for full years. It will discard incomplete years and only compute the index in + those with the full 12 months available. + +Options: +******** + +1. Min latitude: + Minimum latitude to compute + +2. Max latitude: + Maximum latitude to compute + +3. Min depth: + Minimum depth (in levels) + +4. Max depth: + Maximum depth (in levels) + +5. Basin = 'Global': + Basin to calculate the diagnostic on. + +mixedlayerheatcontent +~~~~~~~~~~~~~~~~~~~~~ + +Compute mixed layer heat content. +See :class:`~earthdiagnostics.ocean.mixedlayerheatcontent.MixedLayerHeatContent` + +Options: +******** + +This diagnostic has no options + +mixedlayersaltcontent +~~~~~~~~~~~~~~~~~~~~~ + +Compute mixed layer salt content. See :class:`~earthdiagnostics.ocean.mixedlayersaltcontent.MixedLayerSaltContent` + +Options: +******** + +This diagnostic has no options + +moc +~~~~ + +Compute the MOC for oceanic basins. Required for 'areamoc' and 'maxmoc' See :class:`~earthdiagnostics.ocean.moc.Moc` + +Options: +******** + +This diagnostic has no options + +mxl +~~~~ + +Compute the mixed layer depth. See :class:`~earthdiagnostics.ocean.mxl.Mxl` + +Options: +******** + +This diagnostic has no options + +psi +~~~~ + +Compute the barotropic stream function. See :class:`~earthdiagnostics.ocean.psi.Psi` + +Options: +******** + +This diagnostic has no options + +regmean +~~~~~~~ + +Computes the mean value of the field (3D, weighted). For 3D fields, +a horizontal mean for each level is also given. If a spatial window +is specified, the mean value is computed only in this window. +See :class:`~earthdiagnostics.ocean.regionmean.RegionMean` + + +Options: +******** + +1. Domain: + Variable domain + +2. Variable: + Variable to average + +3. Grid_point: + NEMO grid point used to store the variable: T, U, V ... + +4. Basin = Global: + Basin to compute + +5. Save 3d = True: + If True, it also stores the average per level + +6. Min depth: + Minimum depth to compute in levels. If -1, average from the surface + +7. Max depth: + Maximum depth to compute in levels. If -1, average to the bottom + +8. Variance = False: + If True, it also stores the variance + +9. Original grid = '': + Source grid to choose. By default this is the original data, but sometimes you will want to use another + (for example, the 'rotated' one produced by the rotation diagnostic) + + +rotate +~~~~~~ + +Rotates the given variables +See :class:`~earthdiagnostics.ocean.rotation.Rotation` + + +Options: +******** + +1. Variable u: + Variable's u component + +2. Variable v: + Variable's u component + +3. Domain = ocean: + Variable domain: + +4. Executable = /home/Earth/jvegas/pyCharm/cfutools/interpolation/rotateUVorca: + Path to the executable that will compute the rotation + +.. warning:: + This default executable has been compiled for ORCA1 experiments. For other resolutions you must use other + executables compiled ad-hoc for them + +siasiesiv +~~~~~~~~~ + +Compute the sea ice extent , area and volume in both hemispheres or a specified region. +See :class:`~earthdiagnostics.ocean.siasiesiv.Siasiesiv` + +Options: +******** + +1. Basin = 'Global': + Basin to restrict the computation to. + +vgrad +~~~~~ + +Calculates the gradient between two levels in a 3D ocean variable. +See :class:`~earthdiagnostics.ocean.verticalgradient.VerticalGradient` + +Options: +******** + +1. Variable: + Variable to compute + +2. Upper level = 1: + Upper level. Will be used as the reference to compute the gradient + +3. Lower level = 2: + Lower level. + +verticalmean +~~~~~~~~~~~~ + +Chooses vertical level in ocean, or vertically averages between 2 or more ocean levels. +See :class:`~earthdiagnostics.ocean.verticalmean.VerticalMean` + +Options: +******** + +1. Variable: + Variable to average + +2. Min depth = -1: + Minimum level to compute. If -1, average from the surface + +3. Max depth: + Maximum level to compute. If -1, average to the bottom + + +verticalmeanmeters +~~~~~~~~~~~~~~~~~~ + +Averages vertically any given variable. +See :class:`~earthdiagnostics.ocean.verticalmeanmeters.VerticalMeanMeters` + +Options: +******** + +1. Variable: + Variable to average + +2. Min depth = -1: + Minimum depth to compute in meters. If -1, average from the surface + +3. Max depth: + Maximum depth to compute in meters. If -1, average to the bottom + + +Statistics +---------- + +climpercent +~~~~~~~~~~~ + +Calculates the specified climatological percentile of a given variable. +See :class:`~earthdiagnostics.statistics.climatologicalpercentile.ClimatologicalPercentile` + +Options: +******** + +1. Domain: + Variable's domain + +2. Variable: + Variable to compute diagnostic on + +3. Leadtimes: + Leadtimes to compute + +4. Bins: + Number of bins to use to discretize the variable + +monpercent +~~~~~~~~~~ + +Calculates the specified monthly percentile of a given variable. +See :class:`~earthdiagnostics.statistics.monthlypercentile.MonthlyPercentile` + +Options: +******** + +1. Domain: + Variable's domain + +2. Variable: + Variable to compute diagnostic on + +3. Percentiles: + List of requested percentiles ('-' separated) + diff --git a/doc/source/errors.rst b/doc/source/errors.rst new file mode 100644 index 0000000000000000000000000000000000000000..d71948c1cf2c596167e3f9993418a3e30b123c37 --- /dev/null +++ b/doc/source/errors.rst @@ -0,0 +1,38 @@ +What to do if you have an error +=============================== + +Sometimes, the diagnostics may crash and you will not know why. This section will give you a procedure to follow before +reporting the issue. This procedure is intended to solve some common problems or, at least, to help you in creating +good issue reports. Remember: a good issue report reduces the time required to solve it! + +.. hint:: + + Please, read carefully the error message. Most times the error message will point you to the problem's source and + sometimes even give you a hint of how to solve it by yourself. And if this it not the case or if you find it + obscure, even if it was helpful, please contact the developers so it can be improved in further versions + +Try this simple steps BEFORE reporting an issue + +* Clean scratch folder +* Update to the latest compatible tag: maybe your issue is already solved in it +* If you get the error for the first chunk of a given diagnostic, change the number of chunks to 1 +* Call the diags with the -lc DEBUG -log log.txt options + +Now, you have two options: if everything is fine, the error was probably due to some corrupted files or some unstable +machine state. Nevertheless, try running the diagnostic with -lc DEBUG -log log.txt for all the chunks. If everything +it's fine that's all. + +If you experienced the same problem again, go to the GitLab portal and look into the open issues +( https://earth.bsc.es/gitlab/es/ocean_diagnostics/issues ). If you find your issue or a very similar one, use it to +report your problems. If you can not find an open one that suites your problem, create a new one and explain what is +happening to you. In any case, it will be very useful if you can attach your diags.conf and log.txt files and specify +the machine you were using. + +After that, it's just a matter of waiting for the developers to do their work and answering the questions that they may +have. Please, be patient. + +.. caution:: + + Of course, there is a third option: you keep experiencing an error that appears randomly on some executions but you + are not able to reproduce it in a consistent manner. Report it and attach as much logs and configuration files as + you have, along with the date and time of the errors. \ No newline at end of file diff --git a/doc/source/faq.rst b/doc/source/faq.rst new file mode 100644 index 0000000000000000000000000000000000000000..656a2a6684c7b7f7fee490ea4e98437343356591 --- /dev/null +++ b/doc/source/faq.rst @@ -0,0 +1,4 @@ +Frequently Asked Questions +========================== + +Here will be the answers to the most usual questions. For the moment, there is nothing to see here... \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..af80a81d839e7576465d416b928020bcaea78874 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,19 @@ +.. Earth Diagnostics documentation master file, created by + sphinx-quickstart on Fri May 13 12:40:01 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Earth Diagnostics's documentation! +============================================= + +.. toctree:: + :maxdepth: 3 + + tutorial + config_file + diagnostic_list + tips + errors + developers + faq + codedoc/main diff --git a/doc/source/tips.rst b/doc/source/tips.rst new file mode 100644 index 0000000000000000000000000000000000000000..2ef743657bb699f882fd0862e416a8a3f8e46299 --- /dev/null +++ b/doc/source/tips.rst @@ -0,0 +1,31 @@ +Tips and tricks +=============== + +Working with ORCA1 +------------------ + +If you plan to run diagnostics for ORCA1 resolution, be aware that your workstation will be more than capable to run +them. At this resolution, memory and CPU consumption is low enough to allow you keep using the machine while running, +specially if you reserve a pair of cores for other uses. + +Configuring core usage +---------------------- + +By default, the Earth Diagnostics creates a thread for each available core for the execution. If you are using a queueing +system, the diagnostics will always use the number of cores that you reserved. If you are running outside a queueing +system, the diagnostics will try to use all the cores on the machine. To avoid this, add the MAX_CORES parameter to the +DIAGNOSTICS section inside the diags.conf file that you are using. + +Cleaning temp file +------------------ + +By default, EarthDiagnostics removes the temporary directory after execution. This behaviour can be avoided be setting +ra + +By default + +.. code-block:: bash + + earthdiags -f PATH_TO_CONF --clean + + diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst new file mode 100644 index 0000000000000000000000000000000000000000..52d071c54ead176430a7a1b0aa6cbf3c6acbe62f --- /dev/null +++ b/doc/source/tutorial.rst @@ -0,0 +1,75 @@ +Tutorial +======== + +So, you are planning to use the Earth Diagnostics? You don't know how to use them? This is the place to go. +From now on this tutorial will guide you through all the process from installation to running. + +.. Hint:: + If you have any problem with this tutorial, please report it to so it can be corrected. + A lof of people will benefit from it. + +Installation +------------ + +If you have access to the BSC-ES machines, you don't need to install it. Just use the available module: + +.. code-block:: + + module load EarthDiagnostics + + +In case that you need a custom installation for development or can not use the BSC-ES machines, +install it from BSC-ES GitLab repository: + +.. code-block:: sh + + pip install git+https://earth.bsc.es/gitlab/es/ocean_diagnostics.git + +You will also need + +* CDO version 1.7.2 (other versions could work, but this is the one we use) +* NCO version 4.5.4 or newer +* Python 2.7 or newer (but no 3.x) with bscearth.utils, CDO and NCO packages, among others. +* Access to CDFTOOLS_3.0 executables for BSC-ES. The source code is available on Github (https://github.com/jvegasbsc/CDFTOOLS) and it can be compiled with CMake + + +Creating a config file +---------------------- + +Go to the folder where you installed the EarthDiagnostics. You will see a folder called earthdiagnostics, +and, inside it, the model_diags.conf file that can be used as a template for your config file. Create a copy of it +wherever it suites you. + +Now open your brand new copy with your preferred text editor. The file contains commentaries explaining each +one of its options, so read it carefully and edit whatever you need. Don't worry about DIAGS option, we will +talk about it next. + +After this, you need to choose the diagnostics you want to run. For a simple test, it's recommended to use the monmean +diagnostic to compute monthly means from daily data. We recommend it because it can be used with any variable, the user +has to provide parameters but they are quite intuitive and it's relatively fast to compute. If your experiment does not +have daily data, you can use any other diagnostic. Check next section for a list of available diagnostics and choose +whichever suits you better. From now on, we will assume that you are going to run the monmean diagnostic. + +.. hint:: + + For old Ocean Diagnostics users: you can use most of the old names as aliases to launch one or multiple diagnostics. + Check the ALIAS section on the model_diags.conf to see which ones are available. + +First, choose a variable that has daily data. Then replace the DIAGS option with the next one where $VARIABLE represents +the variable's name and $DOMAIN its domain (atmos, ocean, seaice, landice...) + +.. code-block:: sh + + DIAGS = monmean,$DOMAIN,$VARIABLE + +Prepare the run script +---------------------- + +Once you have configured your experiment you can execute any diagnostic with the provided model_launch_diags.sh script. +Create a copy and change the variable PATH_TO_CONF_FILE so it points to your conf file . + +Now, execute the script (or submit it to bsceslogin01, it has the correct header) and... that's it! +You will find your results directly on the storage and a folder for the temp files in the scratch named after the EXPID. + + + diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b7d4ac394205ea62d2f2d4e3308211f97ddacf44 Binary files /dev/null and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/__init__.py b/earthdiagnostics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..75244852b819646f7172bbc4187109cad3850719 --- /dev/null +++ b/earthdiagnostics/__init__.py @@ -0,0 +1,14 @@ +# coding=utf-8 +""" +Module containing the Earth Diagnostics. +""" +from cdo import Cdo +from nco import Nco +from earthdiagnostics.cdftools import CDFTools +import os + +cdo = Cdo() +nco = Nco() +cdftools = CDFTools('/home/Earth/jvegas/CDFTOOLS_3.0/bin') +DEVNULL = open(os.devnull, 'wb') + diff --git a/earthdiagnostics/box.py b/earthdiagnostics/box.py new file mode 100644 index 0000000000000000000000000000000000000000..f3dc76693e3130c51c453294f593561cc28cbbec --- /dev/null +++ b/earthdiagnostics/box.py @@ -0,0 +1,162 @@ +# coding=utf-8 +class Box(object): + """ + Represents a box in the 3D space. Also allows easy conversion from the coordinate values to significant string + representations + """ + def __init__(self, depth_in_meters=False): + self.depth_in_meters = depth_in_meters + """ + If True, treats the depth as if it is given in meters. If False, as it is given in levels + :rtype: bool + """ + self._max_lat = None + self._min_lat = None + self._max_lon = None + self._min_lon = None + self.max_depth = None + """ + Maximum depth + :rtype: float + """ + self.min_depth = None + """ + Minimum depth + :rtype: float + """ + + def __eq__(self, other): + return self.depth_in_meters == other.depth_in_meters and self.max_lat == other.max_lat and \ + self.min_lat == other.min_lat and self.max_lon == other.max_lon and self.min_lon == other.min_lon and \ + self.max_depth == other.max_depth and self.min_depth == other.min_depth + + def __str__(self): + return self.get_lat_str() + self.get_lon_str() + self.get_depth_str() + + @property + def max_lat(self): + """ + Maximum latitude + :rtype: float + """ + return self._max_lat + + @max_lat.setter + def max_lat(self, value): + if value > 90 or value < -90: + raise ValueError('{0} is not a valid latitude. Must be between -90 and -90'.format(value)) + self._max_lat = value + + @property + def min_lat(self): + """ + Minimum latitude + :rtype: float + """ + return self._min_lat + + @min_lat.setter + def min_lat(self, value): + if value > 90 or value < -90: + raise ValueError('{0} is not a valid latitude. Must be between -90 and 90'.format(value)) + self._min_lat = value + + @property + def max_lon(self): + """ + Maximum longitude + :rtype: float + """ + return self._max_lon + + @max_lon.setter + def max_lon(self, value): + if value >= 360 or value <= -360: + raise ValueError('{0} is not a valid longitude. Must be between -360 and 360'.format(value)) + self._max_lon = value + + @property + def min_lon(self): + """ + Minimum longitude + :rtype: float + """ + return self._min_lon + + @min_lon.setter + def min_lon(self, value): + if value >= 360 or value <= -360: + raise ValueError('{0} is not a valid longitude. Must be between -360 and 360'.format(value)) + self._min_lon = value + + def get_lat_str(self): + """ + Gets a string representation of the latitude in the format XX{N/S}. + If min_lat is different from max_lat, it concatenates the two values + :return: string representation for latitude + :rtype: str + """ + if self.max_lat is None or self.min_lat is None: + return '' + if self.min_lat < 0: + direction = 'S' + else: + direction = 'N' + + string = str(abs(self.min_lat)) + direction + + if self.max_lat != self.min_lat: + if self.max_lat < 0: + direction = 'S' + else: + direction = 'N' + string += str(abs(self.max_lat)) + direction + + return string + + def get_lon_str(self): + """ + Gets a string representation of the longitude in the format XX{E/W}. + If min_lon is different from max_lon, it concatenates the two values + :return: string representation for longitude + :rtype: str + """ + if self.max_lon is None or self.min_lon is None: + return '' + if self.min_lon < 0: + direction = 'W' + else: + direction = 'E' + + string = str(abs(self.min_lon)) + direction + + if self.max_lon != self.min_lon: + if self.max_lon < 0: + direction = 'W' + else: + direction = 'E' + string += str(abs(self.max_lon)) + direction + return string + + def get_depth_str(self): + """ + Gets a string representation of depth. For depth expressed in meters, it adds th character 'm' to the end + If min_depth is different from max_depth, it concatenates the two values + :return: string representation for depth + :rtype: str + """ + if self.max_depth is None or self.min_depth is None: + return '' + + if self.depth_in_meters: + suffix = 'm' + else: + suffix = '' + + string = str(abs(self.max_depth)) + suffix + + if self.min_depth != self.max_depth: + string = '{0}-{1}'.format(str(abs(self.min_depth)), string) + return string + + diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py new file mode 100644 index 0000000000000000000000000000000000000000..36fffdaacce1d61d790900f21e8c75d626c7ebf9 --- /dev/null +++ b/earthdiagnostics/cdftools.py @@ -0,0 +1,94 @@ +# coding=utf-8 +from earthdiagnostics.utils import Utils +import os +from bscearth.utils.log import Log +import six + + +class CDFTools(object): + """ + Class to run CDFTools executables + + :param path: path to CDFTOOLS binaries + :type path: str + """ + + def __init__(self, path=''): + self.path = path + + # noinspection PyShadowingBuiltins + def run(self, command, input, output=None, options=None, log_level=Log.INFO, input_option=None): + """ + Runs one of the CDFTools + + :param command: executable to run + :type command: str | iterable + :param input: input file + :type input: str + :param output: output file. Not all tools support this parameter + :type options: str + :param options: options for the tool. + :type options: str | [str] | Tuple[str] | NoneType + :param log_level: log level at which the output of the cdftool command will be added + :type log_level: int + :param input_option: option to add before input file + :type input_option: str + """ + + line = [os.path.join(self.path, command)] + + self._check_command_existence(line[0]) + if input_option: + line.append(input_option) + self._check_input(command, input, line) + if options: + if isinstance(options, six.string_types): + options = options.split() + for option in options: + line.append(str(option)) + if output: + if input == output: + raise ValueError('Input and output file can not be the same on CDFTools') + line.append('-o') + line.append(output) + Log.debug('Executing {0}', ' '.join(line)) + shell_output = Utils.execute_shell_command(line, log_level) + + self._check_output_was_created(line, output) + return shell_output + + @staticmethod + def _check_output_was_created(line, output): + if output: + if not os.path.isfile(output): + raise Exception('Error executing {0}\n Output file not created', ' '.join(line)) + + # noinspection PyShadowingBuiltins + @staticmethod + def _check_input(command, input, line): + if input: + if isinstance(input, six.string_types): + line.append(input) + if not os.path.isfile(input): + raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, input) + else: + for element in input: + line.append(element) + if not os.path.isfile(element): + raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) + + # noinspection PyMethodMayBeStatic + def is_exe(self, fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + def _check_command_existence(self, command): + if self.path: + if self.is_exe(os.path.join(self.path, command)): + return + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, command) + if self.is_exe(exe_file): + return + raise ValueError('Error executing {0}\n Command does not exist in {1}'.format(command, self.path)) diff --git a/earthdiagnostics/cdftoolspython.so b/earthdiagnostics/cdftoolspython.so new file mode 100755 index 0000000000000000000000000000000000000000..2d3916295e882a4723b0fdda32ceab0ed0d26105 Binary files /dev/null and b/earthdiagnostics/cdftoolspython.so differ diff --git a/earthdiagnostics/cmor_tables/cmip6 b/earthdiagnostics/cmor_tables/cmip6 new file mode 160000 index 0000000000000000000000000000000000000000..78eb04bd32dcc398323b21b1cb0636b2f07ffc68 --- /dev/null +++ b/earthdiagnostics/cmor_tables/cmip6 @@ -0,0 +1 @@ +Subproject commit 78eb04bd32dcc398323b21b1cb0636b2f07ffc68 diff --git a/earthdiagnostics/cmor_tables/default.csv b/earthdiagnostics/cmor_tables/default.csv new file mode 100644 index 0000000000000000000000000000000000000000..fd13cc4fa08ba52ef6288fb180cdf81cf86598d4 --- /dev/null +++ b/earthdiagnostics/cmor_tables/default.csv @@ -0,0 +1,348 @@ +Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables +iiceages:siage:iice_otd,ageice,age_of_sea_ice,Age of sea ice,seaIce,,,,,, +al,al,surface_albedo,Albedo,atmos,,,,,, +bgfrcsal,bgfrcsal,change_over_time_in_heat_content_from_forcing,Change over time in salt content from forcing,ocean,,,,,, +bgfrctem,bgfrctem,change_over_time_in_heat_content_from_forcing,Change over time in heat content from forcing,ocean,,,,,, +bgfrcvol,bgfrcvol,change_over_time_in_volume_from_forcing,Change over time in volume from forcing,ocean,,,,,, +bgheatco,bgheatco,change_over_time_in_heat_content,Change over time in sea water heat content,ocean,,,,,, +bgsaline,bgsaline,change_over_time_in_sea_water_practical_salinity,Change over time in sea water salinity,ocean,,,,,, +bgsaltco,bgsaltco,change_over_time_in_salt_content,Change over time in sea water salt content,ocean,,,,,, +bgtemper,bgtemper,change_over_time_in_sea_water_potential_temperature,Change over time in sea water potential temperature,ocean,,,,,, +bgvole3t,bgvole3t,change_over_time_in_volume_variation,Change over time in volume variation (e3t),ocean,,,,,, +bgvolssh,bgvolssh,change_over_time_in_sea_surface_height,Change over time in sea surface height,ocean,,,,,, +bld,bld,boundary_layer_dissipation,Boundary layer dissipation,atmos,,,,,, +iicebome:iocewflx,bmelt,tendency_of_sea_ice_amount_due_to_basal_melting,Rate of melt at sea ice base,seaIce,,,,,, +sobowlin,bowlin,bowl_index,Bowl index,ocean,,,,,, +cc,cl,cloud_area_fraction_in_atmosphere_layer,Cloud area fraction,atmos,,,,,, +hcc,clh,high_cloud_area_fraction,High cloud fraction,atmos,,,,,, +lcc,cll,low_cloud_area_fraction,Low cloud fraction,atmos,,,,,, +mcc,clm,medium_cloud_area_fraction,Medium cloud fraction,atmos,,,,,, +ciwc,cli,mass_fraction_of_cloud_ice_in_air,Mass fraction of cloud ice,atmos,,,,,, +tcc,clt,cloud_area_fraction,Total cloud fraction,atmos,,,,,, +clw,clw,mass_fraction_of_cloud_liquid_water_in_air,Mass fraction of cloud liquid water,atmos,,,,,, +clwc,clwc,mass_fraction_of_convective_cloud_liquid_water_in_air,Mass fraction of convective cloud liquid water,atmos,,,,,, +tcw,clwvi,atmosphere_cloud_condensed_water_content,Condensed water path,atmos,,,,,, +iicedive:sidive,divice,Strain Rate Divergence of Sea Ice,Divergence_of_sea_ice_velocity,seaIce,,,,,, +e,evspsbl,water_evaporation_flux,Evaporation,atmos,,,,,, +fal,fal,forecast_albedo,Forecast albedo,atmos,,,,,, +sowaflep,fatmosocean,atmosphere_ocean_water_flux,Atmos=>ocean net freshwater,ocean,,,,,, +sowaflcd,fdilution,dilution_water_flux,Concentration/dilution water flux,ocean,,,,,, +sophtldf,fhbasindif,northward_ocean_heat_transport_due_to_diffusion,Northward ocean heat transport due to diffusion,ocean,,,,,, +iowaflup,ficeocean,ice_ocean_water_flux,Ice=>ocean net freshwater,ocean,,,,,, +sorunoff,friver,water_flux_into_sea_water_from_rivers,Water flux into sea water from rivers ,ocean,,,,,, +sowaflup,fupward,upward_water_flux,Net upward water flux,ocean,,,,,, +gwd,gwd,gravity_wave_dissipation,Gravity wave dissipation,atmos,,,,,, +ibgheatco,hcicega,global mean ice heat content,Global mean ice heat content,seaIce,,,,,, +sbgheatco,hcsnga,global mean snow heat content,Global mean snow heat content,seaIce,,,,,, +heatc,heatc,integral_of_sea_water_potential_temperature_wrt_depth_expressed_as_heat_content,Heat content vertically integrated,ocean,,J m-2,,,, +sohtatl,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Atl,,,,, +sohtind,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Ind,,,,, +sohtipc,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,IndPac,,,,, +sohtpac,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Pac,,,,, +sophtadv,hfbasinadv,northward_ocean_heat_transport_due_to_advection,Northward ocean heat transport due to advection ,ocean,,,,,, +sophteiv,hfbasinba,northward_ocean_heat_transport_due_to_bolus_advection,Northward ocean heat transport due to bolus advection ,ocean,,,,,, +qt_oce:sohefldo:qt,hfds,surface_downward_heat_flux_in_sea_water,Downward heat flux at sea water surface,ocean,,,,,, +slhf,hfls,surface_upward_latent_heat_flux,Surface upward latent heat flux,atmos,,,,,, +sshf,hfss,surface_upward_sensible_heat_flux,Surface upward sensible heat flux,atmos,,,,,, +sophtove,htovovrt,northward_ocean_heat_transport_due_to_overturning,Northward ocean heat transport due to overturning ,ocean,,,,,, +q,hus,specific_humidity,Specific humidity,atmos,,,,,, +soicealb,ialb,sea_ice_albedo,Sea ice albedo,seaIce,,,,,, +ibgfrcsfx,ibgfrcsfx,global_mean_forcing_salt,Global mean forcing salt (sfx),seaIce,,,,,, +ibgfrcvol,ibgfrcvol,globa_mean_forcing_volume,Global mean forcing volume (emp),seaIce,,,,,, +ibghfxbog,ibghfxbog,heat_fluxes_causing_bottom_ice_growth,Heat fluxes causing bottom ice growth,seaIce,,,,,, +ibghfxbom,ibghfxbom,heat_fluxes_causing_bottom_ice_melt,Heat fluxes causing bottom ice melt,seaIce,,,,,, +ibghfxdhc,ibghfxdhc,Heat_content_variation_in_snow_and_ice,Heat content variation in snow and ice,seaIce,,,,,, +ibghfxdif,ibghfxdif,heat_fluxes_causing_ice temperature_change,Heat fluxes causing ice temperature change,seaIce,,,,,, +ibghfxdyn,ibghfxdyn,heat_fluxes_from_ice-ocean_exchange_during_dynamic,Heat fluxes from ice-ocean exchange during dynamic,seaIce,,,,,, +ibghfxin,ibghfxin,total_heat_fluxes_at_the_ice_surface,Total heat fluxes at the ice surface,seaIce,,,,,, +ibghfxopw,ibghfxopw,heat_fluxes_causing_open_water_ice_formation,Heat fluxes causing open water ice formation,seaIce,,,,,, +ibghfxout,ibghfxout,non_solar_heat_fluxes_received_by_the_ocean,Non solar heat fluxes received by the ocean,seaIce,,,,,, +ibghfxres,ibghfxres,heat_fluxes_from_ice-ocean_exchange_during_resultant,Heat fluxes from ice-ocean exchange during resultant,seaIce,,,,,, +ibghfxsnw,ibghfxsnw,heat_fluxes_from_snow-ocean_exchange,Heat fluxes from snow-ocean exchange,seaIce,,,,,, +ibghfxspr,ibghfxspr,Heat_content_of_snow_precip,Heat content of snow precip,seaIce,,,,,, +ibghfxsub,ibghfxsub,heat_fluxes_from_sublimation,Heat fluxes from sublimation,seaIce,,,,,, +ibghfxsum,ibghfxsum,heat_fluxes_causing_surface_ice_melt,Heat fluxes causing surface ice melt,seaIce,,,,,, +ibghfxthd,ibghfxthd,heat_fluxes_from_ice-ocean_exchange_during_thermo,Heat fluxes from ice-ocean exchange during thermo,seaIce,,,,,, +ibgsfxbog,ibgsfxbogga,salt_flux_thermo,Global mean salt flux (thermo),seaIce,,,,,, +ibgsfxbom,ibgsfxbomga,salt_flux_bottom_melt,Global mean salt flux (bottom melt),seaIce,,,,,, +ibgsfxbri,ibgsfxbriga,salt_flux_brines,Global mean salt flux (brines),seaIce,,,,,, +ibgsfxdyn,ibgsfxdynga,salt_flux_dynamic,Global mean salt flux (dynamic),seaIce,,,,,, +ibgsfx,ibgsfxga,salt_flux,Global mean salt flux (total),seaIce,,,,,, +ibgsfxopw,ibgsfxopwga,salt_flux_open_waters,Global mean salt flux (open water),seaIce,,,,,, +ibgsfxres,ibgsfxresga,salt_flux_resultant,Global mean salt flux (resultant),seaIce,,,,,, +ibgsfxsni,ibgsfxsniga,salt_flux_snow_ice_growth,Global mean salt flux (snow-ice growth),seaIce,,,,,, +ibgsfxsum,ibgsfxsumga,salt_flux_surface_melt,Global mean salt flux (surface melt),seaIce,,,,,, +ibgvfxbog,ibgvfxbogga,volume_flux_bottom_growth,Global mean volume flux (bottom growth),seaIce,,,,,, +ibgvfxbom,ibgvfxbomga,volume_flux_bottom_melt,Global mean volume flux (bottom melt),seaIce,,,,,, +ibgvfxdyn,ibgvfxdynga,volume_flux_dynamic_growth,Global mean volume flux (dynamic growth),seaIce,,,,,, +ibgvfx,ibgvfxga,volume_flux_emp,Global mean volume flux (emp),seaIce,,,,,, +ibgvfxopw,ibgvfxopwga,volume_flux_open_water_growth,Global mean volume flux (open water growth),seaIce,,,,,, +ibgvfxres,ibgvfxresga,volume_flux_resultant,Global mean volume flux (resultant),seaIce,,,,,, +ibgvfxsni,ibgvfxsniga,volume_flux_snow_ice_growth,Global mean volume flux (snow-ice growth),seaIce,,,,,, +ibgvfxsnw,ibgvfxsnwga,volume_flux_snow_melt,Global mean volume flux (snow melt),seaIce,,,,,, +ibgvfxspr,ibgvfxsprga,snheco,Global mean volume flux (snow precip),seaIce,,,,,, +ibgvfxsub,ibgvfxsubga,volume_flux_snow_sublimation,Global mean volume flux (snow sublimation),seaIce,,,,,, +ibgvfxsum,ibgvfxsumga,volume_flux_surface_melt,Global mean volume flux (surface melt),seaIce,,,,,, +ibgvolgrm,ibgvolgrm,global_mean_ice_growth+melt_volume,Global mean ice growth+melt volume,seaIce,,,,,, +ibrinvol,ibrinvol,brine_volume,Brine volume,seaIce,,,,,, +sibricat,ibrinvolcat,brine_volume_in_categories,Brine volume for categories,seaIce,,,,,, +iicebopr,iicebopr,daily_bottom_thermo_ice_production,Daily bottom thermo ice production,seaIce,,,,,, +iicecolf,iicecolf,frazil_ice_collection_thickness,Frazil ice collection thickness,seaIce,,,,,, +iicedypr,iicedypr,daily_dynamic_ice_production,Daily dynamic ice production,seaIce,,,,,, +iice_etd,iiceetd,brine_volume_distribution,Brine volume distribution,seaIce,,,,,, +iicelapr,iicelapr,daily_lateral_thermo_ice_production,Daily lateral thermo ice prod.,seaIce,,,,,, +iicenflx,iicenflx,nonsolar_flux_ice_ocean_surface,Non-solar flux at ice/ocean surface,seaIce,,,,,, +iicesflx,iicesflx,solar_flux_ice_ocean_surface,Solar flux at ice/ocean surface,seaIce,,,,,, +iiceshea,iiceshea,shear,Shear,seaIce,,,,,, +iicesipr,iicesipr,daily_snowice_ice_production,Daily snowice ice production,seaIce,,,,,, +iicfsbri,iicfsbri,brine_salt_flux,Fsbri - brine salt flux,seaIce,,,,,, +iicfseqv,iicfseqv,equivalent_FW_salt_flux,Fseqv - equivalent fw salt flux,seaIce,,,,,, +ioceflxb,ioceflxb,oceanic_flux_ar_ice_base,Oceanic flux at the ice base,seaIce,,,,,, +iocehebr,iocehebr,heat_flux_due_to_brine_release,Heat flux due to brine release,seaIce,,,,,, +iocesafl,iocesafl,salt_flux_ocean_surface,Salt flux at ocean surface,seaIce,,,,,, +iocesflx,iocesflx,solar_fux_ocean_surface,Solar flux at ocean surface,seaIce,,,,,, +iocetflx,iocetflx,total_flux_ocean_surface,Total flux at ocean surface,seaIce,,,,,, +iocwnsfl,iocwnsfl,nonsolar_flux_ocean_surface,Non-solar flux at ocean surface,seaIce,,,,,, +isstempe,isstempe,sea_surface_temperature,Sea surface temperature,seaIce,,K,,,, +scmastot,masso,sea_water_mass,Sea water mass ,ocean,,,,,, +mldkz5,mldkz5,ocean_mixed_layer_thickness_defined_by_vertical_tracer_diffusivity,Turbocline depth (kz = 5e-4),ocean,,,,,, +somxl010:mldr10_1,mlotst,ocean_mixed_layer_thickness_defined_by_sigma_t,Ocean mixed layer thickness defined by sigma T ,ocean,,,,,, +swvl1,mrlsl1,moisture_content_of_soil_layer_1, Water content of soil layer 1,land,,,,,, +swvl2,mrlsl2,moisture_content_of_soil_layer_2, Water content of soil layer 2,land,,,,,, +swvl3,mrlsl3,moisture_content_of_soil_layer_3, Water content of soil layer 3,land,,,,,, +swvl4,mrlsl4,moisture_content_of_soil_layer_4, Water content of soil layer 4,land,,,,,, +ro,mrro,runoff_flux,Total runoff,atmos,,,,,, +tp:precip,pr,precipitation_flux,Precipitation,atmos,,,,,, +cp,prc,convective_precipitation_flux,Convective precipitation,atmos,,,,,, +lsp,prs,stratiform_precipitation_flux,Stratiform precipitation,atmos,,,,,, +isnowpre,prsn,snowfall_flux,Surface snowfall rate into the sea ice portion of the grid cell,seaIce,,,,,, +sf:snowpre,prsn,snowfall_flux,Snowfall flux,atmos,,,,,, +tcwv,prw,atmosphere_water_vapor_content,Water vapor path,atmos,,,,,, +msl,psl,air_pressure_at_sea_level,Sea level pressure,atmos,,,,,, +qns_ice,qnsice,non_solar_heat_flux_at_ice_surface,Non-solar heat flux at ice surface: sum over categories,seaIce,,,,,, +qt_ice,qtice,surface_downward_heat_flux_in_air,Surface downward heat flux in air,seaIce,,,,,, +strd,rlds,surface_downwelling_longwave_flux_in_air,Surface downwelling longwave radiation,atmos,,,,,, +strc:str,rls,surface_longwave_flux_in_air,Surface longwave radiation,atmos,,,,,, +ttr,rlut,toa_outgoing_longwave_flux,Toa outgoing longwave radiation,atmos,,,,,, +ttrc,rlutcs,toa_outgoing_longwave_flux_assuming_clear_sky,"Top net thermal radiation, clear sky",atmos,,,,,, +ssrd,rsds,surface_downwelling_shortwave_flux_in_air,Surface downwelling shortwave radiation,atmos,,,,,, +tsr,rsdt,toa_incoming_shortwave_flux,Toa incident shortwave radiation,atmos,,,,,, +soshfldo,rsntds,net_downward_shortwave_flux_at_sea_water_surface,Net downward shortwave radiation at sea water surface ,ocean,,,,,, +ssr,rss,surface_shortwave_flux_in_air,Surface shortwave radiation,atmos,,,,,, +ssrc,rsscs,surface_shortwave_flux_in_air_assuming_clear_sky,Surface clear-sky shortwave radiation,atmos,,,,,, +tsrc,rsut,toa_outgoing_shortwave_flux,Toa outgoing shortwave radiation,atmos,,,,,, +saltc,saltc,salt_content_vertically_integrated,Salt content vertically integrated,ocean,,,,,, +es,sbl,surface_snow_and_ice_sublimation_flux,Surface snow and ice sublimation flux,landIce,,,,,, +sosalflx,sfs,salt_flux_surface,Surface salt flux,ocean,,,,,, +si,si,solar_insolation,Solar insolation,atmos,,,,,, +NArea,siarean,sea_ice_area,Total area of sea ice in the northern hemisphere,seaIce,,10^6 km2,,,, +SArea,siareas,sea_ice_area,Total area of sea ice in the southern hemisphere,seaIce,,10^6 km2,,,, +iiceconc:siconc:soicecov:ileadfra:ci,sic,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,,, +ci,sic,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,,ifs, +iice_itd:siconc_cat:siconcat,siccat,ice_area_in_categories,Ice area in categories,seaIce,,,,,, +ibgarea,sicga,sea_ice_content,Global mean sea ice content,seaIce,,,,,, +NExnsidc,siextentn,sea_ice_extent,Total area of all northern-hemisphere grid cells that are covered by at least 15 % areal fraction of sea ice,seaIce,,10^6 km2,,,, +SExnsidc,siextents,sea_ice_extent,Total area of all southern-hemisphere grid cells that are covered by at least 15 % areal fraction of sea ice,seaIce,,10^6 km2,,,, +iiceprod,sigr,ice_production,Ice production,seaIce,,,,,, +iiceheco,siheco,integral_of_sea_ice_temperature_wrt_depth_expressed_as_heat_content,Sea ice heat content,seaIce,,,,,, +ibgsaltco,sisaltcga,global mean ice salt content,Global mean ice salt content,seaIce,,,,,, +iicethic:sithic,sit,sea_ice_thickness,Sea Ice Thickness,seaIce,,m,,,, +iice_hid:sithic_cat:sithicat,sitcat,ice_thicknesss_in_categories,Ice thickness in categories,seaIce,,,,,, +iicetemp,sitemp,ice_temperature,Mean ice temperature,seaIce,,K,,,, +ibgtemper,sitempga,sea_ice_temperature,Global mean sea ice temperature,seaIce,,K,,,, +iicevelo:sivelo,sivelo,ice_velocity,Ice velocity,seaIce,,,,,, +iicevelu:sivelu,sivelu,ice_velocity_u,Ice velocity u,seaIce,,,,,, +iicevelv:sivelv,sivelv,ice_velocity_v,Ice velocity v,seaIce,,,,,, +ibgvoltot,sivolga,sea_ice_volume,Global mean sea ice volume,seaIce,,,,,, +sivoln:NVolume,sivoln,sea_ice_volume,Total volume of sea ice in the northern hemisphere,seaIce,,10^3 km3,,,, +sivols:SVolume,sivols,sea_ice_volume,Total volume of sea ice in the southern hemisphere,seaIce,,10^3 km3,,,, +sivolu,sivolu,sea_ice_volume_per_unit_gridcell_area,Sea ice volume per gridcell area unit,seaIce,,,,,, +sostatl,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,,, +sostind,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,,, +sostipc,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,,, +sostpac,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,,, +sopstadv,sltbasinadv,northward_ocean_salt_transport_due_to_advection,Northward ocean salt transport due to advection ,ocean,,,,,, +sopsteiv,sltbasinba,northward_ocean_salt_transport_due_to_bolus_advection,Northward ocean salt transport due to bolus advection ,ocean,,,,,, +sopstldf,sltbasindif,northward_ocean_salt_transport_due_to_diffusion,Northward ocean salt transport due to diffusion,ocean,,,,,, +sltnortha,sltnortha,northward_ocean_salt_transport,Atlantic northward ocean salt transport,ocean,,,,,, +sopstove,sltovovrt,northward_ocean_salt_transport_due_to_overturning,Northward ocean salt transport due to overturning ,ocean,,,,,, +zosalatl,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Atl,psu,,,, +zosalglo,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Glob,psu,,,, +zosalind,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Ind,psu,,,, +zosalipc,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,IndPac,psu,,,, +zosalpac,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Pac,psu,,,, +asn,snal,snow_albedo,Snow albedo,landIce,,,,,, +iice_hsd:snthicat,sndcat,snow_thickness_in_categories,Snow thickness in in categories,seaIce,,,,,, +isnoheco,snheco,snow_heat_content,Snow total heat content,seaIce,,,,,, +sd,snld,lwe_thickness_of_surface_snow_amount,Snow depth,atmos,,,,,, +smlt,snm,surface_snow_melt_flux,Surface snow melt,landIce,,,,,, +isnowthi,snthic,surface_snow_thickness,Surface snow thickness,seaIce,,,,,, +sbgvoltot,snvolga,snow_volume,Global mean snow volume,seaIce,,,,,, +snvolu,snvolu,snow_volume_per_unit_gridcell_area,Snow volume per gridcell area unit,seaIce,,,,,, +vosaline:mean_3Dsosaline,so,sea_water_salinity,Sea water salinity,ocean,,psu,,,, +scsaltot,soga,sea_water_salinity,Global mean sea water salinity ,ocean,,psu,,,, +soleaeiw,soleaeiw,eddy_induced_velocity_coefficient,Eddy induced vel. coeff. at w-point,ocean,,,,,, +soleahtw,soleahtw,lateral_eddy_diffusivity,Lateral eddy diffusivity,ocean,,,,,, +somixhgt,somixhgt,mixing_layer_depth_turbocline,Mixing layer depth (turbocline),ocean,,,,,, +sosaline:isssalin:mean_sosaline,sos,sea_surface_salinity,Sea surface salinity ,ocean,,psu,,,, +sothedep,sothedep,thermocline_depth,Thermocline depth (max dt/dz),ocean,,,,,, +src,src,skin_reservoir_content,Skin reservoir content,land,,,,,, +zosrfatl,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Atl,,,,, +zosrfglo,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Glob,,,,, +zosrfind,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Ind,,,,, +zosrfipc,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,IndPac,,,,, +zosrfpac,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Pac,,,,, +rsn,srho,snow_density,Snow density,landIce,,,,,, +iicesali:iice_std,ssi,sea_ice_salinity,Sea ice salinity,seaIce,,psu,,,, +salincat,ssicat,sea_ice_salinity_in_categories,Sea-ice bulk salinity for categories,seaIce,,psu,,,, +ibgsaline,ssiga,sea_ice_salinity,Global mean sea ice salinity ,seaIce,,psu,,,, +iicestre,streng,compressive_strength_of_sea_ice,Compressive sea ice strength,seaIce,,,,,, +so20chgt,t20d,depth_of_isosurface_of_sea_water_potential_temperature,,ocean,,,,,, +t,ta,air_temperature,Air temperature,atmos,,K,,,, +t2m,tas,air_temperature,Near-surface air temperature,atmos,,K,170,370,, +mx2t,tasmax,air_temperature,Daily maximum near-surface air temperature,atmos,,K,,,, +mn2t,tasmin,air_temperature,Daily minimum near-surface air temperature,atmos,,K,,,, +ewss,tauu,surface_downward_eastward_stress,Surface downward eastward wind stress,atmos,,,,,, +utau_ice:iocestru:iicestru,strairx,surface_downward_x_stress,X-Component of Atmospheric Stress On Sea Ice,seaIce,,N m-2,,,, +sozotaux,tauuo,surface_downward_x_stress,Surface downward x stress ,ocean,,,,,, +nsss,tauv,surface_downward_northward_stress,Surface downward northward wind stress,atmos,,,,,, +vtau_ice:iocestrv:iicestrv,strairy,surface_downward_y_stress,Y-Component of Atmospheric Stress On Sea Ice,seaIce,,N m-2,,,, +sozotauy:sometauy,tauvo,surface_downward_y_stress,Surface downward y stress ,ocean,,,,,, +d2m,tdps,dew_point_temperature,2m dewpoint temperature,atmos,,K,,,, +votemper:mean_3Dsosstsst,thetao,sea_water_potential_temperature,Sea water potential temperature,ocean,,K,,,, +sctemtot,thetaoga,sea_water_potential_temperature,Global average sea water potential temperature ,ocean,,K,,,, +iicesume,tmelt,tendency_of_sea_ice_amount_due_to_surface_melting,Rate of melt at upper surface of sea ice,seaIce,,,,,, +sosstsst:mean_sosstsst,tos,sea_surface_temperature,Sea surface temperature ,ocean,,K,170,370,, +sstk,tos,sea_surface_temperature,Sea surface temperature ,ocean,,K,170,370,ifs, +tossq,tossq,square_of_sea_surface_temperature,Square of sea surface temperature ,ocean,,K2,,,, +zotematl,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Atl,K,,,, +zotemglo,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Glob,K,,,, +zotemind,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Ind,K,,,, +zotemipc,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,IndPac,K,,,, +zotempac,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Pac,K,,,, +skt,ts,surface_temperature,Surface temperature,atmos,,K,,,, +iicesurt:soicetem:sistem,tsice,surface_temperature,Surface temperature of sea ice,seaIce,,K,,,, +istl1,tsice,surface_temperature,Surface temperature of ice,landIce,,K,,,, +stl1,tsl1,soil_temperature_level_1,Temperature of soil level 1,land,,,,,, +stl2,tsl2,soil_temperature_level_2,Temperature of soil level 2,land,,,,,, +stl3,tsl3,soil_temperature_level_3,Temperature of soil level 3,land,,,,,, +stl4,tsl4,soil_temperature_level_4,Temperature of soil level 4,land,,,,,, +tsn,tsn,temperature_in_surface_snow,Snow internal temperature,landIce,,,,,, +u,ua,eastward_wind,U velocity,atmos,,,,,, +u10m,uas,eastward_wind,Eastward near-surface wind,atmos,,,,,, +vozocrtx,uo,sea_water_x_velocity,Sea water x velocity,ocean,,,,,, +uos,uos,sea_surface_x_velocity,Sea surface x velocity,ocean,,,,,, +v,va,northward_wind,V velocity,atmos,,,,,, +v10m,vas,northward_wind,Northward near-surface wind,atmos,,,,,, +vomecrty,vo,sea_water_y_velocity,Sea water y velocity,ocean,,,,,, +vos,vos,sea_surface_y_velocity,Sea surface y velocity,ocean,,,,,, +voddmavs,voddmavs,salt_vertical_eddy_diffusivity,Salt vertical eddy diffusivity,ocean,,,,,, +vozoeivu,voeivu,sea_water_x_EIV_current,Zonal eiv current,ocean,,,,,, +vomeeivv,voeivv,sea_water_y_EIV_current,Meridional eiv current,ocean,,,,,, +voveeivw,voeivz,sea_water_z_EIV_current,Vertical eiv current,ocean,,,,,, +scvoltot,volo,sea_water_volume,Sea water volume ,ocean,,,,,, +votkeavm,votkeavm,vertical_eddy_viscosity,Vertical eddy viscosity,ocean,,,,,, +votkeavt,votkeavt,vertical_eddy_diffusivity,Vertical eddy diffusivity,ocean,,,,,, +votkeevd,votkeevd,enhanced_vertical_diffusivity,Enhanced vertical diffusivity,ocean,,,,,, +votkeevm,votkeevm,enhanced_vertical_viscosity,Enhanced vertical viscosity,ocean,,,,,, +sobarstf,vsftbarot,ocean_barotropic_volume_streamfunction,Ocean barotropic volume streamfunction ,ocean,,,,,, +zomsfatl,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Atl,,,,, +zomsfglo,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Glob,,,,, +zomsfind,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Ind,,,,, +zomsfipc:zomsfinp,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,IndPac,,,,, +zomsfpac,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Pac,,,,, +zomsfeiv,vsftmyzba,ocean_meridional_overturning_mass_streamfunction_due_to_bolus_advection,Ocean meridional overturning volume streamfunction due to bolus advection ,ocean,,,,,, +w,wa,vertical_velocity,Vertical velocity,atmos,,,,,, +z,zg,geopotential_height,Geopotential height,atmos,,,,,, +vovecrtz,zo,sea_water_z_velocity,Sea water z velocity,ocean,,,,,, +sossheigh:sossheig:mean_sossheig,zos,sea_surface_height_above_geoid,Sea surface height above geoid ,ocean,,,,,, +scsshtot,zosga,global_average_sea_level_change,Global average sea level change ,ocean,,,,,, +scsshste,zossga,global_average_steric_sea_level_change,Global average steric sea level change ,ocean,,,,,, +zossq,zossq,square_of_sea_surface_height_above_geoid,Square of sea surface height above geoid ,ocean,,,,,, +scsshtst,zostoga,snthic,Global average thermosteric sea level change ,ocean,,,,,, +heatcsum,heatcsum,total_ocean_heat_content,Total Ocean heat content,ocean,,J,,,, +heatcvmean,heatcvmean,average_ocean_heat_content,Average Ocean heat content,ocean,,J m-3,,,, +transix,transix,sea_ice_x_transport,X-Component of Sea Ice Mass Transport,seaIce,,kg s-1,,,, +transiy,transiy,sea_ice_y_transport,Y-Component of Sea Ice Mass Transport,seaIce,,kg s-1,,,, +windsp,sfcWind,wind_speed,Near-Surface Wind Speed,atmos,,,,,, +vsfsit,vsfsit,virtual_salt_flux_into_sea_water_due_to_sea_ice_thermodynamics,Virtual Salt Flux into Sea Water due to Sea Ice Thermodynamics ,ocean,,,,,, +sfdsi,sfdsi,downward_sea_ice_basal_salt_flux,Downward Sea Ice Basal Salt Flux,ocean,,,,,, +hfsithermds,hfsithermds,heat_flux_into_sea_water_due_to_sea_ice_thermodynamics,Heat Flux into Sea Water due to Sea Ice Thermodynamics ,ocean,,,,,, +u2o,uosq,square_of_sea_water_x_velocity,Square of Sea Water X Velocity ,ocean,,,,,, +v2o,vosq,square_of_sea_water_y_velocity,Square of Sea Water Y Velocity ,ocean,,,,,, +vozomatr,umo,ocean_mass_x_transport,Ocean Mass X Transport ,ocean,,,,,, +vomematr,vmo,ocean_mass_y_transport,Ocean Mass Y Transport ,ocean,,,,,, +sozohetr,hfx,ocean_heat_x_transport,Ocean Heat X Transport ,ocean,,,,,, +somehetr,hfy,ocean_heat_y_transport,Ocean Heat Y Transport ,ocean,,,,,, +uto,uothetao,product_of_xward_sea_water_velocity_and_temperature,Product of X-ward Sea Water Velocity and Temperature,ocean,,,,,, +vto,vothetao,product_of_yward_sea_water_velocity_and_temperature,Product of Y-ward Sea Water Velocity and Temperature,ocean,,,,,, +uso,uoso,product_of_xward_sea_water_velocity_and_salinity,Product of X-ward Sea Water Velocity and Salinity,ocean,,,,,, +vso,voso,product_of_yward_sea_water_velocity_and_salinity,Product of Y-ward Sea Water Velocity and Salinity,ocean,,,,,, +wfo,wfo,water_flux_into_sea_water,Water Flux into Sea Water ,ocean,,,,,, +emp_oce,evsmpr,evap_minus_precip_over_sea_water,Evap minus Precip over ocean,ocean,,,,,, +emp_ice,evsmpr,evap_minus_precip_over_sea_ice,Evap minus Precip over ice,seaIce,,,,,, +qsr_oce,rsntds,net_downward_shortwave_flux_at_sea_water_surface,Net Downward Shortwave Radiation at Sea Water Surface ,ocean,,,,,, +qns_oce,rlds,surface_net_downward_longwave_flux,Surface Net Downward Longwave Radiation,ocean,,,,,, +qsr_ice,rsdssi,surface_downwelling_shortwave_flux_in_air,Downwelling Shortwave over Sea Ice,seaIce,,,,,, +qns_ice,rldssi,surface_downwelling_longwave_flux_in_air,Downwelling Long Wave over Sea Ice,seaIce,,,,,, +sfx,sfx,downward_salt_flux,Downward Salt Flux,ocean,,,,,, +taum,taum,surface_downward_stress_module,Surface Downward Stress Module,ocean,,,,,, +zfull,zfull,depth_below_geoid,Depth Below Geoid of Ocean Layer,ocean,,,,,, +zhalf,zhalf,depth_below_geoid,Depth Below Geoid of Ocean Layer,ocean,,,,,, +pbo,pbo,sea_water_pressure_at_sea_floor,Sea Water Pressure at Sea Floor,ocean,,,,,, +thkcello,thkcello,cell_thickness,Cell Thickness,ocean,,,,,, +ficeberg,ficeberg,water_flux_into_sea_water_from_icebergs,Water Flux into Sea Water From Icebergs ,ocean,,,,,, +rsdo,rsds,downwelling_shortwave_flux_in_sea_water,Downwelling Shortwave Radiation in Sea Water ,ocean,,,,,, +wo,wo,sea_water_upward_velocity,Sea Water Upward Velocity ,ocean,,,,,, +w2o,wosq,square_of_sea_water_upward_velocity,Square of Sea Water Upward Velocity ,ocean,,,,,, +difvho,difvho,ocean_vertical_heat_diffusivity,Ocean Vertical Heat Diffusivity,ocean,,,,,, +vovematr,wmo,upward_ocean_mass_transport,Upward Ocean Mass Transport,ocean,,,,,, +qtr_ice,qtr,shortwave_flux_transmitted_through_ice,Shortwave Flux Transmitted Through The Ice,seaIce,,,,,, +poc,poc,small_organic_carbon_concentration,Small organic carbon Concentration,ocnBgchem,,,,,, +nanophy,nanophy,nanopthyoplankton_concentration,(Nano)Phytoplankton Concentration,ocnBgchem,,,,,, +dsi,dsi,diatoms_silicate_concentration,Diatoms Silicate Concentration,ocnBgchem,,,,,, +goc,goc,big_organic_carbon_concentration,Big organic carbon Concentration,ocnBgchem,,,,,, +sfe,sfe,small_iron_particles_concentration,Small iron particles Concentration,ocnBgchem,,,,,, +nfe,nfe,nano_iron_concentration,Nano iron Concentration,ocnBgchem,,,,,, +nchl,nchl,nano_chlorophyl_concentration,Nano chlorophyl Concentration,ocnBgchem,,,,,, +pno3tot,pno3tot,global_mean_nitrate_concentration,Global mean nitrate concentration,ocnBgchem,,,,,, +psiltot,psiltot,global_mean_silicate_concentration,Global mean silicate concentration,ocnBgchem,,,,,, +palktot,palktot,global_mean_alkalinity_concentration,Global mean alkalinity concentration,ocnBgchem,,,,,, +pfertot,pfertot,global_mean_iron_concentration,Global mean iron concentration,ocnBgchem,,,,,, +tcflx,tcflx,total_flux_carbon_out_of_the_ocean,total Flux of Carbon out of the ocean,ocnBgchem,,,,,, +tcflxcum,tcflxcum,cumulative_total_flux_of_carbon_out_of_the_ocean,cumulative total Flux of Carbon out of the ocean,ocnBgchem,,,,,, +c-export,c-export,total_carbon_export_at_100m,total Carbon export at 100m,ocnBgchem,,,,,, +tintpp,tintpp,global_total_integrated_primary_production,global total integrated primary production,ocnBgchem,,,,,, +tnfix,tnfix,global_total_nitrogen_fixation,global total nitrogen fixation,ocnBgchem,,,,,, +tdenit,tdenit,total_denitrification,Total denitrification,ocnBgchem,,,,,, +intppnew,intppnew,new_primary_production,Vertically integrated new primary production,ocnBgchem,,,,,, +inttpp,inttpp,total_primary_production_of_phyto,Total Primary production of phyto,ocnBgchem,,,,,, +inttppnew,inttppnew,new_primary_production_of_phyto,New Primary production of phyto,ocnBgchem,,,,,, +intppphy,intppphy,vertically_integrated_primary_production_by_nanophy,Vertically integrated primary production by nanophy,ocnBgchem,,,,,, +ppphy,ppphy,primary_production_of_nanooplakton,Primary production of nanooplakton,ocnBgchem,,,,,, +intpbcal,intpbcal,vertically_integrated_of_calcite_productdic_fluxion,Vertically integrated of calcite productDIC fluxion,ocnBgchem,,,,,, +cflx,cflx,dic_flux,DIC flux,ocnBgchem,,,,,, +remin,remin,oxic_remineralization_of_om,Oxic remineralization of OM,ocnBgchem,,,,,, +denit,denit,anoxic_remineralization_of_om,Anoxic remineralization of OM,ocnBgchem,,,,,, +nfix,nfix,nitrogen_fixation,Nitrogen fixation,ocnBgchem,,,,,, +sdenit,sdenit,nitrate_reduction_in_the_sediments,Nitrate reduction in the sediments,ocnBgchem,,,,,, +par,par,photosynthetically_available_radiation,photosynthetically Available Radiation,ocnBgchem,,,,,, +lnnut,lnnut,nutrient_limitation_term_in_nanophyto,Nutrient limitation term in Nanophyto,ocnBgchem,,,,,, +ldnut,ldnut,nutrient_limitation_term_in_diatoms,Nutrient limitation term in Diatoms,ocnBgchem,,,,,, +lnfe,lnfe,iron_limitation_term_in_nanophyoto,Iron limitation term in Nanophyoto,ocnBgchem,,,,,, +lnlight,lnlight,light_limitation_term_in_nanophyto,Light limitation term in Nanophyto,ocnBgchem,,,,,, +ldlight,ldlight,light_limitation_term_in_diatoms,Light limitation term in Diatoms,ocnBgchem,,,,,, +graz1,graz1,grazing_by_microzooplankton,Grazing by microzooplankton,ocnBgchem,,,,,, +graz2,graz2,grazing_by_mesozooplankto_,Grazing by mesozooplankton,ocnBgchem,,,,,, +mumax,mumax,maximum_growth_rate,Maximum growth rate,ocnBgchem,,,,,, +mun,mun,realized_growth_rate_for_nanophyto,Realized growth rate for nanophyto,ocnBgchem,,,,,, +mud,mud,realized_growth_rate_for_diatomes,Realized growth rate for diatomes,ocnBgchem,,,,,, +ppnewn,ppnewn,new_primary_production_of_nanophyto,New Primary production of nanophyto,ocnBgchem,,,,,, +ppnewd,ppnewd,new_primary_production_of_diatoms,New Primary production of diatoms,ocnBgchem,,,,,, +dic,dic,disolved_inorganic_carbon,Disolved Inorganic Carbon,ocnBgchem,,,,,, +zqla,hflso,surface_downward_latent_heat_flux,Surface Downward Latent Heat Flux,ocean,,W m-2,,,, +zqsb,hfsso,surface_downward_sensible_heat_flux,Surface Downward Sensible Heat Flux,ocean,,W m-2,,,, +zqlw,rlntds,surface_net_downward_longwave_flux,Surface Net Downward Longwave Radiation,ocean,,W m-2,,,, +var78,tclw,total_column_liquid_water,Total column liquid water,atmos,,kg m-2,,,, +var79,tciw,total_column_ice_water,Total column ice water,atmos,,kg m-2,,,, +rho,rhopoto,sea_water_potential_density,Sea Water Potential Density,ocean,,kg m-3,,,, \ No newline at end of file diff --git a/earthdiagnostics/cmor_tables/preface.csv b/earthdiagnostics/cmor_tables/preface.csv new file mode 100644 index 0000000000000000000000000000000000000000..90f01d9150608f127fd1de1aa6b2b5e0f14c6f84 --- /dev/null +++ b/earthdiagnostics/cmor_tables/preface.csv @@ -0,0 +1 @@ +Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables diff --git a/earthdiagnostics/cmor_tables/primavera b/earthdiagnostics/cmor_tables/primavera new file mode 160000 index 0000000000000000000000000000000000000000..f25073770569ea73540d09a058637128db024c55 --- /dev/null +++ b/earthdiagnostics/cmor_tables/primavera @@ -0,0 +1 @@ +Subproject commit f25073770569ea73540d09a058637128db024c55 diff --git a/earthdiagnostics/cmor_tables/specs.csv b/earthdiagnostics/cmor_tables/specs.csv new file mode 100644 index 0000000000000000000000000000000000000000..90f01d9150608f127fd1de1aa6b2b5e0f14c6f84 --- /dev/null +++ b/earthdiagnostics/cmor_tables/specs.csv @@ -0,0 +1 @@ +Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py new file mode 100644 index 0000000000000000000000000000000000000000..7cf8c5c354410fc4024f7d73ea2f605f1ecb00f9 --- /dev/null +++ b/earthdiagnostics/cmorizer.py @@ -0,0 +1,675 @@ +# coding=utf-8 +import glob +import os +import pygrib +import shutil +import uuid +from datetime import datetime + +from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months +from bscearth.utils.log import Log + +from earthdiagnostics.datafile import NetCDFFile +from earthdiagnostics.frequency import Frequency, Frequencies +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import TempFile, Utils + + +class Cmorizer(object): + """ + Class to manage CMORization + + :param data_manager: experiment's data manager + :type data_manager: CMORManager + :param startdate: startdate to cmorize + :type startdate: str + :param member: member to cmorize + :type member: int + + """ + + NON_DATA_VARIABLES = ('lon', 'lat', 'time', 'time_bnds', 'leadtime', 'lev', 'lev_2', 'icethi', + 'deptht', 'depthu', 'depthw', 'depthv', 'time_centered', 'time_centered_bounds', + 'deptht_bounds', 'depthu_bounds', 'depthv_bounds', 'depthw_bounds', + 'deptht_bnds', 'depthu_bnds', 'depthv_bnds', 'depthw_bnds', + 'time_counter_bounds', 'ncatice', 'nav_lat_grid_V', 'nav_lat_grid_U', + 'nav_lat_grid_T', 'nav_lon_grid_V', 'nav_lon_grid_U', 'nav_lon_grid_T', + 'depth', 'depth_2', 'depth_3', 'depth_4', + 'depth_bnds', 'depth_2_bnds', 'depth_3_bnds', 'depth_4_bnds', + 'mlev', 'hyai', 'hybi', 'hyam', 'hybm') + + ALT_COORD_NAMES = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds', 'time_counter_bounds': 'time_bnds', + 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} + + def __init__(self, data_manager, startdate, member): + self.data_manager = data_manager + self.startdate = startdate + self.member = member + self.config = data_manager.config + self.experiment = self.config.experiment + self.cmor = self.config.cmor + self.member_str = self.experiment.get_member_str(member) + self.original_files_path = os.path.join(self.config.data_dir, self.experiment.expid, 'original_files', + self.startdate, self.member_str, 'outputs') + self.atmos_timestep = None + self.cmor_scratch = str(os.path.join(self.config.scratch_dir, 'CMOR', self.startdate, self.member_str)) + + def cmorize_ocean(self): + """ + CMORizes ocean files from MMO files + :return: + """ + if not self.cmor.ocean: + Log.info('Skipping ocean cmorization due to configuration') + return + Log.info('\nCMORizing ocean\n') + self._cmorize_ocean_files('MMO', 'PPO', 'diags') + + def _cmorize_ocean_files(self, *args): + tar_files = () + for prefix in args: + tar_folder = os.path.join(self.original_files_path, '{0}*'.format(prefix)) + tar_files = glob.glob(tar_folder) + tar_files.sort() + if len(tar_files) > 0: + break + + if not len(tar_files): + Log.error('No {1} files found in {0}'.format(self.original_files_path, args)) + + count = 1 + for tarfile in tar_files: + if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), (ModelingRealms.ocean, + ModelingRealms.seaIce, + ModelingRealms.ocnBgchem)): + Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) + count += 1 + continue + + Log.info('Unpacking oceanic file {0}/{1}'.format(count, len(tar_files))) + try: + self._unpack_tar_file(tarfile) + self._cmorize_nc_files() + Log.result('Oceanic file {0}/{1} finished'.format(count, len(tar_files))) + except Exception as ex: + Log.error('Could not CMORize oceanic file {0}: {1}', count, ex) + count += 1 + + def _filter_files(self, file_list): + if not self.cmor.filter_files: + return file_list + filtered = list() + filters = self.cmor.filter_files.split(' ') + for file_path in file_list: + filename = os.path.basename(file_path) + if any(f in filename for f in filters): + filtered.append(file_path) + else: + os.remove(file_path) + if len(filtered) == 0: + Log.warning('Filters {0} do not match any of the files', filters) + return filtered + + def _cmorize_nc_files(self): + nc_files = glob.glob(os.path.join(self.cmor_scratch, '*.nc')) + for filename in nc_files: + self._cmorize_nc_file(filename) + self._clean_cmor_scratch() + + def _correct_fluxes(self): + fluxes_vars = [self.data_manager.variable_list.get_variable(cmor_var, True).short_name + for cmor_var in ('prc', "prsn", "rss", "rls", "rsscs", "rsds", "rlds", "hfss", 'hfls')] + change_sign_vars = [self.data_manager.variable_list.get_variable(cmor_var, True).short_name + for cmor_var in ("hfss", 'hfls')] + total_seconds = (self.experiment.atmos_timestep * 3600) + for filename in glob.glob(os.path.join(self.cmor_scratch, '*.nc')): + handler = Utils.openCdf(filename) + for varname in handler.variables.keys(): + cmor_var = self.data_manager.variable_list.get_variable(varname, True) + + if cmor_var is None or cmor_var.short_name not in fluxes_vars: + continue + + if cmor_var.short_name in change_sign_vars: + sign = -1 + else: + sign = 1 + + var_handler = handler.variables[varname] + var_handler[:] = sign * var_handler[:] / total_seconds + var_handler.units = '{0} {1}'.format(var_handler.units, 's-1') + handler.close() + + def _unpack_tar_file(self, tarfile): + self._clean_cmor_scratch() + os.makedirs(self.cmor_scratch) + Utils.untar((tarfile,), self.cmor_scratch) + zip_files = glob.glob(os.path.join(self.cmor_scratch, '*.gz')) + for zip_file in self._filter_files(zip_files): + try: + Utils.unzip(zip_file) + except Utils.UnzipException as ex: + Log.error('File {0} could not be unzipped: {1}', tarfile, ex) + + def _clean_cmor_scratch(self): + if os.path.exists(self.cmor_scratch): + shutil.rmtree(self.cmor_scratch) + + def _merge_mma_files(self, tarfile): + temp = TempFile.get() + sh_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_SH_*.nc')) + gg_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_GG_*.nc')) + + merged_sh = TempFile.get() + merged_gg = TempFile.get() + + for filename in sh_files: + Utils.cdo.sp2gpl(options='-O', input=filename, output=temp) + shutil.move(temp, filename) + Utils.cdo.mergetime(input=sh_files, output=merged_sh) + Utils.cdo.mergetime(input=gg_files, output=merged_gg) + for filename in sh_files + gg_files: + os.remove(filename) + tar_startdate = tarfile[0:-4].split('_')[5].split('-') + shutil.move(merged_gg, os.path.join(self.cmor_scratch, 'MMAGG_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) + shutil.move(merged_sh, os.path.join(self.cmor_scratch, 'MMASH_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) + + def cmorize_atmos(self): + """ + CMORizes atmospheric data, from grib or MMA files + :return: + """ + if not self.cmor.atmosphere: + Log.info('Skipping atmosphere cmorization due to configuration') + return + + Log.info('\nCMORizing atmosphere\n') + if self.cmor.use_grib and self.gribfiles_available(): + self._cmorize_grib_files() + else: + self._cmorize_mma_files() + + def _cmorize_mma_files(self): + tar_files = glob.glob(os.path.join(self.original_files_path, 'MMA*')) + tar_files.sort() + count = 1 + if len(tar_files) == 0: + Log.error('MMA files not found in {0}'.format(self.original_files_path)) + for tarfile in tar_files: + if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), (ModelingRealms.atmos,)): + Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) + count += 1 + continue + Log.info('Unpacking atmospheric file {0}/{1}'.format(count, len(tar_files))) + try: + self._unpack_tar_file(tarfile) + self._merge_mma_files(tarfile) + self._correct_fluxes() + self._cmorize_nc_files() + Log.result('Atmospheric file {0}/{1} finished'.format(count, len(tar_files))) + except Exception as ex: + Log.error('Could not cmorize atmospheric file {0}: {1}', count, ex) + + count += 1 + + def _cmorize_grib_files(self): + chunk = 1 + chunk_start = parse_date(self.startdate) + + while os.path.exists(self.get_original_grib_path(chunk_start, 'GG')) or \ + os.path.exists(self.get_original_grib_path(chunk_start, 'SH')): + + if self.cmorization_required(chunk, (ModelingRealms.atmos,)): + chunk_end = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', self.experiment.calendar) + chunk_end = previous_day(chunk_end, self.experiment.calendar) + Log.info('CMORizing chunk {0}-{1}', date2str(chunk_start), date2str(chunk_end)) + try: + for grid in ('SH', 'GG'): + Log.info('Processing {0} variables', grid) + + if not os.path.exists(self.get_original_grib_path(chunk_start, grid)): + continue + self.cmorize_grib_file(chunk_end, chunk_start, grid) + except Exception as ex: + Log.error('Can not cmorize GRIB file for chunk {0}-{1}: {2}', + date2str(chunk_start), date2str(chunk_end), ex) + chunk_start = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', self.experiment.calendar) + chunk += 1 + + def cmorize_grib_file(self, chunk_end, chunk_start, grid): + for month in range(0, self.experiment.chunk_size): + current_date = add_months(chunk_start, month, self.experiment.calendar) + original_gribfile = self.get_original_grib_path(current_date, grid) + Log.info('Processing month {1}', grid, date2str(current_date)) + gribfile = self.get_scratch_grib_path(current_date, grid) + if not os.path.isfile(gribfile): + Log.info('Copying file...', grid, date2str(current_date)) + Utils.copy_file(original_gribfile, gribfile) + + self._obtain_atmos_timestep(gribfile) + full_file = self._get_monthly_grib(current_date, gribfile, grid) + self._unpack_grib(full_file, gribfile, grid) + + next_gribfile = self.get_original_grib_path(add_months(current_date, 1, self.experiment.calendar), grid) + + if not os.path.exists(next_gribfile): + os.remove(gribfile) + + cdo_reftime = parse_date(self.startdate).strftime('%Y-%m-%d,00:00') + + self._ungrib_vars(cdo_reftime, gribfile, current_date.month, Frequency('{0}hr'.format(self.atmos_timestep))) + self._ungrib_vars(cdo_reftime, gribfile, current_date.month, Frequencies.daily) + self._ungrib_vars(cdo_reftime, gribfile, current_date.month, Frequencies.monthly) + + for splited_file in glob.glob('{0}_*.128.nc'.format(gribfile)): + os.remove(splited_file) + + Log.result('Month {0}, {1} variables finished', date2str(current_date), grid) + + self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, Frequencies.monthly) + self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, Frequencies.daily) + self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, + '{0}hr'.format(self.atmos_timestep)) + + @staticmethod + def _unpack_grib(full_file, gribfile, grid): + Log.info('Unpacking... ') + # remap on regular Gauss grid + if grid == 'SH': + Utils.cdo.splitparam(input='-sp2gpl {0}'.format(full_file), output=gribfile + '_', options='-f nc4') + else: + Utils.cdo.splitparam(input=full_file, output=gribfile + '_', options='-R -f nc4') + # total precipitation (remove negative values) + Utils.cdo.setcode(228, + input='-setmisstoc,0 -setvrange,0,Inf -add {0}_{{142,143}}.128.nc'.format(gribfile), + output='{0}_228.128.nc'.format(gribfile)) + Utils.remove_file('ICM') + + def _get_monthly_grib(self, current_date, gribfile, grid): + prev_gribfile = self.get_scratch_grib_path(add_months(current_date, -1, self.experiment.calendar), grid) + if os.path.exists(prev_gribfile): + self._merge_grib_files(current_date, prev_gribfile, gribfile) + full_file = 'ICM' + else: + full_file = gribfile + return full_file + + def get_scratch_grib_path(self, current_date, grid): + return os.path.join(self.config.scratch_dir, self._get_grib_filename(grid, current_date)) + + def _obtain_atmos_timestep(self, gribfile): + if self.atmos_timestep is None: + self.atmos_timestep = self._get_atmos_timestep(gribfile) + + def get_original_grib_path(self, current_date, grid): + return os.path.join(self.original_files_path, + self._get_grib_filename(grid, current_date)) + + def _get_grib_filename(self, grid, month): + return 'ICM{0}{1}+{2}.grb'.format(grid, self.experiment.expid, date2str(month)[:-2]) + + def _get_atmos_timestep(self, gribfile): + Log.info('Getting timestep...') + grib_handler = pygrib.open(gribfile) + dates = set() + try: + while True: + mes = grib_handler.next() + dates.add(mes.analDate) + except StopIteration: + pass + dates = list(dates) + dates.sort() + atmos_timestep = dates[1] - dates[0] + atmos_timestep = int(atmos_timestep.total_seconds() / 3600) + self.experiment.atmos_timestep = atmos_timestep + grib_handler.close() + return atmos_timestep + + def _cmorize_nc_file(self, filename): + Log.info('Processing file {0}', filename) + + if not self._contains_requested_variables(filename): + os.remove(filename) + return + + Utils.convert2netcdf4(filename) + frequency = self._get_nc_file_frequency(filename) + Utils.rename_variables(filename, Cmorizer.ALT_COORD_NAMES, False, True) + self._remove_valid_limits(filename) + self._add_common_attributes(filename, frequency) + self._update_time_variables(filename) + + handler = Utils.openCdf(filename) + Log.info('Splitting file {0}', filename) + for variable in handler.variables.keys(): + if variable in Cmorizer.NON_DATA_VARIABLES: + continue + try: + self.extract_variable(filename, handler, frequency, variable) + except Exception as ex: + Log.error('Variable {0} can not be cmorized: {1}', variable, ex) + Log.result('File {0} cmorized!', filename) + handler.close() + os.remove(filename) + + # noinspection PyMethodMayBeStatic + def _remove_valid_limits(self, filename): + handler = Utils.openCdf(filename) + for variable in handler.variables.keys(): + var = handler.variables[variable] + if 'valid_min' in var.ncattrs(): + del var.valid_min + if 'valid_max' in var.ncattrs(): + del var.valid_max + handler.close() + + def _get_nc_file_frequency(self, filename): + file_parts = os.path.basename(filename).split('_') + if self.experiment.expid in [file_parts[1], file_parts[2]]: + frequency = Frequency('m') + elif self.experiment.expid == file_parts[0]: + try: + parse_date(file_parts[1]) + frequency = Frequency('m') + except ValueError: + frequency = Frequency(file_parts[1]) + else: + frequency = Frequency(file_parts[1]) + return frequency + + def _contains_requested_variables(self, filename): + variables = Utils.get_file_variables(filename) + return self.cmor.any_required(variables) + + def extract_variable(self, file_path, handler, frequency, variable): + """ + Extracts a variable from a file and creates the CMOR file + + :param file_path: path to the file + :type file_path: str + :param handler: netCDF4 handler for the file + :type handler: netCDF4.Dataset + :param frequency: variable's frequency + :type frequency: Frequency + :param variable: variable's name + :type variable: str + """ + alias, var_cmor = self.config.var_manager.get_variable_and_alias(variable) + if var_cmor is None: + return + + if not self.cmor.cmorize(var_cmor): + return + + temp = TempFile.get() + Utils.nco.ncks(input=file_path, output=temp, options=('-v {0}'.format(variable),)) + self._rename_level_variables(temp, var_cmor) + + self._add_coordinate_variables(handler, temp) + + if alias.basin is None: + region = None + else: + region = alias.basin.name + + date_str = self.get_date_str(file_path) + if date_str is None: + Log.error('Variable {0} can not be cmorized. Original filename does not match a recognized pattern', + var_cmor.short_name) + raise CMORException('Variable {0}:{1} can not be cmorized. Original filename does not match a recognized ' + 'pattern'.format(var_cmor.domain, var_cmor.short_name)) + + netcdf_file = NetCDFFile() + netcdf_file.data_manager = self.data_manager + netcdf_file.local_file = temp + netcdf_file.remote_file = self.data_manager.get_file_path(self.startdate, self.member, + var_cmor.domain, var_cmor.short_name, var_cmor, + None, frequency, + grid=alias.grid, year=None, date_str=date_str) + + netcdf_file.data_convention = self.config.data_convention + netcdf_file.region = region + + netcdf_file.frequency = frequency + netcdf_file.domain = var_cmor.domain + netcdf_file.var = var_cmor.short_name + netcdf_file.final_name = var_cmor.short_name + + netcdf_file.prepare_to_upload(rename_var=variable) + netcdf_file.add_cmorization_history() + netcdf_file.upload() + + if region: + region_str = ' (Region {})'.format(region) + else: + region_str = '' + Log.info('Variable {0.domain}:{0.short_name} processed{1}', var_cmor, region_str) + + def get_date_str(self, file_path): + file_parts = os.path.basename(file_path).split('_') + if file_parts[0] in (self.experiment.expid, 't00o', 'MMA', 'MMASH', 'MMAGG', 'MMO') or file_parts[0].startswith('ORCA'): + # Model output + if file_parts[-1].endswith('.tar'): + file_parts = file_parts[-1][0:-4].split('-') + return '{0}-{1}'.format(file_parts[0][0:6], file_parts[1][0:6]) + else: + return '{0}-{1}'.format(file_parts[2][0:6], file_parts[3][0:6]) + elif file_parts[1] == self.experiment.expid: + # Files generated by the old version of the diagnostics + return '{0}-{1}'.format(file_parts[4][0:6], file_parts[5][0:6]) + else: + return None + + def get_chunk(self, file_path): + chunk_start = parse_date(self.get_date_str(file_path).split('-')[0]) + current_date = parse_date(self.startdate) + chunk = 1 + while current_date < chunk_start: + current_date = chunk_end_date(current_date, self.experiment.chunk_size, 'month', self.experiment.calendar) + chunk += 1 + + if current_date != chunk_start: + raise Exception('File {0} start date is not a valid chunk start date'.format(file_path)) + return chunk + + @staticmethod + def _add_coordinate_variables(handler, temp): + handler_cmor = Utils.openCdf(temp) + Utils.copy_variable(handler, handler_cmor, 'lon', False) + Utils.copy_variable(handler, handler_cmor, 'lat', False) + if 'time' in handler_cmor.dimensions.keys(): + Utils.copy_variable(handler, handler_cmor, 'leadtime', False) + handler_cmor.close() + + @staticmethod + def _rename_level_variables(temp, var_cmor): + if var_cmor.domain == ModelingRealms.ocean: + Utils.rename_variables(temp, {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', + 'depth': 'lev'}, False, True) + if var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: + Utils.rename_variables(temp, {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', + 'depth_4': 'sdepth'}, False, True) + if var_cmor.domain == ModelingRealms.atmos: + Utils.rename_variables(temp, {'depth': 'plev'}, False, True) + + @staticmethod + def _merge_grib_files(current_month, prev_gribfile, gribfile): + Log.info('Merging data from different files...') + fd = open('rules_files', 'w') + fd.write('if (dataDate >= {0.year}{0.month:02}01) {{ write ; }}\n'.format(current_month)) + fd.close() + # get first timestep for each month from previous file (if possible) + if os.path.exists('ICM'): + os.remove('ICM') + Utils.execute_shell_command('grib_filter -o ICM rules_files ' + '{0} {1}'.format(os.path.basename(prev_gribfile), + os.path.basename(gribfile))) + os.remove('rules_files') + Utils.remove_file(prev_gribfile) + + def _ungrib_vars(self, cdo_reftime, gribfile, month, frequency): + Log.info('Preparing {0} variables'.format(frequency)) + var_codes = self.config.cmor.get_variables(frequency) + for var_code in var_codes: + if not os.path.exists('{0}_{1}.128.nc'.format(gribfile, var_code)): + continue + new_units = None + + cdo_operator = '-selmon,{0}'.format(month) + if frequency == Frequencies.monthly: + if var_code == 201: + cdo_operator = "-monmean -daymax {0}".format(cdo_operator) + elif var_code == 202: + cdo_operator = "-monmean -daymax {0}".format(cdo_operator) + else: + cdo_operator = "-monmean {0} ".format(cdo_operator) + if frequency == Frequencies.daily: + if var_code == 201: + cdo_operator = "-daymax {0} ".format(cdo_operator) + elif var_code == 202: + cdo_operator = "-daymin {0} ".format(cdo_operator) + else: + cdo_operator = "-daymean {0} ".format(cdo_operator) + + if var_code in (144, 146, 147, 169, 175, 176, 177, 179, 180, 181, 182, 201, 202, 205, 212, 228): + cdo_operator = '{0} -shifttime,-{1}hours'.format(cdo_operator, self.experiment.atmos_timestep) + + if var_code == 129: + # geopotential + new_units = "m" + cdo_operator = "-divc,9.81 {0}".format(cdo_operator) + elif var_code in (146, 147, 169, 175, 176, 177, 179, 212): + # radiation + new_units = "W m-2" + cdo_operator = "-divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) + elif var_code in (180, 181): + # momentum flux + new_units = "N m-2" + cdo_operator = "-divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) + elif var_code in (144, 182, 205, 228): + # precipitation/evaporation/runoff + new_units = "kg m-2 s-1" + cdo_operator = "-mulc,1000 -divc,{0}".format(self.experiment.atmos_timestep * 3600) + + levels = self.config.cmor.get_levels(frequency, var_code) + if levels: + cdo_operator = "{0} -sellevel,{1}".format(cdo_operator, levels) + + Utils.execute_shell_command('cdo -t ecmwf setreftime,{0} ' + '{1} {2}_{3}.128.nc ' + '{2}_{3}_{4}.nc'.format(cdo_reftime, cdo_operator, + gribfile, var_code, frequency)) + h_var_file = '{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency) + + handler = Utils.openCdf(h_var_file) + if new_units: + for var in handler.variables.values(): + if 'code' in var.ncattrs() and var.code == var_code: + var.units = new_units + break + + var_name = None + for key in handler.variables.keys(): + if key + '_2' in handler.variables and key not in handler.dimensions: + var_name = key + handler.close() + + if var_name is not None: + Utils.nco.ncks(input='{0}_{1}_1m.nc'.format(gribfile, var_code), + output='{0}_{1}_1m.nc'.format(gribfile, var_code), + options=('-O -v {0}'.format(var_name))) + + def _merge_and_cmorize_atmos(self, chunk_start, chunk_end, grid, frequency): + merged_file = 'MMA_{0}_{1}_{2}_{3}.nc'.format(frequency, date2str(chunk_start), date2str(chunk_end), grid) + files = glob.glob(os.path.join(self.config.scratch_dir, + '{0}_*_{1}.nc'.format(self._get_grib_filename(grid, chunk_start), frequency))) + for first_file in files: + shutil.move(first_file, merged_file) + current_month = add_months(chunk_start, 1, self.experiment.calendar) + while current_month < chunk_end: + month_file = first_file.replace('+{0}.grb'.format(date2str(chunk_start)[:-2]), + '+{0}.grb'.format(date2str(current_month)[:-2])) + Utils.concat_variables(month_file, merged_file, True) + current_month = add_months(current_month, 1, self.experiment.calendar) + + self._cmorize_nc_file(merged_file) + + def _update_time_variables(self, filename): + handler = Utils.openCdf(filename) + time_var = handler.variables['time'] + if "time_bnds" in handler.variables: + time_var.bounds = "time_bnds" + handler.variables['time_bnds'].units = time_var.units + handler.close() + temp = TempFile.get() + Utils.cdo.setreftime('1850-01-01,00:00:00,days', input=filename, output=temp) + Utils.move_file(temp, filename) + + self._set_leadtime_var(filename) + + def _set_leadtime_var(self, filename): + handler = Utils.openCdf(filename) + if 'leadtime' in handler.variables: + var = handler.variables['leadtime'] + else: + var = handler.createVariable('leadtime', float, 'time') + var.units = "days" + var.long_name = "Time elapsed since the start of the forecast" + var.standard_name = "forecast_period" + leadtime = Utils.get_datetime_from_netcdf(handler) + startdate = parse_date(self.startdate) + leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate + for time in leadtime] + for lt in range(0, len(leadtime)): + var[lt] = leadtime[lt].days + handler.close() + + def _add_common_attributes(self, filename, frequency): + cmor = self.config.cmor + experiment = self.config.experiment + handler = Utils.openCdf(filename) + handler.associated_experiment = cmor.associated_experiment + handler.batch = '{0}{1}'.format(experiment.institute, datetime.now().strftime('%Y-%m-%d(T%H:%M:%SZ)')) + handler.contact = 'Pierre-Antoine Bretonniere, pierre-antoine.bretonniere@bsc.es , ' \ + 'Javier Vegas-Regidor, javier.vegas@bsc.es ' + handler.Conventions = 'CF-1.6' + handler.creation_date = datetime.now().strftime('%Y-%m-%d(T%H:%M:%SZ)') + handler.experiment_id = experiment.experiment_name + handler.forecast_reference_time = parse_date(self.startdate).strftime('%Y-%m-%d(T%H:%M:%SZ)') + handler.frequency = frequency.frequency + handler.institute_id = experiment.institute + handler.institution = experiment.institute + handler.initialization_method = cmor.initialization_method + handler.initialization_description = cmor.initialization_description + handler.physics_version = cmor.physics_version + handler.physics_description = cmor.physics_description + handler.model_id = experiment.model + handler.associated_model = cmor.associated_model + handler.project_id = self.config.data_convention.upper() + handler.realization = str(self.member + 1) + handler.source = cmor.source + handler.startdate = 'S{0}'.format(self.startdate) + handler.tracking_id = str(uuid.uuid1()) + handler.title = "{0} model output prepared for {2} {1}".format(experiment.model, experiment.experiment_name, + self.config.data_convention.upper()) + handler.close() + + def gribfiles_available(self): + grb_path = os.path.join(self.original_files_path, '*.grb') + gribfiles = glob.glob(grb_path) + return len(gribfiles) > 0 + + def cmorization_required(self, chunk, domains): + if not self.config.cmor.chunk_cmorization_requested(chunk): + return False + if self.config.cmor.force: + return True + for domain in domains: + if self.data_manager.is_cmorized(self.startdate, self.member, chunk, domain): + return False + return True + + +class CMORException(Exception): + pass diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py new file mode 100644 index 0000000000000000000000000000000000000000..58ce7e2307b2ff418d670df3c1c0bf721e1be1d6 --- /dev/null +++ b/earthdiagnostics/cmormanager.py @@ -0,0 +1,664 @@ +# coding=utf-8 +import glob +import os +from datetime import datetime + +from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, previous_day +from bscearth.utils.log import Log + +from datafile import StorageStatus +from diagnostic import Diagnostic +from earthdiagnostics.cmorizer import Cmorizer +from earthdiagnostics.datamanager import DataManager +from earthdiagnostics.frequency import Frequencies, Frequency +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import TempFile, Utils +from earthdiagnostics.variable_type import VariableType + + +class CMORManager(DataManager): + """ + Data manager class for CMORized experiments + """ + def __init__(self, config): + super(CMORManager, self).__init__(config) + self._dic_cmorized = dict() + data_folders = self.config.data_dir.split(':') + experiment_folder = self.experiment.model.lower() + if experiment_folder.startswith('ec-earth'): + experiment_folder = 'ecearth' + + self.config.data_dir = None + for data_folder in data_folders: + if os.path.isdir(os.path.join(data_folder, self.experiment.expid)): + self.config.data_dir = data_folder + break + test_folder = os.path.join(data_folder, self.experiment.model.lower().replace('-', '')) + if os.path.isdir(os.path.join(test_folder, self.experiment.expid)): + self.config.data_dir = test_folder + break + + test_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) + if os.path.isdir(os.path.join(test_folder, self.experiment.expid)): + self.config.data_dir = test_folder + break + + if not self.config.data_dir: + raise Exception('Can not find model data') + self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') + + # noinspection PyUnusedLocal + def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN, possible_versions=None): + cmor_var = self.variable_list.get_variable(var) + filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) + + # noinspection PyBroadException + if possible_versions is None: + # noinspection PyBroadException + try: + return os.path.isfile(filepath) + except Exception: + return False + else: + for version in possible_versions: + # noinspection PyBroadException + try: + if os.path.isfile(filepath.replace(self.config.cmor.version, version)): + return True + except Exception: + pass + return False + + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param vartype: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :rtype: str + """ + cmor_var = self.variable_list.get_variable(var) + var = self._get_final_var_name(box, var) + filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) + + return self._get_file_from_storage(filepath) + + def request_year(self, diagnostic, domain, var, startdate, member, year, grid=None, box=None, frequency=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param year: + :param diagnostic: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :rtype: str + """ + + job = MergeYear(self, domain, var, startdate, member, year, grid, box, frequency) + job.request_data() + job.declare_data_generated() + if not job.year_file.job_added: + diagnostic.subjobs.append(job) + job.year_file.job_added = True + return job.year_file + + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param diagnostic: + :param region: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + if not frequency: + frequency = self.config.frequency + original_name = var + cmor_var = self.variable_list.get_variable(var) + if cmor_var: + var = cmor_var.short_name + final_name = self._get_final_var_name(box, var) + + filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, chunk, frequency, grid) + netcdf_file = self._declare_generated_file(filepath, domain, final_name, cmor_var, self.config.data_convention, + region, diagnostic, grid, vartype, original_name) + netcdf_file.frequency = frequency + return netcdf_file + + def declare_year(self, domain, var, startdate, member, year, grid=None, box=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param diagnostic: + :param year: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + original_name = var + cmor_var = self.variable_list.get_variable(var) + if cmor_var: + var = cmor_var.short_name + final_name = self._get_final_var_name(box, var) + + filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, None, Frequencies.yearly, grid, + year=year) + netcdf_file = self._declare_generated_file(filepath, domain, final_name, cmor_var, self.config.data_convention, + None, diagnostic, grid, vartype, original_name) + netcdf_file.frequency = Frequencies.yearly + return netcdf_file + + def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency, + grid=None, year=None, date_str=None): + """ + Returns the path to a concrete file + :param cmor_var: + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param domain: file's domain + :type domain: Domain + :param var: file's var + :type var: var + :param chunk: file's chunk + :type chunk: int|NoneType + :param frequency: file's frequency + :type frequency: Frequency + :param grid: file's grid + :type grid: str|NoneType + :param year: file's year + :type year: int|str|NoneType + :param date_str: date string to add directly. Overrides year or chunk configurations + :type date_str: str|NoneType + :return: path to the file + :rtype: str|NoneType + :param cmor_var: variable instance describing the selected variable + :type cmor_var: Variable + """ + if not frequency: + frequency = self.config.frequency + + folder_path = self._get_full_cmor_folder_path(startdate, member, domain, var, frequency, grid, cmor_var) + file_name = self._get_cmor_file_name(startdate, member, domain, var, cmor_var, frequency, + chunk, year, date_str, grid) + + filepath = os.path.join(folder_path, file_name) + return filepath + + def _get_cmor_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + if cmor_var is None: + cmor_table = domain.get_table(frequency, self.config.data_convention) + else: + cmor_table = cmor_var.get_table(frequency, self.config.data_convention) + + if chunk is not None: + time_bound = self._get_chunk_time_bounds(startdate, chunk) + elif year: + if frequency != Frequencies.yearly: + raise ValueError('Year may be provided instead of chunk only if frequency is "yr"') + time_bound = str(year) + elif date_str: + time_bound = date_str + else: + raise ValueError('Chunk, year and date_str can not be None at the same time') + + if time_bound: + time_bound = '_{0}.nc'.format(time_bound) + else: + time_bound = '.nc' + + if self.config.data_convention in ('specs', 'preface'): + + file_name = '{0}_{1}_{2}_{3}_S{4}_{5}{6}'.format(var, cmor_table.name, self.experiment.model, + self.experiment.experiment_name, startdate, + self._get_member_str(member), time_bound) + elif self.config.data_convention in ('primavera', 'cmip6'): + if not grid: + if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: + grid = self.config.cmor.default_ocean_grid + else: + grid = self.config.cmor.default_atmos_grid + file_name = '{0}_{1}_{2}_{3}_{4}_{5}{6}'.format(var, cmor_table.name, self.experiment.experiment_name, + self.experiment.model, self._get_member_str(member), + grid, time_bound) + else: + raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) + return file_name + + def _get_full_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): + if self.config.data_convention in ('specs', 'preface'): + folder_path = os.path.join(self._get_startdate_path(startdate), str(frequency), domain.name, var) + if grid: + folder_path = os.path.join(folder_path, grid) + folder_path = os.path.join(folder_path, self._get_member_str(member)) + if self.config.cmor.version: + folder_path = os.path.join(folder_path, self.config.cmor.version) + else: + if not grid: + if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: + grid = self.config.cmor.default_ocean_grid + else: + grid = self.config.cmor.default_atmos_grid + folder_path = os.path.join(self._get_startdate_path(startdate), self._get_member_str(member), + cmor_var.get_table(frequency, self.config.data_convention).name, var, + grid, self.config.cmor.version) + return folder_path + + def _get_chunk_time_bounds(self, startdate, chunk): + start = parse_date(startdate) + chunk_start = chunk_start_date(start, chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + chunk_end = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', self.experiment.calendar) + chunk_end = previous_day(chunk_end, self.experiment.calendar) + if self.config.data_convention == 'preface': + separator = '_' + else: + separator = '-' + time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, + chunk_end.month, separator) + return time_bound + + def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, + frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): + + """ + Creates the link of a given file from the CMOR repository. + + :param cmor_var: + :param move_old: + :param date_str: + :param year: if frequency is yearly, this parameter is used to give the corresponding year + :type year: int + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + :param cmor_var: variable instance describing the selected variable + :type cmor_var: Variable + """ + + if not frequency: + frequency = self.config.frequency + filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, + grid=grid, year=str(year), date_str=date_str) + self.create_link(domain, filepath, frequency, var, grid, move_old, vartype) + + # noinspection PyPep8Naming + def prepare(self): + """ + Prepares the data to be used by the diagnostic. + + If CMOR data is not created, it show a warning and closes. In the future, an automatic cmorization procedure + will be launched + + If CMOR data is available but packed, the procedure will unpack it. + + :return: + """ + # Check if cmorized and convert if not + + for startdate, member in self.experiment.get_member_list(): + if not self._unpack_cmor_files(startdate, member): + self._cmorize_member(startdate, member) + + def is_cmorized(self, startdate, member, chunk, domain): + identifier = (startdate, member, chunk) + if identifier not in self._dic_cmorized: + self._dic_cmorized[identifier] = {} + self._dic_cmorized[identifier][domain] = self._is_cmorized(startdate, member, chunk, domain) + elif domain not in self._dic_cmorized[identifier]: + self._dic_cmorized[identifier][domain] = self._is_cmorized(startdate, member, chunk, domain) + return self._dic_cmorized[identifier][domain] + + def _is_cmorized(self, startdate, member, chunk, domain): + startdate_path = self._get_startdate_path(startdate) + if not os.path.isdir(startdate_path): + return False + count = 0 + if self.config.data_convention == 'specs': + for freq in os.listdir(startdate_path): + domain_path = os.path.join(startdate_path, freq, + domain.name) + if os.path.isdir(domain_path): + for var in os.listdir(domain_path): + cmor_var = self.variable_list.get_variable(var, True) + var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, + Frequency(freq)) + if os.path.isfile(var_path): + count += 1 + if count >= self.config.cmor.min_cmorized_vars: + return True + else: + continue + else: + member_path = os.path.join(startdate_path, self._get_member_str(member)) + if not os.path.isdir(member_path): + return False + freq = Frequencies.monthly + table = domain.get_table(freq, self.config.data_convention) + table_dir = os.path.join(member_path, table.name) + if not os.path.isdir(table_dir): + return False + for var in os.listdir(table_dir): + cmor_var = self.variable_list.get_variable(var, True) + var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq) + if os.path.isfile(var_path): + count += 1 + if count >= self.config.cmor.min_cmorized_vars: + return True + else: + continue + return False + + def _cmorize_member(self, startdate, member): + start_time = datetime.now() + member_str = self.experiment.get_member_str(member) + Log.info('CMORizing startdate {0} member {1}. Starting at {0}', startdate, member_str, start_time) + cmorizer = Cmorizer(self, startdate, member) + cmorizer.cmorize_ocean() + cmorizer.cmorize_atmos() + Log.result('CMORized startdate {0} member {1}! Elapsed time: {2}\n\n', startdate, member_str, + datetime.now() - start_time) + + def _unpack_cmor_files(self, startdate, member): + if self.config.cmor.force: + return False + chunk = 1 + cmorized = False + + if not self.config.cmor.force_untar: + while self.is_cmorized(startdate, member, chunk, ModelingRealms.atmos) or \ + self.is_cmorized(startdate, member, chunk, ModelingRealms.ocean): + chunk += 1 + + while self._unpack_chunk(startdate, member, chunk): + chunk += 1 + cmorized = True + + if self.experiment.num_chunks <= chunk: + cmorized = True + if cmorized: + Log.info('Startdate {0} member {1} ready', startdate, member) + return cmorized + + def _unpack_chunk(self, startdate, member, chunk): + + filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar.gz') + if len(filepaths) > 0: + if self.config.cmor.chunk_cmorization_requested(chunk): + Log.info('Unzipping cmorized data for {0} {1} {2}...', startdate, member, chunk) + Utils.unzip(filepaths, True) + else: + return True + + if not os.path.exists(self.cmor_path): + os.mkdir(self.cmor_path) + + filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar') + if len(filepaths) > 0: + if self.config.cmor.chunk_cmorization_requested(chunk): + Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) + Utils.untar(filepaths, self.cmor_path) + self._correct_paths(startdate) + self.create_links(startdate, member) + return True + return False + + def _get_transferred_cmor_data_filepaths(self, startdate, member, chunk, extension): + tar_path = os.path.join(self.config.data_dir, self.experiment.expid, 'original_files', 'cmorfiles') + tar_original_files = os.path.join(self.config.data_dir, 'original_files', self.experiment.expid, + 'cmorfiles') + file_name = 'CMOR?_{0}_{1}_{2}_{3}-*.{4}'.format(self.experiment.expid, startdate, + self.experiment.get_member_str(member), + self.experiment.get_chunk_start_str(startdate, chunk), + extension) + filepaths = glob.glob(os.path.join(tar_path, file_name)) + filepaths += glob.glob(os.path.join(tar_path, 'outputs', file_name)) + filepaths += glob.glob(os.path.join(tar_original_files, file_name)) + filepaths += glob.glob(os.path.join(tar_original_files, 'outputs', file_name)) + return filepaths + + def _correct_paths(self, startdate): + self._remove_extra_output_folder() + self._fix_model_as_experiment_error(startdate) + + def _fix_model_as_experiment_error(self, startdate): + if self.experiment.experiment_name != self.experiment.model: + bad_path = os.path.join(self.cmor_path, self.experiment.institute, self.experiment.model, + self.experiment.model) + Log.debug('Correcting double model appearance') + for (dirpath, dirnames, filenames) in os.walk(bad_path, False): + for filename in filenames: + if '_S{0}_'.format(startdate) in filename: + continue + filepath = os.path.join(dirpath, filename) + good = filepath + good = good.replace('_{0}_output_'.format(self.experiment.model), + '_{0}_{1}_S{2}_'.format(self.experiment.model, + self.experiment.experiment_name, + startdate)) + + good = good.replace('/{0}/{0}'.format(self.experiment.model), + '/{0}/{1}'.format(self.experiment.model, + self.experiment.experiment_name)) + + Utils.move_file(filepath, good) + if self.experiment.model != self.experiment.experiment_name: + os.rmdir(dirpath) + Log.debug('Done') + + def _remove_extra_output_folder(self): + bad_path = os.path.join(self.cmor_path, 'output') + if os.path.exists(bad_path): + Log.debug('Moving CMOR files out of the output folder') + Utils.move_tree(bad_path, self.cmor_path) + Log.debug('Done') + + def create_links(self, startdate, member=None): + if member is not None: + member_str = self._get_member_str(member) + else: + member_str = None + Log.info('Creating links for CMOR files ({0})', startdate) + path = self._get_startdate_path(startdate) + for freq in os.listdir(path): + frequency = Frequency.parse(freq) + for domain in os.listdir(os.path.join(path, freq)): + for var in os.listdir(os.path.join(path, freq, domain)): + for member in os.listdir(os.path.join(path, freq, domain, var)): + if member_str != member: + continue + for name in os.listdir(os.path.join(path, freq, domain, var, member)): + filepath = os.path.join(path, freq, domain, var, member, name) + if os.path.isfile(filepath): + self.create_link(domain, filepath, frequency, var, "", False, + vartype=VariableType.MEAN) + else: + for filename in os.listdir(filepath): + self.create_link(domain, os.path.join(filepath, filename), frequency, var, "", + False, vartype=VariableType.MEAN) + Log.debug('Links ready') + + def _get_startdate_path(self, startdate): + """ + Returns the path to the startdate's CMOR folder + :param startdate: target startdate + :type startdate: str + :return: path to the startdate's CMOR º + :rtype: str + """ + if self.config.data_convention == 'specs': + return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.experiment.institute, + self.experiment.model, self.experiment.experiment_name, 'S' + startdate) + elif self.config.data_convention == 'preface': + return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.experiment.institute, + self.experiment.experiment_name, 'S' + startdate) + else: + return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.config.cmor.activity, + self.experiment.institute, self.experiment.model, self.experiment.experiment_name) + + def _get_member_str(self, member): + if self.config.data_convention in ('specs', 'preface'): + template = 'r{0}i{1}p1' + elif self.config.data_convention in ('primavera', 'cmip6'): + template = 'r{0}i{1}p1f1' + else: + raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) + + return template.format(member + 1 - self.experiment.member_count_start, self.config.cmor.initialization_number) + + +class MergeYear(Diagnostic): + @classmethod + def generate_jobs(cls, diags, options): + pass + + def __init__(self, data_manager, domain, var, startdate, member, year, grid=None, box=None, frequency=None): + super(MergeYear, self).__init__(data_manager) + self.chunk_files = [] + self.experiment = self.data_manager.experiment + self.domain = domain + self.var = var + self.startdate = startdate + self.member = member + self.year = year + self.grid = grid + self.box = box + self.frequency = frequency + + def request_data(self): + for chunk in self.experiment.get_year_chunks(self.startdate, self.year): + self.chunk_files.append(self.request_chunk(self.domain, self.var, self.startdate, self.member, chunk, + grid=self.grid, box=self.box, frequency=self.frequency)) + + def declare_data_generated(self): + self.year_file = self.declare_year(self.domain, self.var, self.startdate, self.member, self.year, + grid=self.grid, box=self.box) + self.year_file.storage_status = StorageStatus.NO_STORE + + def compute(self): + temp = self._merge_chunk_files() + temp2 = self._select_data_of_given_year(temp) + self.year_file.set_local_file(temp2) + + def _select_data_of_given_year(self, data_file): + temp2 = TempFile.get() + handler = Utils.openCdf(data_file) + times = Utils.get_datetime_from_netcdf(handler) + x = 0 + first_index = None + last_index = None + while x < times.size: + if times[x].year == self.year: + first_index = x + break + else: + x += 1 + + while x < times.size: + if times[x].year != self.year: + last_index = x + break + else: + x += 1 + if last_index is None: + last_index = times.size + Utils.nco.ncks(input=data_file, output=temp2, options=['-d time,{0},{1}'.format(first_index, last_index - 1)]) + return temp2 + + def _merge_chunk_files(self): + temp = TempFile.get() + if len(self.chunk_files) == 1: + Utils.copy_file(self.chunk_files[0].local_file, temp) + return temp + + Utils.nco.ncrcat(input=' '.join(self.chunk_files), output=temp) + for chunk_file in self.chunk_files: + os.remove(chunk_file) + return temp + + def __str__(self): + return 'Create year CMOR file Startdate: {0.startdate} Member: {0.member} Year: {0.year} ' \ + 'Variable: {0.domain}:{0.var} Grid: {0.grid} Box: {0.box}'.format(self) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.year == other.year and\ + self.domain == other.domain and self.var == other.var and self.grid == other.grid and \ + self.box == other.box diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py new file mode 100644 index 0000000000000000000000000000000000000000..133bd6daa4945a79fd7a8c97c8f738cd05e2cfd7 --- /dev/null +++ b/earthdiagnostics/config.py @@ -0,0 +1,401 @@ +# coding=utf-8 +import os + +import six +from bscearth.utils.config_parser import ConfigParser +from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str +from bscearth.utils.log import Log + +from earthdiagnostics.frequency import Frequency, Frequencies +from earthdiagnostics.variable import VariableManager +from modelingrealm import ModelingRealm + + +class ConfigException(Exception): + pass + + +class Config(object): + """ + Class to read and manage the configuration + + :param path: path to the conf file + :type path: str + """ + + def __init__(self, path): + + parser = ConfigParser() + parser.optionxform = str + parser.read(path) + + # Read diags config + self.data_adaptor = parser.get_choice_option('DIAGNOSTICS', 'DATA_ADAPTOR', ('CMOR', 'THREDDS', 'OBSRECON'), + 'CMOR') + "Scratch folder path" + self.scratch_dir = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_DIR') + "Scratch folder path" + self.use_ramdisk = parser.get_bool_option('DIAGNOSTICS', 'USE_RAMDISK', False) + "If True, the scratch dir is created as a ram disk" + self.auto_clean = parser.get_bool_option('DIAGNOSTICS', 'AUTO_CLEAN', True) + "If True, the scratch dir is removed after finishing" + if not self.auto_clean and self.use_ramdisk: + Log.warning('RAM disk scratch dir is always automatically cleaned.') + self.auto_clean = True + + self.scratch_masks = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_MASKS', '/scratch/Earth/ocean_masks') + "Common scratch folder for masks" + self.data_dir = parser.get_path_option('DIAGNOSTICS', 'DATA_DIR') + "Root data folder path" + self.data_type = parser.get_choice_option('DIAGNOSTICS', 'DATA_TYPE', ('exp', 'obs', 'recon'), 'exp') + "Data type (experiment, observation or reconstruction)" + self.con_files = parser.get_path_option('DIAGNOSTICS', 'CON_FILES') + "Mask and meshes folder path" + self.mesh_mask = parser.get_path_option('DIAGNOSTICS', 'MESH_MASK', '') + "Custom mesh mask file to use" + self.new_mask_glo = parser.get_path_option('DIAGNOSTICS', 'NEW_MASK_GLO', '') + "Custom new mask glo file to use" + self.mask_regions = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS', '') + "Custom mask regions file to use" + self.mask_regions_3d = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS_3D', '') + "Custom mask regions 3D file to use" + + self.data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', + ('specs', 'primavera', 'cmip6', 'preface'), 'specs', + ignore_case=True) + self.var_manager = VariableManager() + self.var_manager.load_variables(self.data_convention) + self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') + self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY')) + "Default data frequency to be used by the diagnostics" + + self.cdftools_path = parser.get_path_option('DIAGNOSTICS', 'CDFTOOLS_PATH', '') + "Path to CDFTOOLS executables" + self.max_cores = parser.get_int_option('DIAGNOSTICS', 'MAX_CORES', 0) + "Maximum number of cores to use" + self.parallel_downloads = parser.get_int_option('DIAGNOSTICS', 'PARALLEL_DOWNLOADS', 1) + "Maximum number of simultaneous downloads" + self.parallel_uploads = parser.get_int_option('DIAGNOSTICS', 'PARALLEL_UPLOADS', 1) + "Maximum number of simultaneous uploads" + self.restore_meshes = parser.get_bool_option('DIAGNOSTICS', 'RESTORE_MESHES', False) + "If True, forces the tool to copy all the mesh and mask files for the model, regardless of existence" + + # Read experiment config + self.experiment = ExperimentConfig(parser) + """ + Configuration related to the experiment + + :rtype: ExperimentConfig + """ + # Read aliases + self._aliases = dict() + if parser.has_section('ALIAS'): + for option in parser.options('ALIAS'): + self._aliases[option.lower()] = parser.get_list_option('ALIAS', option) + Log.debug('Preparing command list') + commands = self._diags.split() + self._real_commands = list() + for command in commands: + if command.lower() in self._aliases: + added_commands = self._aliases[command.lower()] + Log.info('Changing alias {0} for {1}', command, ' '.join(added_commands)) + for add_command in added_commands: + self._real_commands.append(add_command) + else: + self._real_commands.append(command) + Log.debug('Command list ready ') + + self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid) + + self.cmor = CMORConfig(parser, self.var_manager) + self.thredds = THREDDSConfig(parser) + self.report = ReportConfig(parser) + + def get_commands(self): + """ + Returns the list of commands after replacing the alias + :return: full list of commands + :rtype: list(str) + """ + return self._real_commands + + +class CMORConfig(object): + + def __init__(self, parser, var_manager): + self.force = parser.get_bool_option('CMOR', 'FORCE', False) + self.force_untar = parser.get_bool_option('CMOR', 'FORCE_UNTAR', False) + self.filter_files = parser.get_option('CMOR', 'FILTER_FILES', '') + self.ocean = parser.get_bool_option('CMOR', 'OCEAN_FILES', True) + self.atmosphere = parser.get_bool_option('CMOR', 'ATMOSPHERE_FILES', True) + self.use_grib = parser.get_bool_option('CMOR', 'USE_GRIB', True) + self._chunks = parser.get_int_list_option('CMOR', 'CHUNKS') + self.associated_experiment = parser.get_option('CMOR', 'ASSOCIATED_EXPERIMENT', 'to be filled') + self.associated_model = parser.get_option('CMOR', 'ASSOCIATED_MODEL', 'to be filled') + self.initialization_description = parser.get_option('CMOR', 'INITIALIZATION_DESCRIPTION', 'to be filled') + self.initialization_method = parser.get_option('CMOR', 'INITIALIZATION_METHOD', '1') + self.initialization_number = parser.get_int_option('CMOR', 'INITIALIZATION_NUMBER', 1) + self.physics_description = parser.get_option('CMOR', 'PHYSICS_DESCRIPTION', 'to be filled') + self.physics_version = parser.get_option('CMOR', 'PHYSICS_VERSION', '1') + self.source = parser.get_option('CMOR', 'SOURCE', 'to be filled') + self.version = parser.get_option('CMOR', 'VERSION', '') + self.default_ocean_grid = parser.get_option('CMOR', 'DEFAULT_OCEAN_GRID', 'gn') + self.default_atmos_grid = parser.get_option('CMOR', 'DEFAULT_ATMOS_GRID', 'gr') + self.activity = parser.get_option('CMOR', 'ACTIVITY', 'CMIP') + self.min_cmorized_vars = parser.get_int_option('CMOR', 'MIN_CMORIZED_VARS', 10) + + vars_string = parser.get_option('CMOR', 'VARIABLE_LIST', '') + self.var_manager = var_manager + if vars_string: + self._variable_list = list() + for domain_var in vars_string.split(' '): + if domain_var.startswith('#'): + break + splitted = domain_var.split(':') + cmor_var = self.var_manager.get_variable(splitted[1], silent=True) + if not cmor_var: + Log.warning('Variable {0} not recognized. It will not be cmorized', domain_var) + continue + if ModelingRealm(splitted[0]) != cmor_var.domain: + Log.warning('Domain {0} for variable {1} is not correct: is {2}', splitted[0], cmor_var.short_name, + cmor_var.domain) + continue + self._variable_list.append('{0.domain}:{0.short_name}'.format(cmor_var)) + if len(self._variable_list) == 0: + raise ConfigException('Variable list value is specified, but no variables were found') + else: + self._variable_list = None + + self._var_hourly = CMORConfig._parse_variables(parser.get_option('CMOR', 'ATMOS_HOURLY_VARS', '')) + self._var_daily = CMORConfig._parse_variables(parser.get_option('CMOR', 'ATMOS_DAILY_VARS', '')) + self._var_monthly = CMORConfig._parse_variables(parser.get_option('CMOR', 'ATMOS_MONTHLY_VARS', '')) + + def cmorize(self, var_cmor): + """ + Checks if var_cmor is on variable list + + :param var_cmor: CMOR variable object + :rtype var_cmor: Variablle + :return: + """ + if self._variable_list is None: + return True + if not var_cmor: + return False + return '{0}:{1}'.format(var_cmor.domain, var_cmor.short_name) in self._variable_list + + def any_required(self, variables): + if self._variable_list is None: + return True + for var in variables: + if self.cmorize(self.var_manager.get_variable(var, silent=True)): + return True + + return False + + def chunk_cmorization_requested(self, chunk): + if len(self._chunks) == 0: + return True + return chunk in self._chunks + + @staticmethod + def _parse_variables(raw_string): + variables = dict() + if raw_string: + splitted = raw_string.split(',') + for var_section in splitted: + splitted_var = var_section.split(':') + if len(splitted_var) == 1: + levels = None + else: + levels = ','.join(map(str, CMORConfig._parse_levels(splitted_var[1:]))) + variables[int(splitted_var[0])] = levels + return variables + + @staticmethod + def _parse_levels(levels_splitted): + if len(levels_splitted) == 1: + return map(int, levels_splitted[0].split('-')) + start = int(levels_splitted[0]) + end = int(levels_splitted[1]) + if len(levels_splitted) == 3: + step = int(levels_splitted[2]) + else: + step = 1 + return range(start, end, step) + + def get_variables(self, frequency): + if frequency in (Frequencies.three_hourly, Frequencies.six_hourly): + return self._var_hourly + elif frequency == Frequencies.daily: + return self._var_daily + elif frequency == Frequencies.monthly: + return self._var_monthly + raise ValueError('Frequency not recognized: {0}'.format(frequency)) + + def get_levels(self, frequency, variable): + return self.get_variables(frequency)[variable] + + +class THREDDSConfig(object): + def __init__(self, parser): + self.server_url = parser.get_option('THREDDS', 'SERVER_URL', '') + + +class ExperimentConfig(object): + """ + Encapsulates all chunk related tasks + + :param parser: parser for the config file + :type parser: Parser + """ + + def __init__(self, parser): + self.institute = parser.get_option('EXPERIMENT', 'INSTITUTE') + self.expid = parser.get_option('EXPERIMENT', 'EXPID') + self.experiment_name = parser.get_option('EXPERIMENT', 'NAME', self.expid) + self.members = parser.get_list_option('EXPERIMENT', 'MEMBERS') + self.member_digits = parser.get_int_option('EXPERIMENT', 'MEMBER_DIGITS', 1) + self.member_prefix = parser.get_option('EXPERIMENT', 'MEMBER_PREFIX', 'fc') + self.member_count_start = parser.get_int_option('EXPERIMENT', 'MEMBER_COUNT_START', 0) + + members = [] + for mem in self.members: + if '-' in mem: + start, end = mem.split('-') + if start.startswith(self.member_prefix): + start = start[len(self.member_prefix):] + if end.startswith(self.member_prefix): + end = end[len(self.member_prefix):] + for member in range(int(start), int(end) + 1): + members.append(member) + else: + if mem.startswith(self.member_prefix): + mem = mem[len(self.member_prefix):] + members.append(int(mem)) + self.members = members + + startdates = parser.get_list_option('EXPERIMENT', 'STARTDATES') + + import exrex + self.startdates = [] + for startdate_pattern in startdates: + for startdate in exrex.generate(startdate_pattern): + self.startdates.append(startdate) + + self.chunk_size = parser.get_int_option('EXPERIMENT', 'CHUNK_SIZE') + self.num_chunks = parser.get_int_option('EXPERIMENT', 'CHUNKS') + self.chunk_list = parser.get_int_list_option('EXPERIMENT', 'CHUNK_LIST', []) + self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') + self.model = parser.get_option('EXPERIMENT', 'MODEL') + self.model_version = parser.get_option('EXPERIMENT', 'MODEL_VERSION', '') + self.atmos_grid = parser.get_option('EXPERIMENT', 'ATMOS_GRID', '') + self.atmos_timestep = parser.get_int_option('EXPERIMENT', 'ATMOS_TIMESTEP', 6) + self.ocean_timestep = parser.get_int_option('EXPERIMENT', 'OCEAN_TIMESTEP', 6) + + def get_chunk_list(self): + """ + Return a list with all the chunks + :return: List containing tuples of startdate, member and chunk + :rtype: tuple[str, int, int] + """ + chunk_list = list() + for startdate in self.startdates: + for member in self.members: + if len(self.chunk_list) == 0: + for chunk in range(1, self.num_chunks + 1): + chunk_list.append((startdate, member, chunk)) + else: + for chunk in self.chunk_list: + chunk_list.append((startdate, member, chunk)) + return chunk_list + + def get_member_list(self): + """ + Return a list with all the members + :return: List containing tuples of startdate and member + :rtype: tuple[str, int, int] + """ + member_list = list() + for startdate in self.startdates: + for member in self.members: + member_list.append((startdate, member)) + return member_list + + def get_year_chunks(self, startdate, year): + """ + Get the list of chunks containing timesteps from the given year + :param startdate: startdate to use + :type startdate: str + :param year: reference year + :type year: int + :return: list of chunks containing data from the given year + :rtype: list[int] + """ + date = parse_date(startdate) + chunks = list() + for chunk in range(1, self.num_chunks + 1): + chunk_start = self.get_chunk_start(date, chunk) + if chunk_start.year > year: + break + elif chunk_start.year == year or chunk_end_date(chunk_start, self.chunk_size, 'month', + self.calendar).year == year: + chunks.append(chunk) + + return chunks + + def get_chunk_start(self, startdate, chunk): + # noinspection PyTypeChecker + if isinstance(startdate, six.string_types): + startdate = parse_date(startdate) + return chunk_start_date(startdate, chunk, self.chunk_size, 'month', self.calendar) + + def get_chunk_start_str(self, startdate, chunk): + return date2str(self.get_chunk_start(startdate, chunk)) + + def get_chunk_end(self, startdate, chunk): + return chunk_end_date(self.get_chunk_start(startdate, chunk), self.chunk_size, 'month', self.calendar) + + def get_chunk_end_str(self, startdate, chunk): + return date2str(self.get_chunk_end(startdate, chunk)) + + def get_full_years(self, startdate): + """ + Returns the list of full years that are in the given startdate + :param startdate: startdate to use + :type startdate: str + :return: list of full years + :rtype: list[int] + """ + chunks_per_year = 12 / self.chunk_size + date = parse_date(startdate) + first_january = 0 + first_year = date.year + if date.month != 1: + month = date.month + first_year += 1 + while month + self.chunk_size < 12: + month += self.chunk_size + first_january += 1 + + years = list() + for chunk in range(first_january, chunks_per_year, self.num_chunks): + years.append(first_year) + first_year += 1 + return years + + def get_member_str(self, member): + """ + Returns the member name for a given member number. + :param member: member's number + :type member: int + :return: member's name + :rtype: str + """ + return '{0}{1}'.format(self.member_prefix, str(member).zfill(self.member_digits)) + + +class ReportConfig(object): + def __init__(self, parser): + self.maximum_priority = parser.get_int_option('REPORT', 'MAXIMUM_PRIORITY', 10) + self.path = parser.get_path_option('REPORT', 'PATH', '') diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..c1e51013879b1417bbb0960b3963bc949a2d691d --- /dev/null +++ b/earthdiagnostics/constants.py @@ -0,0 +1,205 @@ +# coding=utf-8 +""" +Contains the enumeration-like classes used by the diagnostics +""" +import netCDF4 +from singleton import SingletonType + + +class Basin(object): + """ + Class representing a given basin + + :param name: full basin's name + :type name: str + """ + + def __init__(self, name): + self._name = name + + def __eq__(self, other): + if self.name != other.name: + return False + return True + + def __ne__(self, other): + return not self == other + + def __str__(self): + return self._name + + @property + def name(self): + """ + Basin's full name + :rtype: str + """ + return self._name + + +class Basins(object): + """ + Predefined basins + """ + + __metaclass__ = SingletonType + + def __init__(self): + + self.aliases = { + 'Atlantic_Ocean': ('atl', 'atlantic'), + 'North_Atlantic_Ocean': ('natl', 'north_atlantic'), + 'Tropical_Atlantic_Ocean': ('tatl', 'tropical_atlantic'), + + 'Pacific_Ocean': ('pac', 'pacific'), + 'North_Pacific_Ocean': ('npac', 'north_pacific'), + 'Tropical_Pacific_Ocean': ('tpac', 'tropical_pacific'), + 'Indo_Pacific_Ocean': ('indpac', 'indo_pacific'), + + 'Indian_Ocean': ('ind', 'indian'), + 'Tropical_Indian_Ocean': ('tind', 'tropical_indian'), + + 'Antarctic_Ocean': ('anta', 'antarctiv'), + 'Antarctic_Atlantic_Sector': ('antaatl', 'antarctic_atlantic'), + 'Antarctic_Indian_Sector': ('antaind', 'antarctic_indian'), + + 'Arctic_Ocean': ('arct', 'arctic'), + 'Arctic_Ocean_North_Atlantic': ('arctnatl', 'artic_north_atlantic'), + 'Arctic_Marginal_Seas': ('arctmarg', 'arctic_marginal'), + + 'Baffin': ('Baffin',), + 'Baffin_Bay': ('BaffBay',), + 'Baltic_Sea': ('Baltic',), + 'BarKara': ('BarKara',), + 'Barents_Sea': ('Barents',), + 'Beaufort_Chukchi_Sea': ('BeaufortChukchi',), + 'Beaufort_Sea': ('Beaufort',), + 'Bellingshausen_Sea': ('Bellingshausen',), + 'Bering': ('Bering',), + 'Bering_Strait': ('BeringStr',), + 'CanArch': ('CanArch',), + 'Canadian_Waters': ('Canadian',), + 'Caspian_Sea': ('Caspian',), + 'Central_Arctic': ('CArct',), + 'Chukchi_Sea': ('Chukchi',), + 'East_Siberian_Sea': ('ESiberian',), + 'Eastern_Central_Arctic': ('ECArct',), + 'Fram_Strait': ('Fram',), + 'Greenland_Sea': ('Greenland',), + 'Grnland': ('Grnland',), + 'Hudson': ('Hudson',), + 'Icelandic_Sea': ('Iceland',), + 'Irminger_Sea': ('Irminger',), + 'Kara_Gate_Strait': ('KaraGate',), + 'Kara_Sea': ('Kara',), + 'Labrador_Sea': ('Labrador',), + 'Laptev_East_Siberian_Chukchi_Seas': ('LaptevESiberianChukchi',), + 'Laptev_East_Siberian_Seas': ('LaptevESiberian',), + 'Laptev_Sea': ('Laptev',), + 'Lincoln_Sea': ('Lincoln',), + 'Mediterranean_Sea': ('Medit',), + 'Nares_Strait': ('Nares',), + 'Nordic_Barents_Seas': ('NordicBarents',), + 'Nordic_Seas': ('Nordic',), + 'NorthWest_Passage': ('NWPass',), + 'North_Atlantic-Arctic': ('North_Atlantic-Arctic',), + 'North_Hemisphere_Ocean': ('NHem',), + 'Norwegian_Sea': ('Norwegian',), + 'Okhotsk': ('Okhotsk',), + 'OpenOcean': ('OpenOcean',), + 'Ross_Sea': ('Ross',), + 'Serreze_Arctic': ('SerArc',), + 'Southern_Hemisphere': ('SHem',), + 'StLawr': ('StLawr',), + 'Subpolar_Gyre': ('Subpolar_Gyre',), + 'TotalArc': ('TotalArc',), + 'Vilkitsky_Strait': ('Vilkitsky_Strait',), + 'Weddell_Sea': ('Weddell_Sea',), + 'Western_Central_Arctic': ('Western_Central_Arctic',), + } + + self.Global = Basin('Global') + self.Atlantic = Basin('Atlantic_Ocean') + self.Pacific = Basin('Pacific_Ocean') + self.IndoPacific = Basin('Indo_Pacific_Ocean') + self.Indian = Basin('Indian_Ocean') + self._known_aliases = {} + self._add_alias('glob', self.Global) + self._add_alias(self.Global.name, self.Global) + + def get_available_basins(self, handler): + """ + + :param handler: + :type handler: netCDF4.Dataset + """ + basin_names = handler.variables.keys() + + ignored_names = ('lat', 'lon', 'i', 'j', 'time', 'lev') + + for basin in basin_names: + if basin in ignored_names: + continue + basin_object = Basin(basin) + setattr(self, basin, basin_object) + self._add_alias(basin, basin_object) + try: + for alias in self.aliases[basin]: + self._add_alias(alias, basin_object) + except KeyError: + pass + + def _add_alias(self, basin, basin_object): + self._known_aliases[basin.lower()] = basin_object + + def parse(self, basin): + """ + Return the basin matching the given name. If the parameter basin is a Basin instance, directly returns the same + instance. This bahaviour is intended to facilitate the development of methods that can either accept a name + or a Basin instance to characterize the basin. + + :param basin: basin name or basin instance + :type basin: str | Basin + :return: basin instance corresponding to the basin name + :rtype: Basin + """ + if isinstance(basin, Basin): + return basin + basin = basin.lower() + try: + return self._known_aliases[basin] + except KeyError: + return None + + +class Models(object): + """ + Predefined models + """ + + ECEARTH_2_3_O1L42 = 'Ec2.3_O1L42' + """ EC-Earth 2.3 ORCA1 L42""" + ECEARTH_3_0_O1L46 = 'Ec3.0_O1L46' + """ EC-Earth 3 ORCA1 L46 """ + ECEARTH_3_0_O25L46 = 'Ec3.0_O25L46' + """ EC-Earth 3 ORCA0.25 L46 """ + ECEARTH_3_0_O25L75 = 'Ec3.0_O25L75' + """ EC-Earth 3 ORCA0.25 L75 """ + ECEARTH_3_1_O25L75 = 'Ec3.1_O25L75' + """ EC-Earth 3.1 ORCA0.25 L75 """ + ECEARTH_3_2_O1L75 = 'Ec3.2_O1L75' + """ EC-Earth 3.2 ORCA1 L75 """ + ECEARTH_3_2_O25L75 = 'Ec3.2_O25L75' + """ EC-Earth 3.2 ORCA0.25 L75 """ + + NEMO_3_2_O1L42 = 'N3.2_O1L42' + """ NEMO 3.2 ORCA1 L42 """ + NEMO_3_3_O1L46 = 'N3.3_O1L46' + """ NEMO 3.3 ORCA1 L46 """ + NEMO_3_6_O1L46 = 'N3.6_O1L75' + """ NEMO 3.6 ORCA1 L75 """ + + NEMOVAR_O1L42 = 'nemovar_O1L42' + """ NEMOVAR ORCA1 L42 """ + GLORYS2_V1_O25L75 = 'glorys2v1_O25L75' + """ GLORYS2v1 ORCA0.25 L75 """ diff --git a/earthdiagnostics/conversions.csv b/earthdiagnostics/conversions.csv new file mode 100644 index 0000000000000000000000000000000000000000..5e07878874b74395c3bc95e46fb58e881711279e --- /dev/null +++ b/earthdiagnostics/conversions.csv @@ -0,0 +1,8 @@ +original,converted,factor ,offset +C,K,1,273.15 +degC,K,1,273.15 +m,km,1000,0 +m2,km2,1.00E+006,0 +m3,km3,1.00E+009,0 +"[0,1]",%,100,0 +1e-3,psu,1,0 diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py new file mode 100644 index 0000000000000000000000000000000000000000..ddaf1b102e54e16dc8bce3e3faaf5cbd3f4adb50 --- /dev/null +++ b/earthdiagnostics/datafile.py @@ -0,0 +1,475 @@ +# coding: utf-8 +import csv +import os +import shutil +from datetime import datetime + +import numpy as np +from bscearth.utils.log import Log + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile +from publisher import Publisher +from variable_type import VariableType + + +class LocalStatus(object): + PENDING = 0 + DOWNLOADING = 1 + READY = 2 + FAILED = 3 + NOT_REQUESTED = 4 + COMPUTING = 5 + + +class StorageStatus(object): + PENDING = 0 + UPLOADING = 1 + READY = 2 + FAILED = 3 + NO_STORE = 4 + + +class DataFile(Publisher): + + def __init__(self): + super(DataFile, self).__init__() + self.remote_file = None + self.local_file = None + self.domain = None + self.var = None + self.cmor_var = None + self.region = None + self.frequency = None + self.data_convention = None + self.diagnostic = None + self.grid = None + self.data_manager = None + self.final_name = None + self.var_type = VariableType.MEAN + self._local_status = LocalStatus.NOT_REQUESTED + self._storage_status = StorageStatus.READY + self.job_added = False + self._modifiers = [] + + def __str__(self): + return 'Data file for {0}'.format(self.remote_file) + + def unsubscribe(self, who): + super(DataFile, self).unsubscribe(who) + self._clean_local() + + @property + def size(self): + if self.local_status == LocalStatus.READY: + os.path.getsize(self.local_file) + return None + + def _clean_local(self): + if self.local_status != LocalStatus.READY or len(self.suscribers) > 0 or self.upload_required(): + return + Log.debug('File {0} no longer needed. Deleting from scratch...'.format(self.remote_file)) + os.remove(self.local_file) + Log.debug('File {0} deleted from scratch'.format(self.remote_file)) + self.local_file = None + self.local_status = LocalStatus.PENDING + + def upload_required(self): + return self.local_status == LocalStatus.READY and self.storage_status == StorageStatus.PENDING + + def download_required(self): + if not self.local_status == LocalStatus.PENDING: + return False + + if self.storage_status == StorageStatus.READY: + return True + + if self.has_modifiers(): + return True + + def add_modifier(self, diagnostic): + self._modifiers.append(diagnostic) + + def has_modifiers(self): + return len(self._modifiers) > 0 + + def ready_to_run(self, diagnostic): + if not self.local_status == LocalStatus.READY: + return False + if len(self._modifiers) == 0: + return True + return self._modifiers[0] is diagnostic + + @property + def local_status(self): + return self._local_status + + @local_status.setter + def local_status(self, value): + if self._local_status == value: + return + self._local_status = value + self.dispatch(self) + + @property + def storage_status(self): + return self._storage_status + + @storage_status.setter + def storage_status(self, value): + if self._storage_status == value: + return + self._storage_status = value + self.dispatch(self) + + @classmethod + def from_storage(cls, filepath): + file_object = cls() + file_object.remote_file = filepath + file_object.local_status = LocalStatus.PENDING + return file_object + + @classmethod + def to_storage(cls, remote_file): + new_object = cls() + new_object.remote_file = remote_file + new_object.storage_status = StorageStatus.PENDING + return new_object + + def download(self): + raise NotImplementedError('Class must implement the download method') + + def prepare_to_upload(self, rename_var): + Utils.convert2netcdf4(self.local_file) + if rename_var: + original_name = rename_var + else: + original_name = self.var + if self.final_name != original_name: + Utils.rename_variable(self.local_file, original_name, self.final_name) + self._rename_coordinate_variables() + self._correct_metadata() + self._prepare_region() + + self.add_diagnostic_history() + + def upload(self): + self.storage_status = StorageStatus.UPLOADING + try: + Utils.copy_file(self.local_file, self.remote_file, save_hash=True) + except Exception as ex: + Log.error('File {0} can not be uploaded: {1}', self.remote_file, ex) + self.storage_status = StorageStatus.FAILED + return + + Log.info('File {0} uploaded!', self.remote_file) + try: + self.create_link() + except Exception as ex: + Log.warning('Link for file {0} can not be created: {1}', self.remote_file, ex) + self.storage_status = StorageStatus.READY + self._clean_local() + + def set_local_file(self, local_file, diagnostic=None, rename_var=''): + if diagnostic in self._modifiers: + self._modifiers.remove(diagnostic) + self.local_file = local_file + self.prepare_to_upload(rename_var) + self.local_status = LocalStatus.READY + + def create_link(self): + pass + + def _correct_metadata(self): + handler = Utils.openCdf(self.local_file) + var_handler = handler.variables[self.final_name] + coords = set.intersection({'time', 'lev', 'lat', 'lon'}, set(handler.variables.keys())) + var_handler.coordinates = ' '.join(coords) + if not self.cmor_var: + handler.close() + return + + self._fix_variable_name(var_handler) + handler.modeling_realm = self.cmor_var.domain.name + table = self.cmor_var.get_table(self.frequency, self.data_convention) + handler.table_id = 'Table {0} ({1})'.format(table.name, table.date) + if self.cmor_var.units: + self._fix_units(var_handler) + handler.sync() + self._fix_coordinate_variables_metadata(handler) + var_type = var_handler.dtype + handler.close() + self._fix_values_metadata(var_type) + + def _fix_variable_name(self, var_handler): + var_handler.standard_name = self.cmor_var.standard_name + var_handler.long_name = self.cmor_var.long_name + # var_handler.short_name = self.cmor_var.short_name + + def _fix_values_metadata(self, var_type): + if self.cmor_var.valid_min != '': + valid_min = '-a valid_min,{0},o,{1},"{2}" '.format(self.final_name, var_type.char, self.cmor_var.valid_min) + else: + valid_min = '' + if self.cmor_var.valid_max != '': + valid_max = '-a valid_max,{0},o,{1},"{2}" '.format(self.final_name, var_type.char, self.cmor_var.valid_max) + else: + valid_max = '' + Utils.nco.ncatted(input=self.local_file, output=self.local_file, + options=('-O -a _FillValue,{0},o,{1},"1.e20" ' + '-a missingValue,{0},o,{1},"1.e20" {2}{3}'.format(self.final_name, var_type.char, + valid_min, valid_max),)) + + def _fix_coordinate_variables_metadata(self, handler): + if 'lev' in handler.variables: + handler.variables['lev'].short_name = 'lev' + if self.domain == ModelingRealms.ocean: + handler.variables['lev'].standard_name = 'depth' + if 'lon' in handler.variables: + handler.variables['lon'].short_name = 'lon' + handler.variables['lon'].standard_name = 'longitude' + if 'lat' in handler.variables: + handler.variables['lat'].short_name = 'lat' + handler.variables['lat'].standard_name = 'latitude' + + def _fix_units(self, var_handler): + if 'units' not in var_handler.ncattrs(): + return + if var_handler.units == '-': + var_handler.units = '1.0' + if var_handler.units == 'PSU': + var_handler.units = 'psu' + if var_handler.units == 'C' and self.cmor_var.units == 'K': + var_handler.units = 'deg_C' + if self.cmor_var.units != var_handler.units: + self._convert_units(var_handler) + var_handler.units = self.cmor_var.units + + def _convert_units(self, var_handler): + try: + Utils.convert_units(var_handler, self.cmor_var.units) + except ValueError as ex: + Log.warning('Can not convert {3} from {0} to {1}: {2}', var_handler.units, self.cmor_var.units, ex, + self.cmor_var.short_name) + factor, offset = UnitConversion.get_conversion_factor_offset(var_handler.units, + self.cmor_var.units) + + var_handler[:] = var_handler[:] * factor + offset + if 'valid_min' in var_handler.ncattrs(): + var_handler.valid_min = float(var_handler.valid_min) * factor + offset + if 'valid_max' in var_handler.ncattrs(): + var_handler.valid_max = float(var_handler.valid_max) * factor + offset + + def _prepare_region(self): + if not self.region: + return + if not os.path.exists(self.remote_file): + self._add_region_dimension_to_var() + else: + self._update_var_with_region_data() + self._correct_metadata() + Utils.nco.ncks(input=self.local_file, output=self.local_file, options=['--fix_rec_dmn region']) + + def _update_var_with_region_data(self): + temp = TempFile.get() + shutil.copyfile(self.remote_file, temp) + Utils.nco.ncks(input=temp, output=temp, options=['--mk_rec_dmn region']) + handler = Utils.openCdf(temp) + handler_send = Utils.openCdf(self.local_file) + value = handler_send.variables[self.final_name][:] + var_region = handler.variables['region'] + basin_index = np.where(var_region[:] == self.region) + if len(basin_index[0]) == 0: + var_region[var_region.shape[0]] = self.region + basin_index = var_region.shape[0] - 1 + + else: + basin_index = basin_index[0][0] + handler.variables[self.final_name][..., basin_index] = value + handler.close() + handler_send.close() + Utils.move_file(temp, self.local_file) + + def _add_region_dimension_to_var(self): + handler = Utils.openCdf(self.local_file) + handler.createDimension('region') + var_region = handler.createVariable('region', str, 'region') + var_region[0] = self.region + original_var = handler.variables[self.final_name] + new_var = handler.createVariable('new_var', original_var.datatype, + original_var.dimensions + ('region',)) + new_var.setncatts({k: original_var.getncattr(k) for k in original_var.ncattrs()}) + value = original_var[:] + new_var[..., 0] = value + handler.close() + Utils.nco.ncks(input=self.local_file, output=self.local_file, options='-O -x -v {0}'.format(self.final_name)) + Utils.rename_variable(self.local_file, 'new_var', self.final_name) + + def _rename_coordinate_variables(self): + variables = dict() + variables['x'] = 'i' + variables['y'] = 'j' + variables['nav_lat_grid_V'] = 'lat' + variables['nav_lon_grid_V'] = 'lon' + variables['nav_lat_grid_U'] = 'lat' + variables['nav_lon_grid_U'] = 'lon' + variables['nav_lat_grid_T'] = 'lat' + variables['nav_lon_grid_T'] = 'lon' + Utils.rename_variables(self.local_file, variables, False, True) + + def add_diagnostic_history(self): + if not self.diagnostic: + return + from earthdiagnostics.earthdiags import EarthDiags + history_line = 'Diagnostic {1} calculated with EarthDiagnostics version {0}'.format(EarthDiags.version, + self.diagnostic) + self._add_history_line(history_line) + + def add_cmorization_history(self): + from earthdiagnostics.earthdiags import EarthDiags + history_line = 'CMORized with Earthdiagnostics version {0}'.format(EarthDiags.version) + self._add_history_line(history_line) + + def _add_history_line(self, history_line): + utc_datetime = 'UTC ' + datetime.utcnow().isoformat() + history_line = '{0}: {1};'.format(utc_datetime, history_line) + + handler = Utils.openCdf(self.local_file) + try: + history_line = history_line + handler.history + except AttributeError: + history_line = history_line + handler.history = Utils.convert_to_ASCII_if_possible(history_line) + handler.close() + + +class UnitConversion(object): + """ + Class to manage unit conversions + """ + _dict_conversions = None + + @classmethod + def load_conversions(cls): + """ + Load conversions from the configuration file + """ + cls._dict_conversions = dict() + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'rb') as csvfile: + reader = csv.reader(csvfile, dialect='excel') + for line in reader: + if line[0] == 'original': + continue + cls.add_conversion(UnitConversion(line[0], line[1], line[2], line[3])) + + @classmethod + def add_conversion(cls, conversion): + """ + Adds a conversion to the dictionary + + :param conversion: conversion to add + :type conversion: UnitConversion + """ + cls._dict_conversions[(conversion.source, conversion.destiny)] = conversion + + def __init__(self, source, destiny, factor, offset): + self.source = source + self.destiny = destiny + self.factor = float(factor) + self.offset = float(offset) + + @classmethod + def get_conversion_factor_offset(cls, input_units, output_units): + """ + Gets the conversion factor and offset for two units . The conversion has to be done in the following way: + converted = original * factor + offset + + :param input_units: original units + :type input_units: str + :param output_units: destiny units + :type output_units: str + :return: factor and offset + :rtype: [float, float] + """ + units = input_units.split() + if len(units) == 1: + scale_unit = 1 + unit = units[0] + else: + if '^' in units[0]: + values = units[0].split('^') + scale_unit = pow(int(values[0]), int(values[1])) + else: + scale_unit = float(units[0]) + unit = units[1] + + units = output_units.split() + if len(units) == 1: + scale_new_unit = 1 + new_unit = units[0] + else: + if '^' in units[0]: + values = units[0].split('^') + scale_new_unit = pow(int(values[0]), int(values[1])) + else: + scale_new_unit = float(units[0]) + new_unit = units[1] + + factor, offset = UnitConversion._get_factor(new_unit, unit) + if factor is None: + return None, None + factor = factor * scale_unit / float(scale_new_unit) + offset /= float(scale_new_unit) + + return factor, offset + + @classmethod + def _get_factor(cls, new_unit, unit): + # Add only the conversions with a factor greater than 1 + if unit == new_unit: + return 1, 0 + elif (unit, new_unit) in cls._dict_conversions: + conversion = cls._dict_conversions[(unit, new_unit)] + return conversion.factor, conversion.offset + elif (new_unit, unit) in cls._dict_conversions: + conversion = cls._dict_conversions[(new_unit, unit)] + return 1 / conversion.factor, -conversion.offset + else: + return None, None + + +class NetCDFFile(DataFile): + + def download(self): + try: + self.local_status = LocalStatus.DOWNLOADING + Log.debug('Downloading file {0}...', self.remote_file) + if not self.local_file: + self.local_file = TempFile.get() + Utils.get_file_hash(self.remote_file, use_stored=True, save=True) + Utils.copy_file(self.remote_file, self.local_file) + Log.info('File {0} ready!', self.remote_file) + self.local_status = LocalStatus.READY + + except Exception as ex: + if os.path.isfile(self.local_file): + os.remove(self.local_file) + Log.error('File {0} not available: {1}', self.remote_file, ex) + self.local_status = LocalStatus.FAILED + + def create_link(self): + try: + self.data_manager.create_link(self.domain, self.remote_file, self.frequency, self.final_name, + self.grid, True, self.var_type) + except Exception as ex: + Log.error('Can not create link to {1}: {0}'.format(ex, self.remote_file)) + + @property + def size(self): + if self.local_status == LocalStatus.READY: + return os.path.getsize(self.local_file) + if self.storage_status == StorageStatus.READY: + return os.path.getsize(self.remote_file) + return None + + diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py new file mode 100644 index 0000000000000000000000000000000000000000..e3799d31ef13dc1ac1aaf7396ee3cbb1e8914f6d --- /dev/null +++ b/earthdiagnostics/datamanager.py @@ -0,0 +1,285 @@ +# coding: utf-8 +import csv +import os +import re +import shutil +import threading + +from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils +from earthdiagnostics.variable_type import VariableType + + +class DataManager(object): + """ + Class to manage the data repositories + + :param config: + :type config: Config + """ + def __init__(self, config): + self.config = config + self.experiment = config.experiment + self._checked_vars = list() + self.variable_list = config.var_manager + UnitConversion.load_conversions() + self.lock = threading.Lock() + self.requested_files = {} + + def _get_file_from_storage(self, filepath): + if filepath not in self.requested_files: + self.requested_files[filepath] = NCfile.from_storage(filepath) + file_object = self.requested_files[filepath] + file_object.local_satatus = LocalStatus.PENDING + return self.requested_files[filepath] + + def _declare_generated_file(self, remote_file, domain, final_var, cmor_var, data_convention, + region, diagnostic, grid, var_type, original_var): + if remote_file not in self.requested_files: + self.requested_files[remote_file] = NCfile.to_storage(remote_file) + file_object = self.requested_files[remote_file] + file_object.diagnostic = diagnostic + file_object.var_type = var_type + file_object.grid = grid + file_object.data_manager = self + file_object.domain = domain + file_object.var = original_var + file_object.final_name = final_var + file_object.cmor_var = cmor_var + file_object.region = region + file_object.data_convention = data_convention + file_object.storage_status = StorageStatus.PENDING + return file_object + + @staticmethod + def _get_final_var_name(box, var): + if box: + var += box.get_lon_str() + box.get_lat_str() + box.get_depth_str() + return var + + def get_varfolder(self, domain, var, grid=None): + if grid: + var = '{0}-{1}'.format(var, grid) + + if domain in [ModelingRealms.ocean, ModelingRealms.seaIce, ModelingRealms.ocnBgchem]: + return '{0}_f{1}h'.format(var, self.experiment.ocean_timestep) + else: + return '{0}_f{1}h'.format(var, self.experiment.atmos_timestep) + + def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + freq_str = frequency.folder_name(vartype) + + if not grid: + grid = 'original' + + variable_folder = self.get_varfolder(domain, var) + vargrid_folder = self.get_varfolder(domain, var, grid) + + self.lock.acquire() + try: + if grid == 'original': + link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) + if os.path.islink(link_path): + link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) + + Utils.create_folder_tree(link_path) + else: + link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) + Utils.create_folder_tree(link_path) + default_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) + original_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, + vargrid_folder.replace('-{0}_f'.format(grid), '-original_f')) + + if os.path.islink(default_path): + os.remove(default_path) + elif os.path.isdir(default_path): + shutil.move(default_path, original_path) + os.symlink(link_path, default_path) + + if move_old and link_path not in self._checked_vars: + self._checked_vars.append(link_path) + old_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, + 'old_{0}_f{1}h'.format(var, self.experiment.atmos_timestep)) + regex = re.compile(var + '_[0-9]{6,8}\.nc') + for filename in os.listdir(link_path): + if regex.match(filename): + Utils.create_folder_tree(old_path) + Utils.move_file(os.path.join(link_path, filename), + os.path.join(old_path, filename)) + + link_path = os.path.join(link_path, os.path.basename(filepath)) + if os.path.lexists(link_path): + os.remove(link_path) + if not os.path.exists(filepath): + raise ValueError('Original file {0} does not exists'.format(filepath)) + if not os.path.isdir(os.path.dirname(link_path)): + Utils.create_folder_tree(os.path.dirname(link_path)) + relative_path = os.path.relpath(filepath, os.path.dirname(link_path)) + os.symlink(relative_path, link_path) + except Exception: + raise + finally: + self.lock.release() + + # Overridable methods (not mandatory) + def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, + frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): + """ + Creates the link of a given file from the CMOR repository. + + :param cmor_var: + :param move_old: + :param date_str: + :param year: if frequency is yearly, this parameter is used to give the corresponding year + :type year: int + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + pass + + def prepare(self): + """ + Prepares the data to be used by the diagnostic. + :return: + """ + pass + + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :rtype: str + """ + raise NotImplementedError('Class must override request_chunk method') + + +class UnitConversion(object): + """ + Class to manage unit conversions + """ + _dict_conversions = None + + @classmethod + def load_conversions(cls): + """ + Load conversions from the configuration file + """ + cls._dict_conversions = dict() + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'rb') as csvfile: + reader = csv.reader(csvfile, dialect='excel') + for line in reader: + if line[0] == 'original': + continue + cls.add_conversion(UnitConversion(line[0], line[1], line[2], line[3])) + + @classmethod + def add_conversion(cls, conversion): + """ + Adds a conversion to the dictionary + + :param conversion: conversion to add + :type conversion: UnitConversion + """ + cls._dict_conversions[(conversion.source, conversion.destiny)] = conversion + + def __init__(self, source, destiny, factor, offset): + self.source = source + self.destiny = destiny + self.factor = float(factor) + self.offset = float(offset) + + @classmethod + def get_conversion_factor_offset(cls, input_units, output_units): + """ + Gets the conversion factor and offset for two units . The conversion has to be done in the following way: + converted = original * factor + offset + + :param input_units: original units + :type input_units: str + :param output_units: destiny units + :type output_units: str + :return: factor and offset + :rtype: [float, float] + """ + units = input_units.split() + if len(units) == 1: + scale_unit = 1 + unit = units[0] + else: + if '^' in units[0]: + values = units[0].split('^') + scale_unit = pow(int(values[0]), int(values[1])) + else: + scale_unit = float(units[0]) + unit = units[1] + + units = output_units.split() + if len(units) == 1: + scale_new_unit = 1 + new_unit = units[0] + else: + if '^' in units[0]: + values = units[0].split('^') + scale_new_unit = pow(int(values[0]), int(values[1])) + else: + scale_new_unit = float(units[0]) + new_unit = units[1] + + factor, offset = UnitConversion._get_factor(new_unit, unit) + if factor is None: + return None, None + factor = factor * scale_unit / float(scale_new_unit) + offset /= float(scale_new_unit) + + return factor, offset + + @classmethod + def _get_factor(cls, new_unit, unit): + # Add only the conversions with a factor greater than 1 + if unit == new_unit: + return 1, 0 + elif (unit, new_unit) in cls._dict_conversions: + conversion = cls._dict_conversions[(unit, new_unit)] + return conversion.factor, conversion.offset + elif (new_unit, unit) in cls._dict_conversions: + conversion = cls._dict_conversions[(new_unit, unit)] + return 1 / conversion.factor, -conversion.offset + else: + return None, None + diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py new file mode 100644 index 0000000000000000000000000000000000000000..4e90373dbbd6e79389ad0d9210c2751bbac1956e --- /dev/null +++ b/earthdiagnostics/diagnostic.py @@ -0,0 +1,424 @@ +# coding=utf-8 +import datetime + +from datafile import StorageStatus, LocalStatus +from earthdiagnostics.constants import Basins, Basin +from earthdiagnostics.frequency import Frequency +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.variable_type import VariableType +from publisher import Publisher + + +class DiagnosticStatus(object): + WAITING = 0 + READY = 1 + RUNNING = 2 + COMPLETED = 3 + FAILED = 4 + + +class Diagnostic(Publisher): + """ + Base class for the diagnostics. Provides a common interface for them and also + has a mechanism that allows diagnostic retrieval by name. + + :param data_manager: data manager that will be used to store and retrieve the necessary data + :type data_manager: DataManager + """ + + alias = None + """ + Alias to call the diagnostic. Must be overridden at the derived clases + """ + _diag_list = dict() + + def __init__(self, data_manager): + super(Diagnostic, self).__init__() + self._generated_files = [] + self.data_manager = data_manager + self._status = DiagnosticStatus.WAITING + self._requests = [] + self.consumed_time = datetime.timedelta() + self.subjobs = [] + + def __repr__(self): + return str(self) + + @property + def status(self): + return self._status + + @status.setter + def status(self, value): + if self._status == value: + return + self._status = value + if self.status == DiagnosticStatus.RUNNING: + for generated_file in self._generated_files: + generated_file.local_status = LocalStatus.COMPUTING + if self.status in (DiagnosticStatus.FAILED, DiagnosticStatus.COMPLETED): + self._unsuscribe_requests() + self.dispatch(self) + + @staticmethod + def register(cls): + """ + Register a new diagnostic using the given alias. It must be call using the derived class. + :param cls: diagnostic class to register + :type cls: Type[Diagnostic] + """ + if not issubclass(cls, Diagnostic): + raise ValueError('Class {0} must be derived from Diagnostic'.format(cls)) + if cls.alias is None: + raise ValueError('Diagnostic class {0} must have defined an alias'.format(cls)) + Diagnostic._diag_list[cls.alias] = cls + + # noinspection PyProtectedMember + @staticmethod + def get_diagnostic(name): + """ + Return the class for a diagnostic given its name + + :param name: diagnostic alias + :type name: str + :return: the selected Diagnostic class, None if name can not be found + :rtype: Diagnostic + """ + if name in Diagnostic._diag_list.keys(): + return Diagnostic._diag_list[name] + return None + + def compute(self): + """ + Calculates the diagnostic and stores the output + + Must be implemented by derived classes + """ + raise NotImplementedError("Class must override compute method") + + def request_data(self): + """ + Calculates the diagnostic and stores the output + + Must be implemented by derived classes + """ + raise NotImplementedError("Class must override request_data method") + + def declare_data_generated(self): + """ + Calculates the diagnostic and stores the output + + Must be implemented by derived classes + """ + raise NotImplementedError("Class must override declare_data_generated method") + + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + + :param domain: + :type domain: ModelingRealm + :param var: + :param startdate: + :param member: + :param chunk: + :param grid: + :param region: + :param box: + :param frequency: + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: datafile object + :rtype: earthdiagnostics.datafile.DataFile + """ + if isinstance(region, Basin): + region = region.name + generated_chunk = self.data_manager.declare_chunk(domain, var, startdate, member, chunk, grid, region, box, + diagnostic=self, vartype=vartype, frequency=frequency) + self._generated_files.append(generated_chunk) + return generated_chunk + + def declare_year(self, domain, var, startdate, member, year, grid=None, box=None, + vartype=VariableType.MEAN): + """ + + :param domain: + :type domain: ModelingRealm + :param var: + :param startdate: + :param member: + :param grid: + :param box: + :param year: + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: datafile object + :rtype: DataFile + """ + generated_year = self.data_manager.declare_year(domain, var, startdate, member, year, grid, box, + diagnostic=self, vartype=vartype) + self._generated_files.append(generated_year) + return generated_year + + @classmethod + def generate_jobs(cls, diags, options): + """ + Generate the instances of the diagnostics that will be run by the manager + + Must be implemented by derived classes. + + :param diags: diagnostics manager + :type diags: Diags + :param options: list of strings containing the options passed to the diagnostic + :type options: list[str] + :return: + """ + raise NotImplementedError("Class must override generate_jobs class method") + + @classmethod + def process_options(cls, options, options_available): + processed = dict() + options = options[1:] + if len(options) > len(options_available): + raise DiagnosticOptionError('You have specified more options than available for diagnostic ' + '{0}'.format(cls.alias)) + for x in range(len(options_available)): + option_definition = options_available[x] + if len(options) <= x: + option_value = '' + else: + option_value = options[x] + processed[option_definition.name] = option_definition.parse(option_value) + return processed + + def __str__(self): + """ + Must be implemented by derived classes + :return: + """ + return 'Developer must override base class __str__ method' + + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + to_modify=False, vartype=VariableType.MEAN): + request = self.data_manager.request_chunk(domain, var, startdate, member, chunk, grid, box, frequency, vartype) + if to_modify: + request.add_modifier(self) + self._requests.append(request) + request.subscribe(self, self._updated_request) + return request + + def request_year(self, domain, var, startdate, member, year, grid=None, box=None, frequency=None, to_modify=False): + request = self.data_manager.request_year(self, domain, var, startdate, member, year, grid, box, frequency) + if to_modify: + request.add_modifier(self) + self._requests.append(request) + request.subscribe(self, self._updated_request) + return request + + def _updated_request(self, request): + if self.status != DiagnosticStatus.WAITING: + return + from datafile import LocalStatus + if request.local_status == LocalStatus.FAILED: + self.message = 'Required file {0} is not available'.format(request.remote_file) + self.status = DiagnosticStatus.FAILED + return + + if request.local_status == LocalStatus.READY: + self.check_is_ready() + + def check_is_ready(self): + if all([request.ready_to_run(self) for request in self._requests]): + self.status = DiagnosticStatus.READY + + def _unsuscribe_requests(self): + for request in self._requests: + request.unsubscribe(self) + + def all_requests_in_storage(self): + return self.pending_requests() == 0 + + def pending_requests(self): + return len([request.storage_status != StorageStatus.READY for request in self._requests]) + +class DiagnosticOption(object): + + def __init__(self, name, default_value=None): + self.name = name + self.default_value = default_value + + def parse(self, option_value): + option_value = self.check_default(option_value) + return option_value + + def check_default(self, option_value): + if option_value == '': + if self.default_value is None: + raise DiagnosticOptionError('Option {0} is not optional'.format(self.name)) + else: + return self.default_value + return option_value + + +class DiagnosticFloatOption(DiagnosticOption): + def parse(self, option_value): + return float(self.check_default(option_value)) + + +class DiagnosticIntOption(DiagnosticOption): + + def __init__(self, name, default_value=None, min_limit=None, max_limit=None): + super(DiagnosticIntOption, self).__init__(name, default_value) + self.min_limit = min_limit + self.max_limit = max_limit + + def parse(self, option_value): + value = int(self.check_default(option_value)) + if self.min_limit is not None and value < self.min_limit: + raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) + if self.max_limit is not None and value > self.max_limit: + raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) + return value + + +class DiagnosticListIntOption(DiagnosticOption): + """ + :param name: + :type name: str + :param default_value: + :type default_value: int|NoneType + :param min_limit: + :type min_limit: int|NoneType + :param max_limit: + :type max_limit: int|NoneType + """ + + def __init__(self, name, default_value=None, min_limit=None, max_limit=None): + super(DiagnosticListIntOption, self).__init__(name, default_value) + self.min_limit = min_limit + """ Lower limit """ + self.max_limit = max_limit + """ Upper limit """ + + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, tuple) or isinstance(option_value, list): + return option_value + values = [int(i) for i in option_value.split('-')] + for value in values: + # noinspection PyTypeChecker + if self.min_limit is not None and value < self.min_limit: + raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) + # noinspection PyTypeChecker + if self.max_limit is not None and value > self.max_limit: + raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) + + return values + + +class DiagnosticListFrequenciesOption(DiagnosticOption): + + def __init__(self, name, default_value=None): + super(DiagnosticListFrequenciesOption, self).__init__(name, default_value) + + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, tuple) or isinstance(option_value, list): + return option_value + values = [Frequency(i) for i in option_value.split('-')] + return values + + +class DiagnosticVariableOption(DiagnosticOption): + def __init__(self, var_manager, name='variable', default_value=None): + super(DiagnosticVariableOption, self).__init__(name, default_value) + self.var_manager = var_manager + + def parse(self, option_value): + option_value = self.check_default(option_value) + real_name = self.var_manager.get_variable(option_value, False) + if real_name is None: + return option_value + return real_name.short_name + + +class DiagnosticVariableListOption(DiagnosticOption): + + def __init__(self, var_manager, name, default_value=None): + super(DiagnosticVariableListOption, self).__init__(name, default_value) + self.var_manager = var_manager + + def parse(self, option_value): + option_value = self.check_default(option_value) + var_names = [] + for value in option_value.split('-'): + real_name = self.var_manager.get_variable(value, False) + if real_name is None: + var_names.append(value) + else: + var_names.append(real_name.short_name) + return var_names + + +class DiagnosticDomainOption(DiagnosticOption): + def __init__(self, name='domain', default_value=None): + super(DiagnosticDomainOption, self).__init__(name, default_value) + + def parse(self, option_value): + return ModelingRealms.parse(self.check_default(option_value)) + + +class DiagnosticFrequencyOption(DiagnosticOption): + def __init__(self, name='frequency', default_value=None): + super(DiagnosticFrequencyOption, self).__init__(name, default_value) + + def parse(self, option_value): + return Frequency.parse(self.check_default(option_value)) + + +class DiagnosticBasinOption(DiagnosticOption): + def parse(self, option_value): + return Basins().parse(self.check_default(option_value)) + + +class DiagnosticComplexStrOption(DiagnosticOption): + def parse(self, option_value): + return self.check_default(option_value).replace('&;', ',').replace('&.', ' ') + + +class DiagnosticBoolOption(DiagnosticOption): + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, bool): + return option_value + else: + return option_value.lower() in ('true', 't', 'yes') + + +class DiagnosticChoiceOption(DiagnosticOption): + def __init__(self, name, choices, default_value=None, ignore_case=True): + super(DiagnosticChoiceOption, self).__init__(name, default_value) + self.choices = choices + self.ignore_case = ignore_case + + # To check if it is valid + if default_value is not None: + self.parse(default_value) + + def parse(self, value): + value = self.check_default(value) + if self.ignore_case: + value = value.lower() + for choice in self.choices: + if value == choice.lower(): + return choice + else: + if value in self.choices: + return value + raise DiagnosticOptionError('Value {1} in option {0} is not a valid choice. ' + 'Options are {2}'.format(self.name, value, self.choices)) + + +class DiagnosticOptionError(Exception): + pass diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py new file mode 100755 index 0000000000000000000000000000000000000000..b9ef5aa2935518a28a7094b32e64be73afc7c14e --- /dev/null +++ b/earthdiagnostics/earthdiags.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python +# coding=utf-8 +import argparse +import os +import shutil +import tempfile +from distutils.spawn import find_executable + +import bscearth.utils.path +import netCDF4 +import pkg_resources +from bscearth.utils.date import * + +from earthdiagnostics import cdftools +from earthdiagnostics.cmormanager import CMORManager +from earthdiagnostics.config import Config +from earthdiagnostics.constants import Basins +from earthdiagnostics.obsreconmanager import ObsReconManager +from earthdiagnostics.threddsmanager import THREDDSManager +from earthdiagnostics.utils import TempFile, Utils +from work_manager import WorkManager + + +class EarthDiags(object): + """ + Launcher class for the diagnostics + + :param config_file: path to the configuration file + :type config_file: str + """ + # Get the version number from the relevant file. If not, from autosubmit package + scriptdir = os.path.abspath(os.path.dirname(__file__)) + if not os.path.exists(os.path.join(scriptdir, 'VERSION')): + scriptdir = os.path.join(scriptdir, os.path.pardir) + + version_path = os.path.join(scriptdir, 'VERSION') + readme_path = os.path.join(scriptdir, 'README') + changes_path = os.path.join(scriptdir, 'CHANGELOG') + documentation_path = os.path.join(scriptdir, 'EarthDiagnostics.pdf') + if os.path.isfile(version_path): + with open(version_path) as f: + version = f.read().strip() + else: + version = pkg_resources.require("earthdiagnostics")[0].version + + def __init__(self, config_file): + Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) + self.config = Config(config_file) + + TempFile.scratch_folder = self.config.scratch_dir + cdftools.path = self.config.cdftools_path + self._create_dic_variables() + self.time = dict() + self.data_manager = None + self.threads = None + self.had_errors = False + Log.debug('Diags ready') + Log.info('Running diags for experiment {0}, startdates {1}, members {2}', self.config.experiment.expid, + self.config.experiment.startdates, self.config.experiment.members) + + @staticmethod + def parse_args(): + """ + Entry point for the Earth Diagnostics. For more detailed documentation, use -h option + """ + # try: + parser = argparse.ArgumentParser(description='Main executable for Earth Diagnostics.') + parser.add_argument('-v', '--version', action='version', version=EarthDiags.version, + help="returns Earth Diagnostics's version number and exit") + parser.add_argument('--doc', action='store_true', + help="opens documentation and exits") + parser.add_argument('--clean', action='store_true', + help="clean the scratch folder and exits") + parser.add_argument('--report', action='store_true', + help="generates a report about the available files") + parser.add_argument('-lf', '--logfile', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', + 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'), + default='DEBUG', type=str, + help="sets file's log level.") + parser.add_argument('-lc', '--logconsole', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', + 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'), + default='INFO', type=str, + help="sets console's log level") + + parser.add_argument('-log', '--logfilepath', default=None, type=str) + + parser.add_argument('-f', '--configfile', default='diags.conf', type=str) + + args = parser.parse_args() + if args.doc: + Log.info('Opening documentation...') + doc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'EarthDiagnostics.pdf') + Utils.execute_shell_command(('xdg-open', doc_path)) + Log.result('Documentation opened!') + return True + Log.set_console_level(args.logconsole) + Log.set_file_level(args.logfile) + + if Log.console_handler.level <= Log.DEBUG: + Utils.cdo.debug = True + Utils.nco.debug = False # This is due to a bug in nco. Must change when it's solved + + Utils.cdo.CDO = find_executable('cdo') + + if args.logfilepath: + Log.set_file(bscearth.utils.path.expand_path(args.logfilepath)) + + config_file_path = bscearth.utils.path.expand_path(args.configfile) + if not os.path.isfile(config_file_path): + Log.critical('Configuration file {0} can not be found', config_file_path) + return False + try: + diags = EarthDiags(config_file_path) + if args.clean: + result = diags.clean() + elif args.report: + result = diags.report() + else: + result = diags.run() + except Exception: + raise + finally: + TempFile.clean() + + return result + + def _create_dic_variables(self): + self.dic_variables = dict() + self.dic_variables['x'] = 'i' + self.dic_variables['y'] = 'j' + self.dic_variables['z'] = 'lev' + self.dic_variables['nav_lon'] = 'lon' + self.dic_variables['nav_lat'] = 'lat' + self.dic_variables['nav_lev'] = 'lev' + self.dic_variables['time_counter'] = 'time' + self.dic_variables['t'] = 'time' + + def run(self): + """ + Run the diagnostics + """ + self.had_errors = False + Log.debug('Using netCDF version {0}', netCDF4.getlibversion()) + + self._prepare_scratch_dir() + + self._prepare_mesh_files() + self._initialize_basins() + + self._prepare_data_manager() + + # Run diagnostics + Log.info('Running diagnostics') + + work_manager = WorkManager(self.config, self.data_manager) + work_manager.prepare_job_list() + + result = work_manager.run() + + if self.config.auto_clean: + self._remove_scratch_dir() + return result + + def _initialize_basins(self): + self._read_basins_from_file('mask_regions.nc') + self._read_basins_from_file('mask_regions.3d.nc') + + @staticmethod + def _read_basins_from_file(filename): + if not os.path.isfile(filename): + return + handler = Utils.openCdf(filename) + Basins().get_available_basins(handler) + handler.close() + + def _prepare_scratch_dir(self): + if self.config.use_ramdisk: + self._remove_scratch_dir() + tempfile.mkdtemp(dir='/dev/shm') + os.symlink(tempfile.mkdtemp(dir='/dev/shm'), self.config.scratch_dir) + else: + if not os.path.exists(self.config.scratch_dir): + os.makedirs(self.config.scratch_dir) + os.chdir(self.config.scratch_dir) + + def _prepare_data_manager(self): + if self.config.data_adaptor == 'CMOR': + self.data_manager = CMORManager(self.config) + elif self.config.data_adaptor == 'THREDDS': + self.data_manager = THREDDSManager(self.config) + elif self.config.data_adaptor == 'OBSRECON': + self.data_manager = ObsReconManager(self.config) + self.data_manager.prepare() + + def clean(self): + Log.info('Removing scratch folder...') + self._remove_scratch_dir() + Log.result('Scratch folder removed') + return True + + def _remove_scratch_dir(self): + if os.path.islink(self.config.scratch_dir): + shutil.rmtree(os.path.realpath(self.config.scratch_dir)) + os.remove(self.config.scratch_dir) + elif os.path.isdir(self.config.scratch_dir): + shutil.rmtree(self.config.scratch_dir) + + def report(self): + Log.info('Looking for existing vars...') + self._prepare_data_manager() + base_folder = self.config.report.path + if not base_folder: + base_folder = self.config.scratch_dir + Utils.create_folder_tree(base_folder) + for startdate in self.config.experiment.startdates: + for member in self.config.experiment.members: + results = self._get_variable_report(startdate, member) + + report_path = os.path.join(base_folder, + '{0}_{1}.report'.format(startdate, + self.config.experiment.get_member_str(member))) + + self.create_report(report_path, results) + + Log.result('Report finished') + return True + + def _get_variable_report(self, startdate, member): + var_manager = self.config.var_manager + results = list() + for var in var_manager.get_all_variables(): + if var.domain is None: + continue + for table, priority in var.tables: + if priority is None or priority > self.config.report.maximum_priority: + continue + if not self.data_manager.file_exists(var.domain, var.short_name, startdate, member, 1, + frequency=table.frequency): + results.append((var, table, priority)) + Log.debug('Variable {0.short_name} not found in {1.name}', var, table) + else: + Log.result('Variable {0.short_name} found in {1.name}', var, table) + + return results + + @staticmethod + def create_report(report_path, results): + tables = set([result[1].name for result in results]) + for table in tables: + file_handler = open('{0}.{1}'.format(report_path, table), 'w') + table_results = [result for result in results if result[1].name == table] + + file_handler.write('\nTable {0}\n'.format(table)) + file_handler.write('===================================\n') + + priorities = set([result[2] for result in table_results]) + priorities = sorted(priorities) + for priority in priorities: + priority_results = [result[0] for result in table_results if result[2] == priority] + priority_results = sorted(priority_results, key=lambda v: v.short_name) + file_handler.write('\nMissing variables with priority {0}:\n'.format(priority)) + file_handler.write('--------------------------------------\n') + + for var in priority_results: + file_handler.write('{0:12}: {1}\n'.format(var.short_name, var.standard_name)) + file_handler.flush() + file_handler.close() + + def _prepare_mesh_files(self): + model_version = self.config.experiment.model_version + if not model_version: + Log.info('No model version defined. Skipping mesh files copy!') + return + Log.info('Copying mesh files') + con_files = self.config.con_files + model_version = self.config.experiment.model_version + restore_meshes = self.config.restore_meshes + + mesh_mask = 'mesh_mask_nemo.{0}.nc'.format(model_version) + new_mask_glo = 'new_maskglo.{0}.nc'.format(model_version) + mask_regions = 'mask.regions.{0}.nc'.format(model_version) + mask_regions_3d = 'mask.regions.3d.{0}.nc'.format(model_version) + + if self.config.mesh_mask: + mesh_mask_path = self.config.mesh_mask + else: + mesh_mask_path = os.path.join(con_files, mesh_mask) + + if self.config.new_mask_glo: + new_mask_glo_path = self.config.new_mask_glo + else: + new_mask_glo_path = os.path.join(con_files, new_mask_glo) + + if self.config.mask_regions: + mask_regions_path = self.config.mask_regions + else: + mask_regions_path = os.path.join(con_files, mask_regions) + + if self.config.mask_regions_3d: + mask_regions_3d_path = self.config.mask_regions_3d + else: + mask_regions_3d_path = os.path.join(con_files, mask_regions_3d) + + if self.config.scratch_masks: + Utils.create_folder_tree(self.config.scratch_masks) + Utils.give_group_write_permissions(self.config.scratch_masks) + + mesh_mask_scratch_path = os.path.join(self.config.scratch_masks, mesh_mask) + + if self._copy_file(mesh_mask_path, mesh_mask_scratch_path, + restore_meshes): + Utils.give_group_write_permissions(mesh_mask_scratch_path) + self._link_file(mesh_mask_scratch_path, 'mesh_hgr.nc') + self._link_file(mesh_mask_scratch_path, 'mesh_zgr.nc') + self._link_file(mesh_mask_scratch_path, 'mask.nc') + + new_maskglo_scratch_path = os.path.join(self.config.scratch_masks, new_mask_glo) + if self._copy_file(new_mask_glo_path, + new_maskglo_scratch_path, restore_meshes): + Utils.give_group_write_permissions(new_maskglo_scratch_path) + self._link_file(new_maskglo_scratch_path, 'new_maskglo.nc') + + mask_regions_scratch_path = os.path.join(self.config.scratch_masks, mask_regions) + if self._copy_file(mask_regions_path, + mask_regions_scratch_path, restore_meshes): + Utils.give_group_write_permissions(mask_regions_scratch_path) + self._link_file(mask_regions_scratch_path, 'mask_regions.nc') + + mask_regions3d_scratch_path = os.path.join(self.config.scratch_masks, mask_regions_3d) + if self._copy_file(mask_regions_3d_path, + mask_regions3d_scratch_path, restore_meshes): + Utils.give_group_write_permissions(mask_regions3d_scratch_path) + self._link_file(mask_regions3d_scratch_path, 'mask_regions.3d.nc') + else: + self._copy_file(mesh_mask_path, 'mesh_hgr.nc', restore_meshes) + self._link_file('mesh_hgr.nc', 'mesh_zgr.nc') + self._link_file('mesh_hgr.nc', 'mask.nc') + self._copy_file(new_mask_glo_path, 'new_maskglo.nc', + restore_meshes) + self._copy_file(mask_regions_path, + 'mask_regions.nc', restore_meshes) + self._copy_file(mask_regions_3d_path, + 'mask_regions.3d.nc', restore_meshes) + + Log.result('Mesh files ready!') + + def _copy_file(self, source, destiny, force): + if not os.path.exists(source): + Log.user_warning('File {0} is not available for {1}', destiny, self.config.experiment.model_version) + return False + + if not force and os.path.exists(destiny): + # Small size differences can be due to the renaming of variables + delta_size = abs(os.stat(source).st_size - os.stat(destiny).st_size) + if delta_size < 512: + Log.info('File {0} already exists', destiny) + return True + + Log.info('Copying file {0}', destiny) + shutil.copyfile(source, destiny) + Log.info('File {0} ready', destiny) + Utils.rename_variables(destiny, self.dic_variables, False, True) + return True + + def _link_file(self, source, destiny): + if not os.path.exists(source): + Log.user_warning('File {0} is not available for {1}', destiny, self.config.experiment.model_version) + return + + if os.path.lexists(destiny): + try: + os.remove(destiny) + except OSError: + pass + + os.symlink(source, destiny) + Log.info('File {0} ready', destiny) + + +def main(): + if not EarthDiags.parse_args(): + exit(1) + + +if __name__ == "__main__": + main() diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py new file mode 100644 index 0000000000000000000000000000000000000000..09ebe3de5d3b3e972cadcd60b9ff68308c1e81fc --- /dev/null +++ b/earthdiagnostics/frequency.py @@ -0,0 +1,63 @@ +# coding=utf-8 +from earthdiagnostics.variable_type import VariableType + + +class Frequency(object): + + _recognized = {'f': 'fx', 'fx': 'fx', 'fixed': 'fx', + 'c': 'clim', 'clim': 'clim', 'climatology': 'clim', 'monclim': 'clim', '1hrclimmon': 'clim', + 'dec': 'dec', 'decadal': 'dec', + 'y': 'year', 'yr': 'year', 'year': 'year', 'yearly': 'year', + 'm': 'mon', '1m': 'mon', 'mon': 'mon', 'monthly': 'mon', 'mm': 'mon', + 'd': 'day', '1d': 'day', 'daily': 'day', 'day': 'day', + '15': '15hr', '15h': '15hr', '15hr': '15hr', '15_hourly': '15hr', '15hourly': '15hr', + '15 hourly': '15hr', + '6': '6hr', '6h': '6hr', '6hr': '6hr', '6_hourly': '6hr', '6hourly': '6hr', '6 hourly': '6hr', + '3': '3hr', '3h': '3hr', '3hr': '3hr', '3_hourly': '3hr', '3hourly': '3hr', '3 hourly': '3hr', + '1': '1hr', 'hr': '1hr', '1h': '1hr', 'hourly': '1hr', '1hr': '1hr', '1 hourly': '1hr', + '450mn': '450mn', + 'subhr': 'subhr'} + + def __init__(self, freq): + freq = freq.lower() + try: + self.frequency = Frequency._recognized[freq] + except KeyError: + raise ValueError('Frequency {0} not supported'.format(freq)) + + def __eq__(self, other): + return self.frequency == other.frequency + + def __str__(self): + return self.frequency + + def folder_name(self, vartype): + if self == Frequencies.daily: + freq_str = 'daily_{0}'.format(VariableType.to_str(vartype)) + elif self == Frequencies.climatology: + freq_str = 'clim' + elif self in (Frequencies.three_hourly, Frequencies.six_hourly, Frequencies.hourly): + freq_str = self.frequency[:-2] + 'hourly' + if vartype != VariableType.MEAN: + freq_str = '{0}_{1}'.format(freq_str, VariableType.to_str(vartype)) + else: + freq_str = 'monthly_{0}'.format(VariableType.to_str(vartype)) + return freq_str + + @staticmethod + def parse(freq): + if isinstance(freq, Frequency): + return freq + return Frequency(freq) + + +class Frequencies(object): + fixed = Frequency('fx') + climatology = Frequency('clim') + yearly = Frequency('year') + monthly = Frequency('mon') + daily = Frequency('day') + six_hourly = Frequency('6hr') + three_hourly = Frequency('3hr') + hourly = Frequency('hr') + subhourly = Frequency('subhr') diff --git a/earthdiagnostics/general/__init__.py b/earthdiagnostics/general/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..34820058a09b80bbd78b343158abef14094df0fb --- /dev/null +++ b/earthdiagnostics/general/__init__.py @@ -0,0 +1,13 @@ +# coding=utf-8 +from earthdiagnostics.general.monthlymean import MonthlyMean +from earthdiagnostics.general.dailymean import DailyMean +from earthdiagnostics.general.yearlymean import YearlyMean +from earthdiagnostics.general.rewrite import Rewrite +from earthdiagnostics.general.relink import Relink +from earthdiagnostics.general.scale import Scale +from earthdiagnostics.general.attribute import Attribute +from earthdiagnostics.general.relinkall import RelinkAll +from earthdiagnostics.general.simplify_dimensions import SimplifyDimensions +from earthdiagnostics.general.select_levels import SelectLevels +from earthdiagnostics.general.module import Module +from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py new file mode 100644 index 0000000000000000000000000000000000000000..1c6185787d95a7ed3fa3525f477f49ff02d930c8 --- /dev/null +++ b/earthdiagnostics/general/attribute.py @@ -0,0 +1,102 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticComplexStrOption, \ + DiagnosticDomainOption, DiagnosticVariableOption +from earthdiagnostics.utils import Utils +from earthdiagnostics.modelingrealm import ModelingRealm + + +class Attribute(Diagnostic): + """ + Rewrites files without doing any calculations. + Can be useful to convert units or to correct wrong metadata + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + """ + + alias = 'att' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, + attributte_name, attributte_value): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.grid = grid + self.attributte_name = attributte_name + self.attributte_value = attributte_value + + def __str__(self): + return 'Write attributte output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Variable: {0.domain}:{0.variable} Attributte: {0.attributte_name}:{0.attributte_value} ' \ + 'Grid: {0.grid}'.format(self) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.grid == other.grid and \ + self.attributte_name == other.attributte_name and self.attributte_value == other.attributte_value + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ + + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticOption('name'), + DiagnosticComplexStrOption('value'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Attribute(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'], options['name'], + options['value'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + variable_file = self.variable_file.local_file + handler = Utils.openCdf(variable_file) + handler.setncattr(self.attributte_name, self.attributte_value) + handler.close() + if not Utils.check_netcdf_file(variable_file): + raise Exception('Attribute {0} can not be set correctly to {1}'.format(self.attributte_name, + self.attributte_value)) + + self.corrected.set_local_file(variable_file, self) + diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py new file mode 100644 index 0000000000000000000000000000000000000000..7fb4736eba58980d40dce6f247b85f22b91850f3 --- /dev/null +++ b/earthdiagnostics/general/dailymean.py @@ -0,0 +1,109 @@ +# coding=utf-8 + +import os +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ + DiagnosticFrequencyOption, DiagnosticVariableOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealm + + +class DailyMean(Diagnostic): + """ + Calculates daily mean for a given variable + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: str + :param grid: original data grid + :type grid: str + """ + + alias = 'daymean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.frequency = frequency + self.grid = grid + + def __str__(self): + return 'Calculate daily mean Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Original frequency: {5} Grid: {6}'.format(self.startdate, self.member, self.chunk, + self.domain, self.variable, + self.frequency, self.grid) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.grid == other.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFrequencyOption(), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(DailyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + + def declare_data_generated(self): + self.daymean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.daily, grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + handler = Utils.openCdf(self.variable_file.local_file) + if 'region' in handler.variables: + noregion = TempFile.get() + Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) + Utils.cdo.daymean(input=noregion, output=temp) + os.remove(noregion) + monmean_handler = Utils.openCdf(temp) + Utils.copy_variable(handler, monmean_handler, 'region') + monmean_handler.close() + else: + Utils.cdo.daymean(input=self.variable_file.local_file, output=temp) + self.daymean.set_local_file(temp) + diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py new file mode 100644 index 0000000000000000000000000000000000000000..f72aa5f568d43f6e70fe277c528ea065d74db443 --- /dev/null +++ b/earthdiagnostics/general/module.py @@ -0,0 +1,112 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import * +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealm +import numpy as np + + +class Module(Diagnostic): + """ + Scales a variable by the given value also adding at offset + Can be useful to correct units or other known errors + (think of a tas file declaring K as units but with the data stored as Celsius) + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int : + :param domain: variable's domain + :type domain: ModelingRealm + """ + + alias = 'module' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, componentu, componentv, module_var, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.domain = domain + self.componentu = componentu + self.componentv = componentv + self.module = module_var + self.grid = grid + + self.original_values = None + + def __str__(self): + return 'Calculate module Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variables: {3}:{4},{5},{6} ' \ + 'Grid: {7}'.format(self.startdate, self.member, self.chunk, self.domain, self.componentu, + self.componentv, self.module, self.grid) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.componentu == other.componentu and \ + self.componentv == other.componentv and self.module == other.module and self.grid == other.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'componentu'), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'componentv'), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'module'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Module(diags.data_manager, startdate, member, chunk, + options['domain'], options['componentu'], options['componentv'], options['module'], + options['grid'])) + return job_list + + def request_data(self): + self.component_u_file = self.request_chunk(self.domain, self.componentu, self.startdate, self.member, + self.chunk, grid=self.grid) + self.component_v_file = self.request_chunk(self.domain, self.componentv, self.startdate, self.member, + self.chunk, grid=self.grid) + + def declare_data_generated(self): + self.module_file = self.declare_chunk(self.domain, self.module, self.startdate, self.member, self.chunk, + grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + Utils.copy_file(self.component_u_file.local_file, temp) + component_u = Utils.openCdf(temp) + component_v = Utils.openCdf(self.component_v_file.local_file) + variable_u = component_u.variables[self.componentu] + variable_v = component_v.variables[self.componentv] + + variable_u[:] = np.sqrt(variable_u[:] ** 2 + variable_v[:] ** 2) + + if 'table' in variable_u.ncattrs(): + del variable_u.table + if 'code' in variable_u.ncattrs(): + del variable_u.code + + component_u.close() + component_v.close() + + self.module_file.set_local_file(temp, rename_var=self.componentu) diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py new file mode 100644 index 0000000000000000000000000000000000000000..dca5e730af29859750df5055dc331976dfcf9df8 --- /dev/null +++ b/earthdiagnostics/general/monthlymean.py @@ -0,0 +1,111 @@ +# coding=utf-8 + +import os +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ + DiagnosticFrequencyOption, DiagnosticVariableOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealm + + +class MonthlyMean(Diagnostic): + """ + Calculates monthly mean for a given variable + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: Frequency + :param grid: original data grid + :type grid: str + """ + + alias = 'monmean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.frequency = frequency + self.grid = grid + + def __str__(self): + return 'Calculate monthly mean Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.grid == other.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFrequencyOption('frequency', Frequencies.daily), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + + def declare_data_generated(self): + self.monmean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + handler = Utils.openCdf(self.variable_file.local_file) + temp = TempFile.get() + if 'region' in handler.variables: + noregion = TempFile.get() + Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) + Utils.cdo.monmean(input=noregion, output=temp) + os.remove(noregion) + monmean_handler = Utils.openCdf(temp) + Utils.copy_variable(handler, monmean_handler, 'region') + monmean_handler.close() + else: + Utils.cdo.monmean(input=self.variable_file.local_file, output=temp) + handler.close() + self.monmean.set_local_file(temp) + + + diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py new file mode 100644 index 0000000000000000000000000000000000000000..60c69f4cf0894b38ec2df16ee2a379f37603587c --- /dev/null +++ b/earthdiagnostics/general/relink.py @@ -0,0 +1,90 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption, \ + DiagnosticVariableOption +from earthdiagnostics.modelingrealm import ModelingRealm +from earthdiagnostics.variable import VariableManager + + +class Relink(Diagnostic): + """ + Recreates the links for the variable specified + + :original author: Javier Vegas-Regidor + + :created: September 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param move_old: if true, looks for files following the old convention and moves to avoid collisions + :type move_old: bool + """ + + alias = 'relink' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, move_old, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.move_old = move_old + self.grid = grid + self.var_manager = data_manager.config.var_manager + + def __str__(self): + return 'Relink output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Move old: {0.move_old} ' \ + 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.move_old == other.move_old and \ + self.grid == other.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, move_old=False + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticBoolOption('move_old', True), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Relink(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['move_old'], options['grid'])) + return job_list + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + def compute(self): + """ + Runs the diagnostic + """ + self.data_manager.link_file(self.domain, self.variable, self.var_manager.get_variable(self.variable), + self.startdate, self.member, self.chunk, + move_old=self.move_old, grid=self.grid) + diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py new file mode 100644 index 0000000000000000000000000000000000000000..d5fffc4ae4ba02b9a23f526c3fd031139050f871 --- /dev/null +++ b/earthdiagnostics/general/relinkall.py @@ -0,0 +1,61 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic + + +class RelinkAll(Diagnostic): + """ + Recreates the links for the variable specified + + :original author: Javier Vegas-Regidor + + :created: September 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + """ + + alias = 'relinkall' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + + def __str__(self): + return 'Relink all output Startdate: {0}'.format(self.startdate) + + def __eq__(self, other): + return self.startdate == other.startdate + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, move_old=False + :type options: list[str] + :return: + """ + if len(options) > 1: + raise Exception('The Relink All diagnostic has no options') + job_list = list() + for startdate in diags.config.experiment.startdates: + job_list.append(RelinkAll(diags.data_manager, startdate)) + return job_list + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + def compute(self): + """ + Runs the diagnostic + """ + self.data_manager.create_links(self.startdate) + diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py new file mode 100644 index 0000000000000000000000000000000000000000..2aa937baa7dd988fd872047331f54a39461489f1 --- /dev/null +++ b/earthdiagnostics/general/rewrite.py @@ -0,0 +1,84 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption +from earthdiagnostics.modelingrealm import ModelingRealm + + +class Rewrite(Diagnostic): + """ + Rewrites files without doing any calculations. + Can be useful to convert units or to correct wrong metadata + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + """ + + alias = 'rewrite' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.grid = grid + + def __str__(self): + return 'Rewrite output Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, + self.grid) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.grid == self.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + self.corrected.set_local_file(self.variable_file.local_file, self) + diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py new file mode 100644 index 0000000000000000000000000000000000000000..116a978ddc632c5d7e64745f4425ef46c85da68f --- /dev/null +++ b/earthdiagnostics/general/scale.py @@ -0,0 +1,119 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import * +from earthdiagnostics.utils import Utils +from earthdiagnostics.modelingrealm import ModelingRealm +import math + + +class Scale(Diagnostic): + """ + Scales a variable by the given value also adding at offset + Can be useful to correct units or other known errors + (think of a tas file declaring K as units but with the data stored as Celsius) + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int : + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + """ + + alias = 'scale' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, value, offset, domain, variable, grid, + min_limit, max_limit, frequency): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.grid = grid + self.value = value + self.offset = offset + self.min_limit = min_limit + self.max_limit = max_limit + self.frequency = frequency + + self.original_values = None + + def __str__(self): + return 'Scale output Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Scale value: {5} Offset: {6} Variable: {3}:{4} ' \ + 'Frequency: {7}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, + self.value, self.offset, self.frequency) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFloatOption('value'), + DiagnosticFloatOption('offset'), + DiagnosticOption('grid', ''), + DiagnosticFloatOption('min_limit', float('nan')), + DiagnosticFloatOption('max_limit', float('nan')), + DiagnosticListFrequenciesOption('frequencies', [diags.config.frequency])) + options = cls.process_options(options, options_available) + job_list = list() + for frequency in options['frequencies']: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Scale(diags.data_manager, startdate, member, chunk, + options['value'], options['offset'], options['domain'], options['variable'], + options['grid'], options['min_limit'], options['max_limit'], frequency)) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, frequency=self.frequency, to_modify=True) + + def declare_data_generated(self): + self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, frequency=self.frequency) + + def compute(self): + """ + Runs the diagnostic + """ + variable_file = self.variable_file.local_file + + handler = Utils.openCdf(variable_file) + var_handler = handler.variables[self.variable] + self.original_values = var_handler[:] + if self._check_limits(): + var_handler[:] = self.original_values * self.value + self.offset + handler.close() + self.corrected.set_local_file(self.variable_file.local_file, self) + + def _check_limits(self): + if not math.isnan(self.min_limit) and (self.original_values.min() < self.min_limit): + return False + if not math.isnan(self.max_limit) and (self.original_values.max() > self.max_limit): + return False + return True + + diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py new file mode 100644 index 0000000000000000000000000000000000000000..1d2fb9cac33b0ebed3d5c3daf8fa3ab5b287b7bf --- /dev/null +++ b/earthdiagnostics/general/select_levels.py @@ -0,0 +1,121 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ + DiagnosticVariableListOption, DiagnosticIntOption +from earthdiagnostics.modelingrealm import ModelingRealm +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.box import Box + + +class SelectLevels(Diagnostic): + """ + Convert i j files to lon lat when there is no interpolation required, + i.e. lon is constant over i and lat is constat over j + + :original author: Javier Vegas-Regidor + + :created: April 2017 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + """ + + alias = 'selev' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, first_level, last_level): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.grid = grid + self.box = Box(False) + self.box.min_depth = first_level + self.box.max_depth = last_level + + def __str__(self): + return 'Select levels Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Levels: {6}-{7} ' \ + 'Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, + self.grid, self.box.min_depth, self.box.max_depth) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.grid == self.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain,variables,grid + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), + DiagnosticIntOption('first_level'), + DiagnosticIntOption('last_level'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + variables = options['variables'] + for var in variables: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + + job_list.append(SelectLevels(diags.data_manager, startdate, member, chunk, + options['domain'], var, options['grid'], + options['first_level'], options['last_level'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + self.result = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + + Utils.nco.ncks(input=self.variable_file, output=temp, + options=('-O -d lev,{0.min_depth},{0.max_depth}'.format(self.box),)) + self.result.set_local_file(temp) + + @staticmethod + def _create_var(var_name, var_values, source, destiny): + old_var = source.variables[var_name] + new_var = destiny.createVariable(var_name, old_var.dtype, dimensions=(var_name, )) + new_var[:] = var_values + Utils.copy_attributes(new_var, old_var) + + vertices_name = '{0}_vertices'.format(var_name) + + if vertices_name in source.variables: + var_vertices = source.variables[vertices_name] + if var_name == 'lon': + vertices_values = var_vertices[0:1, ...] + else: + vertices_values = var_vertices[:, 0:1, :] + new_lat_vertices = destiny.createVariable(vertices_name, var_vertices.dtype, + dimensions=(var_name, 'vertices')) + new_lat_vertices[:] = vertices_values + Utils.copy_attributes(new_lat_vertices, var_vertices) + diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py new file mode 100644 index 0000000000000000000000000000000000000000..579a5473faf0dcc0804ba30c361688de6b03c535 --- /dev/null +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -0,0 +1,148 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ + DiagnosticVariableListOption +from earthdiagnostics.modelingrealm import ModelingRealm +from earthdiagnostics.utils import Utils, TempFile +import numpy as np + + +class SimplifyDimensions(Diagnostic): + """ + Convert i j files to lon lat when there is no interpolation required, + i.e. lon is constant over i and lat is constat over j + + :original author: Javier Vegas-Regidor + + :created: April 2017 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + """ + + alias = 'simdim' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.grid = grid + + def __str__(self): + return 'Simplify dimension Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, + self.grid) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.grid == self.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain,variables,grid + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + variables = options['variables'] + for var in variables: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + + job_list.append(SimplifyDimensions(diags.data_manager, startdate, member, chunk, + options['domain'], var, options['grid'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + self.simplified = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + handler = Utils.openCdf(self.variable_file.local_file) + if 'i' not in handler.dimensions: + raise Exception('Variable {0.domain}:{0.variable} does not have i,j dimensions'.format(self)) + lat = handler.variables['lat'] + lat_values = lat[:, 0:1] + # noinspection PyTypeChecker + if np.any((lat[:] - lat_values) != 0): + raise Exception('Latitude is not constant over i dimension for variable ' + '{0.domain}:{0.variable}'.format(self)) + + lon = handler.variables['lon'] + lon_values = lon[0:1, :] + # noinspection PyTypeChecker + if np.any((lon[:] - lon) != 0): + raise Exception('Longitude is not constant over j dimension for variable ' + '{0.domain}:{0.variable}'.format(self)) + + temp = TempFile.get() + new_file = Utils.openCdf(temp, 'w') + for dim in handler.dimensions.keys(): + if dim in ('lat', 'lon', 'i', 'j', 'vertices'): + continue + Utils.copy_dimension(handler, new_file, dim, new_names={'i': 'lon', 'j': 'lat'}) + + new_file.createDimension('lon', handler.dimensions['i'].size) + new_file.createDimension('lat', handler.dimensions['j'].size) + new_file.createDimension('vertices', 2) + + for var in handler.variables.keys(): + if var in ('lat', 'lon', 'i', 'j', 'lat_vertices', 'lon_vertices'): + continue + Utils.copy_variable(handler, new_file, var, new_names={'i': 'lon', 'j': 'lat'}) + + self._create_var('lon', lon_values, handler, new_file) + self._create_var('lat', lat_values, handler, new_file) + handler.close() + new_file.close() + + self.simplified.set_local_file(temp) + + @staticmethod + def _create_var(var_name, var_values, source, destiny): + old_var = source.variables[var_name] + new_var = destiny.createVariable(var_name, old_var.dtype, dimensions=(var_name, )) + new_var[:] = var_values + Utils.copy_attributes(new_var, old_var) + + vertices_name = '{0}_vertices'.format(var_name) + + if vertices_name in source.variables: + var_vertices = source.variables[vertices_name] + if var_name == 'lon': + vertices_values = var_vertices[0:1, :, 2:] + else: + vertices_values = var_vertices[:, 0:1, 1:3] + new_lat_vertices = destiny.createVariable(vertices_name, var_vertices.dtype, + dimensions=(var_name, 'vertices')) + new_lat_vertices[:] = vertices_values + Utils.copy_attributes(new_lat_vertices, var_vertices) + diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py new file mode 100644 index 0000000000000000000000000000000000000000..92de0bef9d587e16d2d452f0fc88b9b74fba8f95 --- /dev/null +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -0,0 +1,126 @@ +# coding=utf-8 +import iris +import iris.analysis +import iris.exceptions + +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticFloatOption, DiagnosticDomainOption, \ + DiagnosticVariableOption +from earthdiagnostics.utils import TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class VerticalMeanMetersIris(Diagnostic): + """ + Averages vertically any given variable + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + :param box: box used to restrict the vertical mean + :type box: Box + + """ + + alias = 'vmean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, box): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.domain = domain + self.variable = variable + self.box = box + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.box == other.box and self.variable == other.variable + + def __str__(self): + return 'Vertical mean meters Startdate: {0} Member: {1} Chunk: {2} Variable: {3}:{4} ' \ + 'Box: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, self.box) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (meters), maximum depth (meters) + :type options: list[str] + :return: + """ + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFloatOption('min_depth', -1), + DiagnosticFloatOption('max_depth', -1), + DiagnosticDomainOption(default_value=ModelingRealms.ocean)) + options = cls.process_options(options, options_available) + + box = Box(True) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(VerticalMeanMetersIris(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], box)) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, + self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(self.domain, self.variable + 'vmean', self.startdate, self.member, + self.chunk, box=self.box) + + def compute(self): + """ + Runs the diagnostic + """ + iris.FUTURE.netcdf_no_unlimited = True + iris.FUTURE.netcdf_promote = True + + var_cube = iris.load_cube(self.variable_file.local_file) + + lev_names = ('lev', 'depth') + coord = None + for coord_name in lev_names: + try: + coord = var_cube.coord(coord_name) + except iris.exceptions.CoordinateNotFoundError: + pass + + if self.box.min_depth is None: + lev_min = coord.points[0] + else: + lev_min = self.box.min_depth + + if self.box.max_depth is None: + lev_max = coord.points[-1] + else: + lev_max = self.box.max_depth + var_cube = var_cube.extract(iris.Constraint(coord_values= + {coord.var_name: lambda cell: lev_min <= cell <= lev_max})) + var_cube = var_cube.collapsed(coord, iris.analysis.MEAN) + temp = TempFile.get() + iris.save(var_cube, temp, zlib=True) + self.results.set_local_file(temp, rename_var=var_cube.var_name) diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py new file mode 100644 index 0000000000000000000000000000000000000000..148f0ca210a07a2661bf68ded705a5edef2c46bd --- /dev/null +++ b/earthdiagnostics/general/yearlymean.py @@ -0,0 +1,111 @@ +# coding=utf-8 + +import os +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ + DiagnosticFrequencyOption, DiagnosticVariableOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealm + + +class YearlyMean(Diagnostic): + """ + Calculates yearly mean for a given variable + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: str + :param grid: original data grid + :type grid: str + """ + + alias = 'yearmean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.frequency = frequency + self.grid = grid + + def __str__(self): + return 'Calculate yearly mean Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Original frequency: {5} Grid: {6}'.format(self.startdate, self.member, self.chunk, + self.domain, self.variable, + self.frequency, self.grid) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.grid == other.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFrequencyOption(default_value=diags.config.frequency), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(YearlyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + + def declare_data_generated(self): + self.yearmean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.yearly, grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + + handler = Utils.openCdf(self.variable_file.local_file) + if 'region' in handler.variables: + noregion = TempFile.get() + Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) + Utils.cdo.yearmean(input=noregion, output=temp) + monmean_handler = Utils.openCdf(temp) + Utils.copy_variable(handler, monmean_handler, 'region') + monmean_handler.close() + else: + Utils.cdo.yearmean(input=self.variable_file.local_file, output=temp) + os.remove(self.variable_file.local_file) + + self.yearmean.set_local_file(temp) + diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py new file mode 100644 index 0000000000000000000000000000000000000000..703d5b47d8c28f8b6aa257eeed1fcf4cc29d143a --- /dev/null +++ b/earthdiagnostics/modelingrealm.py @@ -0,0 +1,103 @@ +# coding=utf-8 +from earthdiagnostics.frequency import Frequencies + + +class ModelingRealm(object): + + def __init__(self, domain_name): + lower_name = domain_name.lower() + if lower_name == 'seaice': + self.name = 'seaIce' + elif lower_name == 'landice': + self.name = 'landIce' + elif lower_name == 'atmoschem': + self.name = 'atmosChem' + elif lower_name == 'ocnbgchem': + self.name = 'ocnBgchem' + elif lower_name in ['ocean', 'atmos', 'land', 'aerosol']: + self.name = domain_name + else: + raise ValueError('Modelling realm {0} not recognized!'.format(domain_name)) + + def __eq__(self, other): + return other.__class__ == ModelingRealm and self.name == other.name + + def __ne__(self, other): + return not (self == other) + + def __str__(self): + return self.name + + def __repr__(self): + return str(self) + + def get_table_name(self, frequency, data_convention): + """ + Returns the table name for a domain-frequency pair + :param data_convention: Data convention in use + :type data_convention: str + :param frequency: variable's frequency + :type frequency: Frequency + :return: variable's table name + :rtype: str + """ + if self.name == 'seaIce': + if data_convention in ('specs', 'preface'): + prefix = 'OI' + else: + prefix = 'SI' + elif self.name == 'landIce': + prefix = 'LI' + else: + prefix = self.name[0].upper() + + if frequency == Frequencies.six_hourly: + table_name = '6hrPlev' + else: + if (frequency in (Frequencies.monthly, Frequencies.climatology)) or data_convention not in ('specs', + 'preface'): + table_name = prefix + str(frequency) + else: + table_name = frequency.frequency + return table_name + + def get_table(self, frequency, data_convention): + table_name = self.get_table_name(frequency, data_convention) + from earthdiagnostics.variable import CMORTable + return CMORTable(table_name, frequency, 'December 2013') + + +class ModelingRealms(object): + seaIce = ModelingRealm('seaice') + ocean = ModelingRealm('ocean') + landIce = ModelingRealm('landIce') + atmos = ModelingRealm('atmos') + land = ModelingRealm('land') + aerosol = ModelingRealm('aerosol') + atmosChem = ModelingRealm('atmosChem') + ocnBgchem = ModelingRealm('ocnBgchem') + + @classmethod + def parse(cls, modelling_realm): + """ + Return the basin matching the given name. If the parameter basin is a Basin instance, directly returns the same + instance. This bahaviour is intended to facilitate the development of methods that can either accept a name + or a Basin instance to characterize the basin. + + :param modelling_realm: basin name or basin instance + :type modelling_realm: str | Basin + :return: basin instance corresponding to the basin name + :rtype: Basin + """ + if isinstance(modelling_realm, ModelingRealm): + return modelling_realm + for name in cls.__dict__.keys(): + if name.startswith('_'): + continue + # noinspection PyCallByClass + value = cls.__getattribute__(cls, name) + if isinstance(value, ModelingRealm): + if modelling_realm.lower() in [value.name.lower()]: + return value + raise ValueError('Modelling realm {0} not recognized!'.format(modelling_realm)) + diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py new file mode 100644 index 0000000000000000000000000000000000000000..661986c9b9d515045e1aca11b56b126c7a1478b9 --- /dev/null +++ b/earthdiagnostics/obsreconmanager.py @@ -0,0 +1,272 @@ +# coding=utf-8 +import os + +from bscearth.utils.log import Log + +from earthdiagnostics.datamanager import DataManager +from earthdiagnostics.variable_type import VariableType + + +class ObsReconManager(DataManager): + """ + Data manager class for CMORized experiments + """ + def __init__(self, config): + super(ObsReconManager, self).__init__(config) + data_folders = self.config.data_dir.split(':') + self.config.data_dir = None + for data_folder in data_folders: + if os.path.isdir(os.path.join(data_folder, self.config.data_type, self.experiment.institute.lower(), + self.experiment.model.lower())): + self.config.data_dir = data_folder + break + + if not self.config.data_dir: + raise Exception('Can not find model data') + + if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size != 1: + raise Exception('For obs and recon data chunk_size must be always 1') + + # noinspection PyUnusedLocal + def request_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, + vartype=VariableType.MEAN): + filepath = self.get_file_path(startdate, domain, variable, frequency, vartype) + return self._get_file_from_storage(filepath) + + def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + pass + + # noinspection PyUnusedLocal + def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param domain: CMOR domain + :type domain: str + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + return NotImplementedError + + def get_file_path(self, startdate, domain, var, frequency, vartype, + box=None, grid=None): + """ + Returns the path to a concrete file + :param startdate: file's startdate + :type startdate: str + :param domain: file's domain + :type domain: str + :param var: file's var + :type var: str + :param frequency: file's frequency + :type frequency: Frequency + :param box: file's box + :type box: Box + :param grid: file's grid + :type grid: str + :return: path to the file + :rtype: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + """ + if not frequency: + frequency = self.config.frequency + var = self._get_final_var_name(box, var) + + folder_path = self._get_folder_path(frequency, domain, var, grid, vartype) + file_name = self._get_file_name(var, startdate) + + filepath = os.path.join(folder_path, file_name) + return filepath + + def _get_folder_path(self, frequency, domain, variable, grid, vartype): + + if not frequency.frequency.endswith('hr'): + var_folder = self.get_varfolder(domain, variable, grid) + else: + var_folder = variable + + folder_path = os.path.join(self.config.data_dir, self.config.data_type, + self.experiment.institute.lower(), + self.experiment.model.lower(), + frequency.folder_name(vartype), + var_folder) + return folder_path + + def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN): + """ + Ge a file containing all the data for one year for one variable + :param domain: variable's domain + :type domain: str + :param var: variable's name + :type var: str + :param startdate: startdate to retrieve + :type startdate: str + :param member: member to retrieve + :type member: int + :param year: year to retrieve + :type year: int + :param grid: variable's grid + :type grid: str + :param box: variable's box + :type box: Box + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: + """ + raise NotImplementedError() + + def get_var_url(self, var, startdate, frequency, box, vartype): + """ + Get url for dataset + :param var: variable to retrieve + :type var: str + :param startdate: startdate to retrieve + :type startdate: str + :param frequency: frequency to get: + :type frequency: Frequency | None + :param box: box to get + :type box: Box + :param vartype: type of variable + :type vartype: VariableType + :return: + """ + if not frequency: + frequency = self.config.frequency + var = self._get_final_var_name(box, var) + full_path = os.path.join(self.config.data_dir, self.config.data_type, self.experiment.institute, + self.experiment.model, frequency.folder_name(vartype)) + full_path = os.path.join(full_path, var, self._get_file_name(var, startdate)) + return full_path + + def _get_file_name(self, var, startdate): + if startdate: + if self.config.data_type != 'exp': + startdate = startdate[0:6] + return '{0}_{1}.nc'.format(var, startdate) + else: + return '{0}.nc'.format(var) + + def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, + frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): + """ + Creates the link of a given file from the CMOR repository. + + :param cmor_var: + :param move_old: + :param date_str: + :param year: if frequency is yearly, this parameter is used to give the corresponding year + :type year: int + :param domain: CMOR domain + :type domain: str + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + # THREDDSManager does not require links + pass + + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param vartype: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :rtype: str + """ + var = self._get_final_var_name(box, var) + filepath = self.get_file_path(startdate, domain, var, frequency, vartype, grid, box) + Log.debug('{0} requested', filepath) + return self._get_file_from_storage(filepath) + + # noinspection PyUnusedLocal + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param diagnostic: + :param region: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + if not frequency: + frequency = self.config.frequency + original_name = var + cmor_var = self.variable_list.get_variable(var) + if cmor_var: + var = cmor_var.short_name + final_name = var + + filepath = self.get_file_path(startdate, domain, final_name, frequency, vartype, box, grid) + netcdf_file = self._declare_generated_file(filepath, domain, final_name, cmor_var, self.config.data_convention, + region, diagnostic, grid, vartype, original_name) + netcdf_file.frequency = frequency + Log.debug('{0} will be generated', filepath) + return netcdf_file + diff --git a/earthdiagnostics/ocean/__init__.py b/earthdiagnostics/ocean/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..854d90d94bd5a3906a916bfe64798296cd774b55 --- /dev/null +++ b/earthdiagnostics/ocean/__init__.py @@ -0,0 +1,26 @@ +# coding=utf-8 +""" +Module containing the diagnostics related to the ocean output +""" +from earthdiagnostics.ocean.heatcontent import HeatContent +from earthdiagnostics.ocean.moc import Moc +from earthdiagnostics.ocean.areamoc import AreaMoc +from earthdiagnostics.ocean.maxmoc import MaxMoc +from earthdiagnostics.ocean.psi import Psi +from earthdiagnostics.ocean.gyres import Gyres +from earthdiagnostics.ocean.convectionsites import ConvectionSites +from earthdiagnostics.ocean.cutsection import CutSection +from earthdiagnostics.ocean.averagesection import AverageSection +from earthdiagnostics.ocean.interpolate import Interpolate +from earthdiagnostics.ocean.interpolatecdo import InterpolateCDO +from earthdiagnostics.ocean.verticalmeanmeters import VerticalMeanMeters +from earthdiagnostics.ocean.verticalmean import VerticalMean +from earthdiagnostics.ocean.mixedlayersaltcontent import MixedLayerSaltContent +from earthdiagnostics.ocean.siasiesiv import Siasiesiv +from earthdiagnostics.ocean.heatcontentlayer import HeatContentLayer +from earthdiagnostics.ocean.mixedlayerheatcontent import MixedLayerHeatContent +from earthdiagnostics.ocean.regionmean import RegionMean +from earthdiagnostics.ocean.rotation import Rotation +from earthdiagnostics.ocean.mxl import Mxl +from earthdiagnostics.ocean.verticalgradient import VerticalGradient +from earthdiagnostics.ocean.mask_land import MaskLand diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py new file mode 100644 index 0000000000000000000000000000000000000000..80c18474a3cb5f3d163467a3f73e04a052edf20c --- /dev/null +++ b/earthdiagnostics/ocean/areamoc.py @@ -0,0 +1,162 @@ +# coding=utf-8 +import numpy as np +from earthdiagnostics.constants import Basins +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption +from earthdiagnostics.box import Box +from earthdiagnostics.utils import Utils, TempFile +import os + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class AreaMoc(Diagnostic): + """ + Compute an Atlantic MOC index by averaging the meridional overturning + in a latitude band between 1km and 2km + or any other index averaging the meridional overturning in + a given basin and a given domain + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: March 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param basin: basin to compute + :type basin: Basin + :param box: box to compute + :type box: Box + """ + + alias = 'mocarea' + "Diagnostic alias for the configuration file" + + vsftmyz = 'vsftmyz' + + def __init__(self, data_manager, startdate, member, chunk, basin, box): + Diagnostic.__init__(self, data_manager) + self.basin = basin + self.startdate = startdate + self.member = member + self.chunk = chunk + self.required_vars = ['vo'] + self.generated_vars = ['vsftmyz'] + self.box = box + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.basin == other.basin and self.box == other.box + + def __str__(self): + return 'Area MOC Startdate: {0} Member: {1} Chunk: {2} Box: {3} Basin: {4}'.format(self.startdate, self.member, + self.chunk, self.box, + self.basin) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: minimum latitude, maximum latitude, minimum depth, maximum depth, basin=Global + :type options: list[str] + :return: + """ + + options_available = (DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins().Global)) + options = cls.process_options(options, options_available) + box = Box() + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(AreaMoc(diags.data_manager, startdate, member, chunk, options['basin'], box)) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, AreaMoc.vsftmyz, + self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(ModelingRealms.ocean, AreaMoc.vsftmyz, + self.startdate, self.member, self.chunk, + box=self.box) + + def compute(self): + """ + Runs the diagnostic + """ + nco = Utils.nco + cdo = Utils.cdo + + temp = TempFile.get() + temp2 = TempFile.get() + + Utils.copy_file(self.variable_file.local_file, temp) + + handler = Utils.openCdf(temp) + if 'i' in handler.dimensions: + handler.close() + nco.ncwa(input=temp, output=temp, options=('-O -a i',)) + handler = Utils.openCdf(temp) + + basin_index = np.where(handler.variables['basin'][:] == self.basin.name) + lat_values = handler.variables['lat'][:] + lat_type = handler.variables['lat'].dtype + lat_units = handler.variables['lat'].units + lat_long_name = handler.variables['lat'].long_name + + handler.close() + + if len(basin_index) == 0: + raise Exception('Basin {0} not defined in file') + basin_index = basin_index[0][0] + # To select basin and remove dimension + nco.ncwa(input=temp, output=temp, options=('-O -d basin,{0} -a basin'.format(basin_index),)) + + source = Utils.openCdf(temp) + destiny = Utils.openCdf(temp2, 'w') + + Utils.copy_dimension(source, destiny, 'time') + Utils.copy_dimension(source, destiny, 'lev') + Utils.copy_dimension(source, destiny, 'j', new_names={'j': 'lat'}) + + lat_variable = destiny.createVariable('lat', lat_type, 'lat') + lat_variable[:] = lat_values[:] + lat_variable.units = lat_units + lat_variable.long_name = lat_long_name + + Utils.copy_variable(source, destiny, 'lev') + Utils.copy_variable(source, destiny, 'time') + Utils.copy_variable(source, destiny, 'vsftmyz', new_names={'j': 'lat'}) + + source.close() + destiny.close() + + nco.ncks(input=temp2, output=temp, + options='-O -d lev,{0:.1f},{1:.1f} -d lat,{2:.1f},{3:.1f}'.format(self.box.min_depth, + self.box.max_depth, + self.box.min_lat, + self.box.max_lat)) + + cdo.vertmean(input=temp, output=temp2) + os.remove(temp) + nco.ncap2(input=temp2, output=temp2, options='-O -s "coslat[lat]=cos(lat[lat]*3.141592657/180.0)"') + nco.ncwa(input=temp2, output=temp2, options='-w coslat -a lat') + nco.ncks(input=temp2, output=temp2, options='-O -v vsftmyz,time') + self.results.set_local_file(temp2) diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py new file mode 100644 index 0000000000000000000000000000000000000000..8ca8abb0a03da07a2f5ea49b92e19f7ae838e2ba --- /dev/null +++ b/earthdiagnostics/ocean/averagesection.py @@ -0,0 +1,106 @@ +# coding=utf-8 +import os +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import * +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealm + + +class AverageSection(Diagnostic): + """ + Compute an average of a given zone. The variable MUST be in a regular grid + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: March 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param box: box to use for the average + :type box: Box + + """ + + alias = 'avgsection' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, box, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.box = box + self.grid = grid + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.box == other.box + + def __str__(self): + return 'Average section Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Box: {0.box} ' \ + 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum longitude, maximum longitude, minimum latitude, maximum latitude, domain=ocean + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticIntOption('min_lon'), + DiagnosticIntOption('max_lon'), + DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + box = Box() + box.min_lon = options['min_lon'] + box.max_lon = options['max_lon'] + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(AverageSection(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], box, options['grid'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def declare_data_generated(self): + self.mean = self.declare_chunk(self.domain, self.variable + 'mean', self.startdate, self.member, self.chunk, + box=self.box, grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + variable_file = self.variable_file.local_file + Utils.cdo.zonmean(input='-sellonlatbox,{0},{1},{2},{3} {4}'.format(self.box.min_lon, self.box.max_lon, + self.box.min_lat, self.box.max_lat, + variable_file), + output=temp) + os.remove(variable_file) + self.mean.set_local_file(temp, rename_var='tos') diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py new file mode 100644 index 0000000000000000000000000000000000000000..74271682754c1072f48ff2eb7e8d2f1735e76d68 --- /dev/null +++ b/earthdiagnostics/ocean/convectionsites.py @@ -0,0 +1,128 @@ +# coding=utf-8 +import numpy as np +from bscearth.utils.log import Log +from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.constants import Models +from earthdiagnostics.modelingrealm import ModelingRealms + + +class ConvectionSites(Diagnostic): + """ + Compute the intensity of convection in the four main convection sites + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: October 2013 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param model_version: model version + :type model_version: str + """ + + alias = 'convection' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, model_version): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.model_version = model_version + self.mlotst_handler = None + + def __str__(self): + return 'Convection sites Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.model_version == other.model_version + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: None + :type options: list[str] + :return: + """ + if len(options) > 1: + raise Exception('The convection sites diagnostic has no options') + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(ConvectionSites(diags.data_manager, startdate, member, chunk, diags.model_version)) + return job_list + + def request_data(self): + self.mixed_layer = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.sites = self.declare_chunk(ModelingRealms.ocean, 'site', self.startdate, self.member, self.chunk) + + def compute(self): + """ + Runs the diagnostic + """ + if self.model_version in [Models.ECEARTH_2_3_O1L42, Models.ECEARTH_3_0_O1L46, + Models.NEMO_3_2_O1L42, Models.NEMO_3_3_O1L46, + Models.NEMOVAR_O1L42]: + labrador = [225, 245, 215, 255] + irminger = [245, 290, 215, 245] + gin = [260, 310, 245, 291] + wedell = [225, 280, 1, 50] + + elif self.model_version in [Models.ECEARTH_3_0_O25L46, Models.ECEARTH_3_0_O25L75, + Models.ECEARTH_3_2_O25L75, Models.ECEARTH_3_2_O1L75, + Models.GLORYS2_V1_O25L75]: + raise Exception("Option convection not available yet for {0}".format(self.model_version)) + else: + raise Exception("Input grid {0} not recognized".format(self.model_version)) + + mlotst_file = self.mixed_layer.local_file + output = TempFile.get() + + self.mlotst_handler = Utils.openCdf(mlotst_file) + handler = Utils.openCdf(output, 'w') + handler.createDimension('time', self.mlotst_handler.variables['time'].shape[0]) + handler.createDimension('region', 4) + Utils.copy_variable(self.mlotst_handler, handler, 'time') + var_region = handler.createVariable('region', str, 'region') + var_gyre = handler.createVariable('site', 'f', ('time', 'region')) + var_gyre.short_name = 'site' + var_gyre.long_name = 'convection sites' + var_gyre.units = 'm^3/s' + + var_region[0] = 'labrador' + var_gyre[:, 0] = self._convection_site(labrador) + + var_region[1] = 'irminger' + var_gyre[:, 1] = self._convection_site(irminger) + + var_region[2] = 'gin' + var_gyre[:, 2] = self._convection_site(gin) + + var_region[3] = 'wedell' + var_gyre[:, 3] = self._convection_site(wedell) + + self.mlotst_handler.close() + handler.close() + + self.sites.set_local_file(output) + Log.info('Finished convection sites for startdate {0}, member {1}, chunk {2}', + self.startdate, self.member, self.chunk) + + def _convection_site(self, site): + return np.max(self.mlotst_handler.variables['mlotst'][:, site[2] - 1:site[3] - 1, site[0] - 1:site[1] - 1], + (1, 2)) diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py new file mode 100644 index 0000000000000000000000000000000000000000..d4e5f89c9e8c6fc43669b73a522937166d57e02c --- /dev/null +++ b/earthdiagnostics/ocean/cutsection.py @@ -0,0 +1,190 @@ +# coding=utf-8 +import numpy as np +from bscearth.utils.log import Log + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBoolOption, DiagnosticIntOption, \ + DiagnosticDomainOption, DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.utils import Utils +from earthdiagnostics.modelingrealm import ModelingRealms + + +class CutSection(Diagnostic): + """ + Cuts a meridional or zonal section + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: September 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: Domain + :param zonal: specifies if section is zonal or meridional + :type zonal: bool + :param value: value of the section's coordinate + :type value: int + """ + + alias = 'cutsection' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, zonal, value): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.zonal = zonal + self.value = value + + self.box = Box() + if self.zonal: + self.box.max_lon = self.value + self.box.min_lon = self.value + else: + self.box.max_lat = self.value + self.box.min_lat = self.value + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.zonal == other.zonal and \ + self.value == other.value + + def __str__(self): + return 'Cut section Startdate: {0} Member: {1} Chunk: {2} Variable: {3}:{4} ' \ + 'Zonal: {5} Value: {6}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, + self.zonal, self.value) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, zonal, value, domain=ocean + :type options: list[str] + :return: + """ + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticBoolOption('zonal'), + DiagnosticIntOption('value'), + DiagnosticDomainOption(default_value=ModelingRealms.ocean)) + options = cls.process_options(options, options_available) + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(CutSection(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['zonal'], options['value'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.section = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + box=self.box) + + def compute(self): + """ + Runs the diagnostic + """ + nco = Utils.nco + + handler = Utils.openCdf('mesh_hgr.nc') + dimi = handler.dimensions['i'].size + dimj = handler.dimensions['j'].size + dimlev = handler.dimensions['lev'].size + + lon = handler.variables['lon'][:] + lat = handler.variables['lat'][:] + handler.close() + + handler = Utils.openCdf('mask.nc') + mask_lev = handler.variables['tmask'][:] + mask_lev = mask_lev.astype(float) + # noinspection PyTypeChecker + np.place(mask_lev, mask_lev == 0, [1e20]) + handler.close() + + # Latitude / longitude of the zonal / meridional section + exactpos = self.value + if not self.zonal: + while exactpos < np.min(lon): + exactpos += 360 + while exactpos > np.max(lon): + exactpos -= 360 + size = dimj + else: + size = dimi + + # Collect the indexes defining the section + + listi = np.empty(size, dtype=int) + listj = np.empty(size, dtype=int) + + for jpt in range(0, size): + if not self.zonal: + vector = lon[jpt, :] + else: + vector = lat[:, jpt] + distance = abs(vector - exactpos) + pos = np.where(distance == min(distance)) + if not self.zonal: + listi[jpt] = pos[0][0] + listj[jpt] = jpt + else: + listi[jpt] = jpt + listj[jpt] = pos[0][0] + + temp = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk) + + handler = Utils.openCdf(temp) + dimtime = handler.dimensions['time'].size + var_array = handler.variables[self.variable][:] + handler.close() + + var = np.empty([dimtime, dimlev, size], dtype=handler.variables[self.variable].dtype) + new_coord = np.empty(size, dtype=float) + if self.zonal: + old_coord = lon + else: + old_coord = lat + + for jpt in range(0, size): + var[:, :, jpt] = np.maximum(var_array[:, :, listj[jpt], listi[jpt]], + mask_lev[:, :, listj[jpt], listi[jpt]]) + new_coord[jpt] = old_coord[listj[jpt], listi[jpt]] + + nco.ncks(input=temp, output=temp, options=('-O -v lev,time',)) + + handler = Utils.openCdf(temp) + if not self.zonal: + handler.createDimension('lat', size) + coord_var = handler.createVariable('lat', float, 'lat') + file_var = handler.createVariable(self.variable, float, ('time', 'lev', 'lat')) + else: + handler.createDimension('lon', size) + coord_var = handler.createVariable('lon', float, 'lon') + file_var = handler.createVariable(self.variable, float, ('time', 'lev', 'lon')) + coord_var[:] = new_coord[:] + file_var[:] = var[:] + file_var.missing_value = 1e20 + handler.close() + + self.section.set_local_file(temp) + Log.info('Finished cut section for startdate {0}, member {1}, chunk {2}', + self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py new file mode 100644 index 0000000000000000000000000000000000000000..b01971488d8e19b447c03ff114bcad32302955d0 --- /dev/null +++ b/earthdiagnostics/ocean/gyres.py @@ -0,0 +1,178 @@ +# coding=utf-8 +import numpy as np +from bscearth.utils.log import Log + +from earthdiagnostics.constants import Models +from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class Gyres(Diagnostic): + """ + Compute the intensity of the subtropical and subpolar gyres + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: October 2013 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param model_version: model version + :type model_version: str + """ + + alias = 'gyres' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, model_version): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.model_version = model_version + self.var_vsftbarot = None + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.model_version == other.model_version + + def __str__(self): + return 'Gyres Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Model version: {0.model_version}'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: None + :type options: list[str] + :return: + """ + if len(options) > 1: + raise Exception('The gyres diagnostic has no options') + job_list = list() + model_version = diags.config.experiment.model_version + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Gyres(diags.data_manager, startdate, member, chunk, model_version)) + return job_list + + def request_data(self): + self.vsftbarot = self.request_chunk(ModelingRealms.ocean, 'vsftbarot', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.gyre = self.declare_chunk(ModelingRealms.ocean, 'gyre', self.startdate, self.member, self.chunk) + + # noinspection PyPep8Naming + def compute(self): + """ + Runs the diagnostic + """ + if self.model_version in [Models.ECEARTH_2_3_O1L42, Models.ECEARTH_3_0_O1L46, + Models.NEMO_3_2_O1L42, Models.NEMO_3_3_O1L46, + Models.NEMOVAR_O1L42]: + + subpolNAtl = [230, 275, 215, 245] + subpolNPac = [70, 145, 195, 235] + subtropNPac = [45, 175, 165, 220] + subtropNAtl = [195, 275, 175, 225] + subtropSPac = [70, 205, 120, 145] + subtropSAtl = [235, 300, 120, 145] + subtropInd = [320, 30, 110, 180] + ACC = [1, 361, 1, 65] + + elif self.model_version in [Models.ECEARTH_3_0_O25L46, Models.ECEARTH_3_0_O25L75, Models.GLORYS2_V1_O25L75, + Models.ECEARTH_3_2_O1L75, Models.ECEARTH_3_2_O25L75]: + raise Exception("Option gyres not available yet for {0}".format(self.model_version)) + else: + raise Exception("Input grid {0} not recognized".format(self.model_version)) + + output = TempFile.get() + vsftbarot_file = self.vsftbarot.local_file + + handler_original = Utils.openCdf(vsftbarot_file) + self.var_vsftbarot = handler_original.variables['vsftbarot'] + handler = Utils.openCdf(output, 'w') + handler.createDimension('time', handler_original.variables['time'].shape[0]) + handler.createDimension('region', 8) + Utils.copy_variable(handler_original, handler, 'time') + var_region = handler.createVariable('region', str, 'region') + + var_gyre = handler.createVariable('gyre', 'f', ('time', 'region'), fill_value=0.0) + var_gyre.valid_max = 2e8 + var_gyre.valid_min = 0.0 + var_gyre.short_name = 'gyre' + var_gyre.long_name = 'gyre' + var_gyre.units = 'm^3/s' + + var_region[0] = 'subpolNAtl' + var_gyre[:, 0] = self._gyre(subpolNAtl, True) + Log.debug('subpolNAtl: {0}', var_gyre[:, 0]) + + var_region[1] = 'subpolNPac' + var_gyre[:, 1] = self._gyre(subpolNPac, True) + Log.debug('subpolNPac: {0}', var_gyre[:, 1]) + + var_region[2] = 'subtropNPac' + var_gyre[:, 2] = self._gyre(subtropNPac) + Log.debug('subtropNPac: {0}', var_gyre[:, 2]) + + var_region[3] = 'subtropSPac' + var_gyre[:, 3] = self._gyre(subtropSPac) + Log.debug('subtropSPac: {0}', var_gyre[:, 3]) + + var_region[4] = 'subtropNAtl' + var_gyre[:, 4] = self._gyre(subtropNAtl) + Log.debug('subtropNAtl: {0}', var_gyre[:, 4]) + + var_region[5] = 'subtropSAtl' + var_gyre[:, 5] = self._gyre(subtropSAtl) + Log.debug('subtropSAtl: {0}', var_gyre[:, 5]) + + var_region[6] = 'subtropInd' + var_gyre[:, 6] = self._gyre(subtropInd) + Log.debug('subtropInd: {0}', var_gyre[:, 6]) + + var_region[7] = 'ACC' + var_gyre[:, 7] = self._gyre(ACC) + Log.debug('ACC: {0}', var_gyre[:, 7]) + + handler.close() + handler_original.close() + self.gyre.set_local_file(output) + Log.info('Finished gyres for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) + + def _gyre(self, site, invert=False): + if invert: + return np.min(self._extract_section(site), (1, 2)) * -1 + else: + return np.max(self._extract_section(site), (1, 2)) + + def _extract_section(self, site): + if site[2] <= site[3]: + if site[0] <= site[1]: + return self.var_vsftbarot[:, site[2] - 1:site[3] - 1, site[0] - 1:site[1] - 1] + else: + return np.concatenate((self.var_vsftbarot[:, site[2] - 1:site[3] - 1, site[0] - 1:], + self.var_vsftbarot[:, site[2] - 1:site[3] - 1, :site[1] - 1]), axis=2) + + else: + if site[0] <= site[1]: + return np.concatenate((self.var_vsftbarot[:, site[2] - 1:, site[0] - 1: site[1] - 1], + self.var_vsftbarot[:, :site[3] - 1, site[0] - 1: site[1] - 1]), axis=1) + else: + temp = np.concatenate((self.var_vsftbarot[:, site[2] - 1:, :], + self.var_vsftbarot[:, :site[3] - 1, :]), axis=1) + return np.concatenate((temp[:, :, site[0] - 1:], + temp[:, :, :site[1] - 1]), axis=2) diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py new file mode 100644 index 0000000000000000000000000000000000000000..31ac9268b1e302515744091ec3ab87b6d1c08a9a --- /dev/null +++ b/earthdiagnostics/ocean/heatcontent.py @@ -0,0 +1,229 @@ +# coding=utf-8 +import shutil + +from earthdiagnostics import cdftools +from earthdiagnostics.constants import Basins +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticIntOption +from earthdiagnostics.box import Box +from earthdiagnostics.modelingrealm import ModelingRealms +import numpy as np + + +class HeatContent(Diagnostic): + """ + Compute the total ocean heat content + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: May 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param mixed_layer: If 1, restricts calculation to the mixed layer, if -1 exclude it. If 0, no effect + :type mixed_layer: int + :param box: box to use for the average + :type box: Box + + """ + + alias = 'ohc' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, basin, mixed_layer, box, min_level, max_level): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.basin = basin + self.mxloption = mixed_layer + self.box = box + self.min_level = min_level + self.max_level = max_level + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.box == other.box and self.basin == other.basin and self.mxloption == other.mxloption + + def __str__(self): + return 'Heat content Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Mixed layer: {0.mxloption} ' \ + 'Box: {0.box} Basin: {0.basin}'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: basin, mixed layer option (1 to only compute at the mixed layer, -1 to exclude it, 0 to ignore), + minimum depth, maximum depth + :type options: list[str] + :return: + """ + options_available = (DiagnosticBasinOption('basin'), + DiagnosticIntOption('mixed_layer', None, -1, 1), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth')) + options = cls.process_options(options, options_available) + box = Box(True) + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] + min_level = 0 + max_level = 0 + + if box.min_depth > 0 or box.max_depth > 0: + + max_level, min_level = cls._get_levels_from_meters(box) + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(HeatContent(diags.data_manager, startdate, member, chunk, + options['basin'], options['mixed_layer'], box, min_level, max_level)) + return job_list + + @classmethod + def _get_levels_from_meters(cls, box): + depth_t, depth_w = cls._read_level_values() + + def find_nearest(array, value): + idx = (np.abs(array - value)).argmin() + return idx + + if box.min_depth > 0: + min_level = find_nearest(depth_t, box.min_depth) + if depth_t[min_level] < box.min_depth: + min_level += 1 + else: + min_level = 0 + numlevels = len(depth_t) + + if box.max_depth > 0: + max_level = find_nearest(depth_t, box.max_depth) + if depth_t[max_level] >= box.max_depth: + max_level -= 1 + else: + max_level = numlevels - 1 + + if min_level < 0: + min_level = 0 + if max_level >= numlevels: + max_level = numlevels - 1 + + box.min_depth = round(depth_w[min_level]) + + if max_level < numlevels - 1: + box.max_depth = round(depth_w[max_level + 1]) + else: + box.max_depth = round(depth_w[-1]) + min_level += 1 + max_level += 1 + return max_level, min_level + + @classmethod + def _read_level_values(cls): + handler = Utils.openCdf('mesh_zgr.nc') + if 'gdepw_1d' in handler.variables: + depth_w = handler.variables['gdepw_1d'][0, :] + elif 'gdepw_0' in handler.variables: + depth_w = handler.variables['gdepw_0'][0, :] + else: + raise Exception('gdepw 1D variable can not be found') + if 'gdept_1d' in handler.variables: + depth_t = handler.variables['gdept_1d'][0, :] + elif 'gdept_0' in handler.variables: + depth_t = handler.variables['gdept_0'][0, :] + else: + raise Exception('gdept 1D variable can not be found') + handler.close() + return depth_t, depth_w + + def request_data(self): + self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) + if self.mxloption != 0: + self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + if self.box.min_depth == 0: + # For cdftools, this is all levels + box_save = None + else: + box_save = self.box + + self.heatcsum = self.declare_chunk(ModelingRealms.ocean, 'heatcsum', self.startdate, self.member, self.chunk, + box=box_save, region=self.basin.fullname) + self.heatcmean = self.declare_chunk(ModelingRealms.ocean, 'heatcvmean', self.startdate, self.member, self.chunk, + box=box_save, region=self.basin.fullname) + + def compute(self): + """ + Runs the diagnostic + """ + nco = Utils.nco + temperature_file = TempFile.get() + Utils.copy_file(self.thetao.local_file, temperature_file) + if self.mxloption != 0: + nco.ncks(input=self.mlotst.local_file, output=temperature_file, options=('-A -v mlotst',)) + + para = list() + if self.min_level != 0: + para.append('-zoom') + para.append(0) + para.append(0) + para.append(0) + para.append(0) + para.append(self.min_level) + para.append(self.max_level) + if self.mxloption != 0: + para.append('-mxloption') + para.append(str(self.mxloption)) + if self.basin != Basins().Global: + handler = Utils.openCdf('mask_regions.3d.nc') + if self.basin.name not in handler.variables: + raise Exception('Basin {0} is not defined on mask_regions.nc'.format(self.basin.name)) + + handler.close() + para.append('-M') + para.append('mask_regions.3d.nc') + para.append(self.basin.name) + + temp2 = TempFile.get() + + cdftools.run('cdfheatc', options=para, input=temperature_file, output=temp2, input_option='-f') + + results = Utils.openCdf(temp2) + heatcsum_temp = TempFile.get() + heatcvmean_temp = TempFile.get() + nco.ncks(input=temperature_file, output=heatcsum_temp, options=('-O -v time',)) + shutil.copy(heatcsum_temp, heatcvmean_temp) + + heatcsum_handler = Utils.openCdf(heatcsum_temp) + thc = heatcsum_handler.createVariable('heatcsum', float, 'time') + thc.standard_name = "integral_of_sea_water_potential_temperature_expressed_as_heat_content" + thc.long_name = "Total heat content" + thc.units = "J" + thc[:] = results.variables['heatc3d'][:, 0, 0] + heatcsum_handler.close() + + heatcvmean_handler = Utils.openCdf(heatcvmean_temp) + uhc = heatcvmean_handler.createVariable('heatcvmean', float, 'time') + uhc.standard_name = "integral_of_sea_water_potential_temperature_expressed_as_heat_content" + uhc.long_name = "Heat content per unit volume" + uhc.units = "J*m^-3" + uhc[:] = results.variables['heatc3dpervol'][:, 0, 0] + heatcvmean_handler.close() + + results.close() + + Utils.setminmax(heatcsum_temp, 'heatcsum') + self.heatcsum.set_local_file(heatcsum_temp) + Utils.setminmax(heatcvmean_temp, 'heatcvmean') + self.heatcmean.set_local_file(heatcvmean_temp) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py new file mode 100644 index 0000000000000000000000000000000000000000..c64b19021e455ecec86dfb501e9f7923a1df205d --- /dev/null +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -0,0 +1,171 @@ +# coding=utf-8 +import numpy as np + +from earthdiagnostics.constants import Basins +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class HeatContentLayer(Diagnostic): + """ + Point-wise Ocean Heat Content in a specified ocean thickness (J/m-2) + + :original author: Isabel Andreu Burillo + :contributor: Virginie Guemas + :contributor: Eleftheria Exarchou + :contributor: Javier Vegas-Regidor + + :created: June 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param box: box to use for the calculations + :type box: Box + """ + + alias = 'ohclayer' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, box, weight, min_level, max_level): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.box = box + self.weight = weight + self.min_level = min_level + self.max_level = max_level + self.required_vars = ['so', 'mlotst'] + self.generated_vars = ['scvertsum'] + + def __str__(self): + return 'Heat content layer Startdate: {0} Member: {1} Chunk: {2} Box: {3}'.format(self.startdate, self.member, + self.chunk, self.box) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: minimum depth, maximum depth, basin=Global + :type options: list[str] + """ + options_available = (DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins().Global)) + options = cls.process_options(options, options_available) + + box = Box(True) + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] + job_list = list() + + handler = Utils.openCdf('mesh_zgr.nc') + # mask = Utils.get_mask(options['basin']) + mask = handler.variables['tmask'][:] + + if 'e3t' in handler.variables: + e3t = handler.variables['e3t'][:] + elif 'e3t_0' in handler.variables: + e3t = handler.variables['e3t_0'][:] + else: + raise Exception('e3t variable can not be found') + + if 'gdepw' in handler.variables: + depth = handler.variables['gdepw'][:] + elif 'gdepw_0' in handler.variables: + depth = handler.variables['gdepw_0'][:] + else: + raise Exception('gdepw variable can not be found') + + e3t_3d = e3t.shape != depth.shape + if e3t_3d: + mask = e3t_3d * mask + else: + e3t = e3t[0, :] + + while len(depth.shape) < 4: + depth = np.expand_dims(depth, -1) + handler.close() + + def calculate_weight(e3t_point, depth_point, mask_point): + """ + Calculates the weight for each cell + """ + if not mask_point: + return 0 + top = depth_point + bottom = top + e3t_point + if bottom < box.min_depth or top > box.max_depth: + return 0 + else: + if top < box.min_depth: + top = box.min_depth + if bottom > box.max_depth: + bottom = box.max_depth + + return (bottom - top) * 1020 * 4000 + + calc = np.vectorize(calculate_weight, otypes='f') + weight = calc(e3t, depth, mask) + + # Now we will reduce to the levels with any weight != 0 to avoid loading too much data on memory + levels = weight.shape[1] + min_level = 0 + while min_level < levels and not weight[:, min_level, :].any(): + min_level += 1 + max_level = min_level + while max_level < (levels - 1) and weight[:, max_level + 1, :].any(): + max_level += 1 + weight = weight[:, min_level:max_level, :] + + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(HeatContentLayer(diags.data_manager, startdate, member, chunk, box, + weight, min_level, max_level)) + return job_list + + def request_data(self): + self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.heatc = self.declare_chunk(ModelingRealms.ocean, 'heatc', self.startdate, self.member, self.chunk, + box=self.box) + + def compute(self): + """ + Runs the diagnostic + """ + nco = Utils.nco + thetao_file = TempFile.get() + results = TempFile.get() + + Utils.copy_file(self.thetao.local_file, thetao_file) + + handler = Utils.openCdf(thetao_file) + Utils.convert_units(handler.variables['thetao'], 'K') + heatc_sl = np.sum(handler.variables['thetao'][:, self.min_level:self.max_level, :] * self.weight, 1) + handler.sync() + handler.renameVariable('thetao', 'heatc_sl') + handler.close() + + nco.ncks(input=thetao_file, output=results, options=('-O -v lon,lat,time',)) + Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False, True) + handler_results = Utils.openCdf(results) + handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) + handler_results.sync() + handler_results.variables['heatc'][:] = heatc_sl + handler_results.close() + + Utils.setminmax(results, 'heatc') + self.heatc.set_local_file(results) diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py new file mode 100644 index 0000000000000000000000000000000000000000..ed64f4bedaf7d5e6a2f8328309c8705b706ca1ec --- /dev/null +++ b/earthdiagnostics/ocean/interpolate.py @@ -0,0 +1,197 @@ +# coding=utf-8 +import shutil +import threading + +import os +from bscearth.utils.log import Log +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption, \ + DiagnosticVariableListOption + +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class Interpolate(Diagnostic): + """ + 3-dimensional conservative interpolation to the regular atmospheric grid. + It can also be used for 2D (i,j) variables + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: November 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: Domain + :param model_version: model version + :type model_version: str + """ + + alias = 'interp' + "Diagnostic alias for the configuration file" + + lock = threading.Lock() + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, target_grid, model_version, + invert_lat, original_grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.model_version = model_version + self.required_vars = [variable] + self.generated_vars = [variable] + self.tempTemplate = '' + self.grid = target_grid + self.invert_latitude = invert_lat + self.original_grid = original_grid + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.model_version == other.model_version and self.domain == other.domain and \ + self.variable == other.variable and self.grid == other.grid and \ + self.invert_latitude == other.invert_latitude and self.original_grid == other.original_grid + + def __str__(self): + return 'Interpolate Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Target grid: {5} Invert lat: {6} ' \ + 'Model: {7} Original grid: {8}'.format(self.startdate, self.member, self.chunk, self.domain, + self.variable, self.grid, self.invert_latitude, + self.model_version, self.original_grid) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: target_grid, variable, domain=ocean + :type options: list[str] + :return: + """ + options_available = (DiagnosticOption('target_grid'), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variable'), + DiagnosticDomainOption(default_value=ModelingRealms.ocean), + DiagnosticBoolOption('invert_lat', False), + DiagnosticOption('original_grid', '')) + options = cls.process_options(options, options_available) + + job_list = list() + for var in options['variable']: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append( + Interpolate(diags.data_manager, startdate, member, chunk, + options['domain'], var, options['target_grid'], + diags.config.experiment.model_version, options['invert_lat'], options['original_grid'])) + return job_list + + def request_data(self): + self.original = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.original_grid) + + def declare_data_generated(self): + self.regridded = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + variable_file = TempFile.get() + Utils.copy_file(self.original.local_file, variable_file) + Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False, rename_dimension=True) + cdo = Utils.cdo + nco = Utils.nco + handler = Utils.openCdf(variable_file) + if 'lev' in handler.dimensions: + num_levels = handler.dimensions['lev'].size + has_levels = True + else: + num_levels = 1 + has_levels = False + handler.close() + for lev in range(0, num_levels): + self._interpolate_level(lev, has_levels, variable_file) + + temp = TempFile.get() + if has_levels: + Interpolate.lock.acquire() + nco.ncrcat(input=self._get_level_file(0), output=temp, + options="-n {0},2,1 -v '{1}'".format(num_levels, self.variable)) + Interpolate.lock.release() + + else: + Utils.move_file(self._get_level_file(0), temp) + + handler = Utils.openCdf(temp) + if 'record' in handler.dimensions: + handler.renameDimension('record', 'lev') + handler.close() + nco.ncpdq(input=temp, output=temp, options=('-O -h -a time,lev',)) + + if has_levels: + nco.ncks(input=variable_file, output=temp, options=('-A -v lev',)) + for lev in range(0, num_levels): + os.remove(self._get_level_file(lev)) + temp2 = TempFile.get() + cdo.setgrid('t106grid', input=temp, output=temp2) + os.remove(temp) + if self.invert_latitude: + cdo.invertlatdata(input=temp2, output=temp) + shutil.move(temp, temp2) + if not has_levels: + nco.ncks(input=temp2, output=temp2, options=('-O -v {0},lat,lon,time'.format(self.variable),)) + + self.regridded.set_local_file(temp2) + + def _get_level_file(self, lev): + if not self.tempTemplate: + self.tempTemplate = TempFile.get(suffix='_01.nc') + # self.tempTemplate = 'temp_01.nc' + return self.tempTemplate.replace('_01.nc', '_{0:02d}.nc'.format(lev + 1)) + + def _interpolate_level(self, lev, has_levels, input_file): + nco = Utils.nco + temp = TempFile.get() + if has_levels: + nco.ncks(input=input_file, output=temp, options='-O -d lev,{0} -v {1},lat,lon'.format(lev, self.variable)) + nco.ncwa(input=temp, output=temp, options=('-O -h -a lev',)) + else: + shutil.copy(input_file, temp) + + weights_file = '/esnas/autosubmit/con_files/weigths/{0}/rmp_{0}_to_{1}_lev{2}.nc'.format(self.model_version, + self.grid, lev + 1) + if not os.path.isfile(weights_file): + raise Exception('Level {0} weights file does not exist for model {1} ' + 'and grid {2}'.format(lev+1, self.model_version, self.grid)) + namelist_file = TempFile.get(suffix='') + scrip_use_in = open(namelist_file, 'w') + scrip_use_in.writelines("&remap_inputs\n") + scrip_use_in.writelines(" remap_wgt = '{0}'\n".format(weights_file)) + scrip_use_in.writelines(" infile = '{0}'\n".format(temp)) + scrip_use_in.writelines(" invertlat = FALSE\n") + scrip_use_in.writelines(" var = '{0}'\n".format(self.variable)) + scrip_use_in.writelines(" fromregular = FALSE\n") + scrip_use_in.writelines(" outfile = '{0}'\n".format(temp)) + scrip_use_in.writelines("/\n") + scrip_use_in.close() + Utils.execute_shell_command('/home/Earth/jvegas/pyCharm/cfutools/interpolation/scrip_use ' + '{0}'.format(namelist_file), Log.INFO) + os.remove(namelist_file) + nco.ncecat(input=temp, output=temp, options=("-O -h",)) + shutil.move(temp, self._get_level_file(lev)) + Log.debug("Level {0} ready", lev) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py new file mode 100644 index 0000000000000000000000000000000000000000..0bc7709b51a8e5b4162562b8be1b19778d273e1d --- /dev/null +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -0,0 +1,240 @@ +# coding=utf-8 +import os + +import numpy as np + +from earthdiagnostics.diagnostic import * +from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms +from earthdiagnostics.utils import Utils, TempFile + + +class InterpolateCDO(Diagnostic): + """ + 3-dimensional conservative interpolation to the regular atmospheric grid. + It can also be used for 2D (i,j) variables + + :original author: Javier Vegas-Regidor + + :created: October 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param model_version: model version + :type model_version: str + """ + + alias = 'interpcdo' + "Diagnostic alias for the configuration file" + + BILINEAR = 'bilinear' + BICUBIC = 'bicubic' + CONSERVATIVE = 'conservative' + CONSERVATIVE2 = 'conservative2' + + METHODS = [BILINEAR, BICUBIC, CONSERVATIVE, CONSERVATIVE2] + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, target_grid, model_version, + mask_oceans, original_grid, weights): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.model_version = model_version + self.required_vars = [variable] + self.generated_vars = [variable] + self.tempTemplate = '' + self.grid = target_grid + self.mask_oceans = mask_oceans + self.original_grid = original_grid + self.weights = weights + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.model_version == other.model_version and self.domain == other.domain and \ + self.variable == other.variable and self.mask_oceans == other.mask_oceans and self.grid == other.grid and \ + self.original_grid == other.original_grid + + def __str__(self): + return 'Interpolate with CDO Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Variable: {0.domain}:{0.variable} Target grid: {0.grid} Original grid: {0.original_grid} ' \ + 'Mask ocean: {0.mask_oceans} Model: {0.model_version}'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: target_grid, variable, domain=ocean + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(default_value=ModelingRealms.ocean), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), + DiagnosticOption('target_grid', diags.config.experiment.atmos_grid.lower()), + DiagnosticChoiceOption('method', InterpolateCDO.METHODS, InterpolateCDO.BILINEAR), + DiagnosticBoolOption('mask_oceans', True), + DiagnosticOption('original_grid', ''), + DiagnosticBoolOption('weights_from_mask', True) + ) + options = cls.process_options(options, options_available) + target_grid = cls._translate_ifs_grids_to_cdo_names(options['target_grid']) + if not target_grid: + raise Exception('Target grid not provided') + job_list = list() + weights = TempFile.get() + method = options['method'].lower() + cls._compute_weights(diags, method, options, target_grid, weights) + for var in options['variables']: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(InterpolateCDO(diags.data_manager, startdate, member, chunk, + options['domain'], var, target_grid, + diags.config.experiment.model_version, options['mask_oceans'], + options['original_grid'], weights)) + return job_list + + @classmethod + def _compute_weights(cls, diags, method, options, target_grid, weights): + if options['weights_from_mask']: + temp = cls.get_sample_grid_file() + else: + startdate, member, chunk = diags.config.experiment.get_chunk_list()[0] + temp = diags.data_manager.get_file(options['domain'], options['variable'], startdate, member, chunk, + grid=options['original_grid']) + if method == InterpolateCDO.BILINEAR: + Utils.cdo.genbil(target_grid, input=temp, output=weights) + elif method == InterpolateCDO.BICUBIC: + Utils.cdo.genbic(target_grid, input=temp, output=weights) + elif method == InterpolateCDO.CONSERVATIVE: + Utils.cdo.genycon(target_grid, input=temp, output=weights) + elif method == InterpolateCDO.CONSERVATIVE2: + Utils.cdo.gencon2(target_grid, input=temp, output=weights) + os.remove(temp) + + @classmethod + def get_sample_grid_file(cls): + temp = TempFile.get() + Utils.nco.ncks(input='mask.nc', output=temp, options=('-O -v tmask,lat,lon,gphif,glamf',)) + handler = Utils.openCdf(temp) + lon = handler.variables['lon'] + lon.units = "degrees_east" + lon.long_name = "Longitude" + lon.nav_model = "Default grid" + lon.standard_name = "longitude" + lon.short_name = "lon" + lon.bounds = 'lon_bnds' + + lat = handler.variables['lat'] + lat.units = "degrees_north" + lat.long_name = "Latitude" + lat.nav_model = "Default grid" + lat.standard_name = "latitude" + lat.short_name = "lat" + lat.bounds = 'lat_bnds' + + handler.createDimension('bounds', 4) + + lon_bnds = handler.createVariable('lon_bnds', lon.datatype, ('j', 'i', 'bounds')) + corner_lat = handler.variables['glamf'][0, ...] + lon_bnds[:, :, 0] = corner_lat + lon_bnds[:, :, 1] = np.roll(corner_lat, 1, 0) + lon_bnds[:, :, 2] = np.roll(corner_lat, -1, 1) + lon_bnds[:, :, 3] = np.roll(lon_bnds[:, :, 1], -1, 1) + + lat_bnds = handler.createVariable('lat_bnds', lat.datatype, ('j', 'i', 'bounds')) + corner_lat = handler.variables['gphif'][0, ...] + lat_bnds[:, :, 0] = corner_lat + lat_bnds[:, :, 1] = np.roll(corner_lat, 1, 0) + lat_bnds[:, :, 2] = np.roll(corner_lat, 1, 1) + lat_bnds[:, :, 3] = np.roll(lat_bnds[:, :, 1], 1, 1) + lat_bnds[0, :, 1] = lat_bnds[1, 0, 1] - 1 + lat_bnds[0, :, 3] = lat_bnds[1, 0, 3] - 1 + + tmask = handler.variables['tmask'] + tmask.coordinates = 'time lev lat lon' + + handler.close() + + Utils.nco.ncks(input=temp, output=temp, options=('-O -x -v gphif,glamf',)) + return temp + + @classmethod + def _translate_ifs_grids_to_cdo_names(cls, target_grid): + if target_grid.upper().startswith('T159L'): + target_grid = 't106grid' + if target_grid.upper().startswith('T255L'): + target_grid = 't170grid' + if target_grid.upper().startswith('T511L'): + target_grid = 't340grid' + return target_grid + + def request_data(self): + self.original = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.original_grid) + + def declare_data_generated(self): + self.regridded = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + variable_file = TempFile.get() + Utils.copy_file(self.original.local_file, variable_file) + Utils.rename_variables(variable_file, {'jpib': 'i', 'jpjb': 'j', 'x': 'i', 'y': 'j', + 'time_counter': 'time', 't': 'time', + 'SSTK_ens0': 'tos', 'SSTK_ens1': 'tos', 'SSTK_ens2': 'tos', + 'nav_lat': 'lat', 'nav_lon': 'lon'}, + must_exist=False, rename_dimension=True) + handler = Utils.openCdf(variable_file) + var = handler.variables[self.variable] + units = var.units + coordinates = list() + for dim in var.dimensions: + if dim == 'i': + if 'lat' in handler.variables: + coordinates.append('lat') + else: + coordinates.append('latitude') + elif dim == 'j': + if 'lon' in handler.variables: + coordinates.append('lon') + else: + coordinates.append('longitude') + else: + coordinates.append(dim) + var.coordinates = ' '.join(coordinates) + + if self.mask_oceans: + mask = Utils.get_mask(Basins().Global).astype(float) + mask[mask == 0] = np.nan + var[:] = mask * var[:] + handler.close() + + temp = TempFile.get() + Utils.cdo.remap(','.join((self.grid.split('_')[0], self.weights)), input=variable_file, output=temp) + + handler = Utils.openCdf(temp) + handler.variables[self.variable].units = units + handler.close() + + self.regridded.set_local_file(temp) + + + + + diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py new file mode 100644 index 0000000000000000000000000000000000000000..a7af9aaa04719fc12837178e4a8af396dc3121f5 --- /dev/null +++ b/earthdiagnostics/ocean/mask_land.py @@ -0,0 +1,112 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableListOption, \ + DiagnosticDomainOption, DiagnosticChoiceOption, DiagnosticOption +from earthdiagnostics.utils import Utils, TempFile +import numpy as np + + +class MaskLand(Diagnostic): + """ + Changes values present in the mask for NaNs + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + """ + + alias = 'maskland' + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, mask, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.domain = domain + self.variable = variable + self.mask = mask + self.grid = grid + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable + + def __str__(self): + return 'Land mask Startdate: {0} Member: {1} Chunk: {2} Variable: {3}:{4} ' \ + 'Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, self.grid) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (level), maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption('domain'), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), + DiagnosticChoiceOption('cell', ('t', 'u', 'v', 'f', 'w'), 't'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + + cell_point = options['cell'] + # W and T share the same mask + if cell_point == 'w': + cell_point = 't' + + mask = cls._get_mask(cell_point) + + job_list = list() + for var in options['variables']: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(MaskLand(diags.data_manager, startdate, member, chunk, + options['domain'], var, mask, options['grid'])) + return job_list + + @classmethod + def _get_mask(cls, cell_point): + mask_file = Utils.openCdf('mask.nc') + mask = mask_file.variables['{0}mask'.format(cell_point)][:].astype(float) + mask[mask == 0] = np.nan + mask_file.close() + return mask + + "Diagnostic alias for the configuration file" + + def request_data(self): + self.var_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def declare_data_generated(self): + self.masked_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + Utils.copy_file(self.var_file.local_file, temp) + + handler = Utils.openCdf(temp) + if 'lev' not in handler.dimensions: + mask = self.mask[:, 0, ...] + else: + mask = self.mask + handler.variables[self.variable][:] *= mask + handler.close() + + self.masked_file.set_local_file(temp) + diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py new file mode 100644 index 0000000000000000000000000000000000000000..2be7110e75edca01ff46cc3cbc877e5807990621 --- /dev/null +++ b/earthdiagnostics/ocean/maxmoc.py @@ -0,0 +1,240 @@ +# coding=utf-8 +import netCDF4 +import numpy as np +from bscearth.utils.log import Log +from earthdiagnostics.constants import Basins +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticFloatOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.variable_type import VariableType + + +class MaxMoc(Diagnostic): + """ + Compute an Atlantic MOC index by finding the maximum of the annual + mean meridional overturning in a latitude / depth region + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: March 2012 + :last modified: June 2016 + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param year: year to compute + :type year: int + :param basin: basin to compute + :type basin: Basin + :param box: box to compute + :type box: Box + """ + + alias = 'mocmax' + "Diagnostic alias for the configuration file" + + vsftmyz = 'vsftmyz' + + def __init__(self, data_manager, startdate, member, year, basin, box): + Diagnostic.__init__(self, data_manager) + self.basin = basin + self.startdate = startdate + self.member = member + self.year = year + self.box = box + + def __str__(self): + return 'Max moc Startdate: {0} Member: {1} Year: {2} Box: {3} ' \ + 'Basin: {4}'.format(self.startdate, self.member, self.year, self.box, self.basin.name) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.year == other.year and \ + self.box == other.box and self.basin == other.basin + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each complete year to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: minimum latitude, maximum latitude, minimum depth, maximum depth, basin=global + :type options: list[str] + :return: + """ + options_available = (DiagnosticFloatOption('min_lat'), + DiagnosticFloatOption('max_lat'), + DiagnosticFloatOption('min_depth'), + DiagnosticFloatOption('max_depth'), + DiagnosticBasinOption('basin', Basins().Global)) + options = cls.process_options(options, options_available) + box = Box(True) + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] + + job_list = list() + for startdate in diags.config.experiment.startdates: + for member in diags.config.experiment.members: + years = diags.config.experiment.get_full_years(startdate) + if len(years) == 0: + Log.user_warning('No complete years are available with the given configuration. ' + 'MaxMoc can not be computed') + for year in years: + job_list.append(MaxMoc(diags.data_manager, startdate, member, year, options['basin'], box)) + return job_list + + def request_data(self): + self.variable_file = self.request_year(ModelingRealms.ocean, MaxMoc.vsftmyz, + self.startdate, self.member, self.year) + + def declare_data_generated(self): + + self.results = {'vsftmyzmax': self.declare_year(ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, + self.year, box=self.box, vartype=VariableType.STATISTIC), + 'vsftmyzmaxlev': self.declare_year(ModelingRealms.ocean, 'vsftmyzmaxlev', self.startdate, + self.member, self.year, box=self.box, + vartype=VariableType.STATISTIC), + 'vsftmyzmaxlat': self.declare_year(ModelingRealms.ocean, 'vsftmyzmaxlat', self.startdate, + self.member, self.year, box=self.box, + vartype=VariableType.STATISTIC), + 'vsftmyzmin': self.declare_year(ModelingRealms.ocean, 'vsftmyzmin', self.startdate, self.member, + self.year, box=self.box, vartype=VariableType.STATISTIC), + 'vsftmyzminlev': self.declare_year(ModelingRealms.ocean, 'vsftmyzminlev', self.startdate, + self.member, self.year, box=self.box, + vartype=VariableType.STATISTIC), + 'vsftmyzminlat': self.declare_year(ModelingRealms.ocean, 'vsftmyzminlat', self.startdate, + self.member, self.year, box=self.box, + vartype=VariableType.STATISTIC)} + + def compute(self): + """ + Runs the diagnostic + """ + nco = Utils.nco + + temp = TempFile.get() + Utils.copy_file(self.variable_file.local_file, temp) + + handler = Utils.openCdf(temp) + if 'i' in handler.dimensions: + handler.close() + nco.ncwa(input=temp, output=temp, options=('-O -a i',)) + else: + handler.close() + handler = Utils.openCdf(temp) + basin_index = np.where(handler.variables['basin'][:] == self.basin.name) + if len(basin_index) == 0: + raise Exception("Basin {1} is not defined in {0}", temp, self.basin.name) + basin_index = basin_index[0][0] + + lev = handler.variables['lev'][:] + lat = handler.variables['lat'][:] + + if self.box.min_lat == self.box.max_lat: + lat_inds = ((np.abs(lat - self.box.min_lat)).argmin(),) + else: + lat_inds = np.where((lat > self.box.min_lat) & (lat < self.box.max_lat))[0] + + if self.box.min_depth == self.box.max_depth: + lev_inds = ((np.abs(lev - self.box.min_depth)).argmin(),) + else: + lev_inds = np.where((lev > self.box.min_depth) & (lev < self.box.max_depth))[0] + + Log.info('Computing year {0}', str(self.year)) + moc = handler.variables['vsftmyz'][:, lev_inds, lat_inds, basin_index] + handler.close() + + moc = np.mean(moc, 0) + + maximum = np.amax(moc) + max_index = np.unravel_index(np.argmax(moc), moc.shape) + # noinspection PyUnresolvedReferences + max_lev = lev[lev_inds[max_index[0]]] + # noinspection PyUnresolvedReferences + max_lat = lat[lat_inds[max_index[1]]] + + minimum = np.amin(moc) + minimum_index = np.unravel_index(np.argmin(moc), moc.shape) + # noinspection PyUnresolvedReferences + min_lev = lev[lev_inds[minimum_index[0]]] + # noinspection PyUnresolvedReferences + min_lat = lat[lat_inds[minimum_index[1]]] + + Log.info('Maximum {0} Sv, latitude {1} depth {2} m', maximum, max_lat, max_lev) + Log.info('Minimum {0} Sv, latitude {1} depth {2} m', minimum, min_lat, min_lev) + + handler, temp = self._create_output_file() + var = handler.createVariable('vsftmyzmax', float, ('time',)) + var.long_name = 'Maximum_Overturning' + var.units = 'Sverdrup' + var.valid_min = -1000. + var.valid_max = 1000. + var[0] = maximum + handler.close() + self.results['vsftmyzmax'].set_local_file(temp) + + handler, temp = self._create_output_file() + var = handler.createVariable('vsftmyzmaxlat', float, ('time',)) + var.long_name = 'Latitude_of_Maximum_Overturning' + var.units = 'Degrees' + var.valid_min = -90. + var.valid_max = 90. + var[0] = max_lat + handler.close() + self.results['vsftmyzmaxlat'].set_local_file(temp) + + handler, temp = self._create_output_file() + var = handler.createVariable('vsftmyzmaxlev', float, ('time',)) + var.long_name = 'Depth_of_Maximum_Overturning' + var.units = 'Meters' + var.valid_min = 0. + var.valid_max = 10000. + var[0] = max_lev + handler.close() + self.results['vsftmyzmaxlev'].set_local_file(temp) + + handler, temp = self._create_output_file() + var = handler.createVariable('vsftmyzmin', float, ('time',)) + var.long_name = 'Minimum_Overturning' + var.units = 'Sverdrup' + var.valid_min = -1000. + var.valid_max = 1000. + var[0] = minimum + handler.close() + self.results['vsftmyzmin'].set_local_file(temp) + + handler, temp = self._create_output_file() + var = handler.createVariable('vsftmyzminlat', float, ('time',)) + var.long_name = 'Latitude_of_Minimum_Overturning' + var.units = 'Degrees' + var.valid_min = -90. + var.valid_max = 90. + var[0] = min_lat + handler.close() + self.results['vsftmyzminlat'].set_local_file(temp) + + handler, temp = self._create_output_file() + var = handler.createVariable('vsftmyzminlev', float, ('time',)) + var.long_name = 'Depth_of_Minimum_Overturning' + var.units = 'Meters' + var.valid_min = 0. + var.valid_max = 10000. + var[0] = min_lev + handler.close() + self.results['vsftmyzminlev'].set_local_file(temp) + + def _create_output_file(self): + temp = TempFile.get() + handler = netCDF4.Dataset(temp, 'w') + handler.createDimension('time') + + time = handler.createVariable('time', 'i2', ('time',)) + time.calendar = 'gregorian' + time.units = 'days since January 1, {0}'.format(self.year) + return handler, temp diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py new file mode 100644 index 0000000000000000000000000000000000000000..e81624d2555f7eb4db05dcdbccaff430d2878e13 --- /dev/null +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -0,0 +1,88 @@ +# coding=utf-8 +import os + +from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics import cdftools +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class MixedLayerHeatContent(Diagnostic): + """ + Compute mixed layer heat content + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + """ + + alias = 'mlotsthc' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.required_vars = ['so', 'mlotst'] + self.generated_vars = ['scvertsum'] + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk + + def __str__(self): + return 'Mixed layer heat content Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, + self.chunk) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: None + :type options: list[str] + :return: + """ + if len(options) > 1: + raise Exception('The mixed layer ocean heat content diagnostic has no options') + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(MixedLayerHeatContent(diags.data_manager, startdate, member, chunk)) + return job_list + + def request_data(self): + self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) + self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.ohcsum = self.declare_chunk(ModelingRealms.ocean, 'ohcvsumlotst', self.startdate, self.member, self.chunk) + + def compute(self): + """ + Runs the diagnostic + """ + temperature_file = TempFile.get() + Utils.copy_file(self.thetao.local_file, temperature_file) + Utils.nco.ncks(input=self.mlotst.local_file, output=temperature_file, options=('-A -v mlotst',)) + + temp = TempFile.get() + cdftools.run('cdfmxlheatc', input=temperature_file, output=temp) + + os.remove(temperature_file) + + Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlheatc': 'ohcvsumlotst'}, False, True) + Utils.setminmax(temp, 'ohcvsumlotst') + self.ohcsum.set_local_file(temp) diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py new file mode 100644 index 0000000000000000000000000000000000000000..ad350961fdcabed51b378f9dd2fd26bc79bef164 --- /dev/null +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -0,0 +1,85 @@ +# coding=utf-8 +import os +from earthdiagnostics import cdftools +from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class MixedLayerSaltContent(Diagnostic): + """ + Compute mixed layer salt content + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + """ + alias = 'mlotstsc' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.required_vars = ['so', 'mlotst'] + self.generated_vars = ['scvertsum'] + + def __str__(self): + return 'Mixed layer salt content Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, + self.chunk) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: None + :type options: list[str] + :return: + """ + if len(options) > 1: + raise Exception('The mixed layer salt content diagnostic has no options') + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(MixedLayerSaltContent(diags.data_manager, startdate, member, chunk)) + return job_list + + def request_data(self): + self.so = self.request_chunk(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) + self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.sosum = self.declare_chunk(ModelingRealms.ocean, 'scvsummlotst', self.startdate, self.member, self.chunk) + + def compute(self): + """ + Runs the diagnostic + """ + salinity_file = TempFile.get() + Utils.copy_file(self.so.local_file, salinity_file) + Utils.nco.ncks(input=self.mlotst.local_file, output=salinity_file, options=('-A -v mlotst',)) + + temp = TempFile.get() + cdftools.run('cdfmxlsaltc', input=salinity_file, output=temp) + os.remove(salinity_file) + + Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlsaltc': 'scvsummlotst'}, False, True) + Utils.setminmax(temp, 'scvsummlotst') + self.sosum.set_local_file(temp) diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py new file mode 100644 index 0000000000000000000000000000000000000000..7686e84b3a616984c53e070840220220e6a2860d --- /dev/null +++ b/earthdiagnostics/ocean/moc.py @@ -0,0 +1,121 @@ +# coding=utf-8 +import numpy as np +from bscearth.utils.log import Log + +from earthdiagnostics import cdftools +from earthdiagnostics.constants import Basins +from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class Moc(Diagnostic): + """ + Compute the MOC for oceanic basins + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: March 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + """ + + alias = 'moc' + "Diagnostic alias for the configuration file" + + vsftmyz = 'vsftmyz' + + def __init__(self, data_manager, startdate, member, chunk): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.required_vars = ['vo'] + self.generated_vars = ['vsftmyz'] + + def __str__(self): + return 'MOC Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: None + :type options: list[str] + :return: + """ + if len(options) > 1: + raise Exception('The MOC diagnostic has no options') + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Moc(diags.data_manager, startdate, member, chunk)) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(ModelingRealms.ocean, Moc.vsftmyz, self.startdate, self.member, self.chunk) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + + Log.debug('Computing MOC') + cdftools.run('cdfmoc', input=self.variable_file.local_file, output=temp) + Utils.nco.ncks(input=self.variable_file.local_file, output=temp, options=('-A -v lev',)) + Utils.convert2netcdf4(temp) + + Log.debug('Reformatting variables') + handler = Utils.openCdf(temp) + + basins_list = [Basins().Global.name] + if 'zomsfatl' in handler.variables: + basins_list += [Basins().Atlantic.name, Basins().Pacific.name, Basins().IndoPacific.name, + Basins().Indian.name] + + handler.createDimension('basin', len(basins_list)) + handler.createVariable('basin', str, 'basin') + handler.variables['basin'][:] = np.array(basins_list, dtype=object) + example = handler.variables['zomsfglo'] + # noinspection PyProtectedMember + moc = handler.createVariable('vsftmyz', example.datatype, + ('time', 'lev', 'i', 'j', 'basin'), + fill_value=example._FillValue) + + moc.units = Utils.convert_to_ASCII_if_possible(example.units) + moc.add_offset = example.add_offset + moc.scale_factor = example.scale_factor + + moc[:, :, :, :, 0] = handler.variables['zomsfglo'][:] + + if 'zomsfatl' in handler.variables: + moc[:, :, :, :, 1] = handler.variables['zomsfatl'][:] + moc[:, :, :, :, 2] = handler.variables['zomsfpac'][:] + moc[:, :, :, :, 3] = handler.variables['zomsfinp'][:] + moc[:, :, :, :, 4] = handler.variables['zomsfind'][:] + + handler.close() + + Utils.nco.ncks(input=temp, output=temp, + options=('-O -x -v zomsfglo,zomsfatl,zomsfpac,zomsfinp,zomsfind,zomsfinp0',)) + Utils.setminmax(temp, 'vsftmyz') + + self.results.set_local_file(temp) diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py new file mode 100644 index 0000000000000000000000000000000000000000..fe531b76cd0ce4e30a27a7eee4bee4206368ffa8 --- /dev/null +++ b/earthdiagnostics/ocean/mxl.py @@ -0,0 +1,79 @@ +# coding=utf-8 +import os + +from earthdiagnostics import cdftools +from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class Mxl(Diagnostic): + """ + Compute the mixed layer depth + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + """ + + alias = 'mxl' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk + + def __str__(self): + return 'Mixed layer Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: None + :type options: list[str] + :return: + """ + if len(options) > 1: + raise Exception('The mxl diagnostic has no options') + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Mxl(diags.data_manager, startdate, member, chunk)) + return job_list + + def request_data(self): + self.thetao_file = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) + self.so_file = self.request_chunk(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.mlotst_file = self.declare_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + cdftools.run('cdfmxl', input=[self.thetao_file, self.so_file], output=temp, options='-nc4') + temp2 = TempFile.get() + source = Utils.openCdf(temp) + destiny = Utils.openCdf(temp2, 'w') + Utils.copy_variable(source, destiny, 'somxl010', must_exist=True, add_dimensions=True) + Utils.copy_variable(source, destiny, 'lat') + Utils.copy_variable(source, destiny, 'lon') + source.close() + destiny.close() + self.mlotst_file.set_local_file(temp2, rename_var='somxl010') + os.remove(temp) diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py new file mode 100644 index 0000000000000000000000000000000000000000..fd1ee553ead2b07aa9bca2191f6fbb98194d5b71 --- /dev/null +++ b/earthdiagnostics/ocean/psi.py @@ -0,0 +1,78 @@ +# coding=utf-8 +from earthdiagnostics import cdftools +from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class Psi(Diagnostic): + """ + Compute the barotropic stream function + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: March 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + """ + + alias = 'psi' + "Diagnostic alias for the configuration file" + + vsftbarot = 'vsftbarot' + + def __init__(self, data_manager, startdate, member, chunk): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk + + def __str__(self): + return 'PSI Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: None + :type options: list[str] + :return: + """ + if len(options) > 1: + raise Exception('The PSI diagnostic has no options') + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Psi(diags.data_manager, startdate, member, chunk)) + return job_list + + def request_data(self): + self.uo = self.request_chunk(ModelingRealms.ocean, 'uo', self.startdate, self.member, self.chunk) + self.vo = self.request_chunk(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.psi = self.declare_chunk(ModelingRealms.ocean, Psi.vsftbarot, self.startdate, self.member, self.chunk) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + cdftools.run('cdfpsi', input=[self.uo.local_file, self.vo.local_file], output=temp, options='-mean -mask') + Utils.rename_variable(temp, 'sobarstf', Psi.vsftbarot) + Utils.setminmax(temp, Psi.vsftbarot) + self.psi.set_local_file(temp) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py new file mode 100644 index 0000000000000000000000000000000000000000..1d6cebb8d7961a60f2ce172b5cfd0e5a5d8727b9 --- /dev/null +++ b/earthdiagnostics/ocean/regionmean.py @@ -0,0 +1,174 @@ +# coding=utf-8 +import os + +from earthdiagnostics import cdftools +from earthdiagnostics.box import Box +from earthdiagnostics.constants import Basins +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticIntOption, DiagnosticDomainOption, \ + DiagnosticBoolOption, DiagnosticBasinOption, DiagnosticVariableOption +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile + + +class RegionMean(Diagnostic): + """ + Computes the mean value of the field (3D, weighted). For 3D fields, + a horizontal mean for each level is also given. If a spatial window + is specified, the mean value is computed only in this window. + + :original author: Javier Vegas-Regidor + + :created: March 2017 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + :param box: box used to restrict the vertical mean + :type box: Box + """ + + alias = 'regmean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid_point, box, save3d, basin, + variance, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.domain = domain + self.variable = variable + self.grid_point = grid_point.upper() + self.box = box + self.save3d = save3d + self.basin = basin + self.variance = variance + self.grid = grid + self.declared = {} + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.box == other.box and self.variable == other.variable + + def __str__(self): + return 'Region mean Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Variable: {0.variable} ' \ + 'Grid point: {0.grid_point} Box: {0.box} Save 3D: {0.save3d} Save variance: {0.variance} ' \ + 'Original grid: {0.grid}'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (level), maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticOption('grid_point', 'T'), + DiagnosticBasinOption('basin', Basins().Global), + DiagnosticIntOption('min_depth', 0), + DiagnosticIntOption('max_depth', 0), + DiagnosticBoolOption('save3D', True), + DiagnosticBoolOption('variance', False), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + + box = Box() + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(RegionMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid_point'], box, + options['save3D'], options['basin'], options['variance'], options['grid'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def declare_data_generated(self): + if self.box.min_depth == 0: + # To cdftools, this means all levels + box_save = None + else: + box_save = self.box + + self.declare_var('mean', False, box_save) + self.declare_var('mean', True, box_save) + + if self.variance: + self.declare_var('var', False, box_save) + self.declare_var('var', True, box_save) + + def compute(self): + """ + Runs the diagnostic + """ + mean_file = TempFile.get() + + variable_file = self.variable_file.local_file + + handler = Utils.openCdf(variable_file) + self.save3d &= 'lev' in handler.dimensions + handler.close() + + cdfmean_options = [self.variable, self.grid_point, 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] + if self.variance: + cdfmean_options += ['-var'] + if self.basin != Basins().Global: + cdfmean_options.append('-M') + cdfmean_options.append('mask_regions.3d.nc') + cdfmean_options.append(self.basin.name) + + cdftools.run('cdfmean', input=variable_file, output=mean_file, options=cdfmean_options) + Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) + + self.send_var('mean', False, mean_file) + self.send_var('mean', True, mean_file) + + if self.variance: + self.send_var('var', False, mean_file) + self.send_var('var', True, mean_file) + + os.remove(mean_file) + + def send_var(self, var, threed, mean_file): + if threed: + if not self.save3d: + return False + original_name = '{0}_{1}'.format(var, self.variable) + final_name = '{1}3d{0}'.format(var, self.variable) + levels = ',lev' + else: + original_name = '{0}_3D{1}'.format(var, self.variable) + final_name = '{1}{0}'.format(var, self.variable) + levels = '' + + temp2 = TempFile.get() + Utils.nco.ncks(input=mean_file, output=temp2, options=('-v {0},lat,lon{1}'.format(original_name, levels),)) + self.declared[final_name].set_local_file(temp2, rename_var=original_name) + + def declare_var(self, var, threed, box_save): + if threed: + if not self.save3d: + return False + final_name = '{1}3d{0}'.format(var, self.variable) + else: + final_name = '{1}{0}'.format(var, self.variable) + + self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, + self.chunk, box=box_save, region=self.basin, grid=self.grid) + diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py new file mode 100644 index 0000000000000000000000000000000000000000..bdd6132c4cba6c04878ded48c8961eef25b88f2e --- /dev/null +++ b/earthdiagnostics/ocean/rotation.py @@ -0,0 +1,186 @@ +# coding=utf-8 +import shutil +from bscearth.utils.log import Log + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class Rotation(Diagnostic): + """ + Cuts a meridional or zonal section + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: September 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param domain: variable's domain + :type domain: Domain + """ + + alias = 'rotate' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variableu, variablev, executable): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variableu = variableu + self.variablev = variablev + self.domain = domain + self.executable = executable + self.tempTemplate = None + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variableu == other.variableu and self.variablev == other.variablev + + def __str__(self): + return 'Rotate variables Startdate: {0} Member: {1} Chunk: {2} Variables: {3}:{4} , ' \ + '{3}:{5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variableu, + self.variablev) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, zonal, value, domain=ocean + :type options: list[str] + :return: + """ + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager, 'variableu'), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'variablev'), + DiagnosticDomainOption(default_value=ModelingRealms.ocean), + DiagnosticOption('executable', + '/home/Earth/jvegas/pyCharm/cfutools/interpolation/rotateUVorca')) + options = cls.process_options(options, options_available) + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Rotation(diags.data_manager, startdate, member, chunk, + options['domain'], options['variableu'], options['variablev'], + options['executable'])) + return job_list + + def request_data(self): + self.ufile = self.request_chunk(self.domain, self.variableu, self.startdate, self.member, self.chunk) + self.vfile = self.request_chunk(self.domain, self.variablev, self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.urotated_file = self.declare_chunk(self.domain, self.variableu, self.startdate, self.member, self.chunk, + grid='rotated') + self.vrotated_file = self.declare_chunk(self.domain, self.variablev, self.startdate, self.member, self.chunk, + grid='rotated') + + def compute(self): + """ + Runs the diagnostic + """ + + handler = Utils.openCdf(self.ufile.local_file) + if 'lev' in handler.dimensions: + self.num_levels = handler.dimensions['lev'].size + self.has_levels = True + else: + self.num_levels = 1 + self.has_levels = False + handler.close() + + for lev in range(0, self.num_levels): + self._rotate_level(lev) + + urotated = self._merge_levels(self.variableu, 'u') + vrotated = self._merge_levels(self.variablev, 'v') + + ufile_handler = Utils.openCdf(self.ufile.local_file) + self._add_metadata_and_vars(ufile_handler, urotated, self.variableu) + ufile_handler.close() + self.urotated_file.set_local_file(urotated) + + vfile_handler = Utils.openCdf(self.vfile.local_file) + self._add_metadata_and_vars(vfile_handler, vrotated, self.variablev) + vfile_handler.close() + self.vrotated_file.set_local_file(urotated) + + def _merge_levels(self, var, direction): + temp = TempFile.get() + if self.has_levels: + Utils.nco.ncecat(input=self._get_level_file(0, direction), output=temp, + options=("-n {0},2,1 -v '{1}'".format(self.num_levels, var),)) + handler = Utils.openCdf(temp) + if 'record' in handler.dimensions: + handler.renameDimension('record', 'lev') + handler.close() + Utils.nco.ncpdq(input=temp, output=temp, options=('-O -h -a time,lev',)) + Utils.rename_variables(temp, {'x': 'i', 'y': 'j'}, must_exist=False, rename_dimension=True) + else: + Utils.move_file(self._get_level_file(0, direction), temp) + return temp + + def _rotate_level(self, lev): + ufile = self._extract_level(self.ufile.local_file, self.variableu, lev) + vfile = self._extract_level(self.vfile.local_file, self.variablev, lev) + namelist_file = self._create_namelist(ufile, self._get_level_file(lev, 'u'), + vfile, self._get_level_file(lev, 'v')) + Utils.execute_shell_command('{0} {1}'.format(self.executable, namelist_file), Log.INFO) + + def _extract_level(self, input_file, var, level): + temp = TempFile.get() + if self.has_levels: + Utils.nco.ncks(input=input_file, output=temp, options=('-O -d lev,{0} -v {1},lat,lon'.format(level, var),)) + Utils.nco.ncwa(input=temp, output=temp, options=('-O -h -a lev',)) + else: + shutil.copy(input_file, temp) + return temp + + def _create_namelist(self, ufile, urotated, vfile, vrotated): + namelist_file = TempFile.get(suffix='') + rotate_namelist = open(namelist_file, 'w') + rotate_namelist.write('&nam_rotUV\n') + rotate_namelist.write(' Ufilein = "{0}"\n'.format(ufile)) + rotate_namelist.write(' Uvarin = "{0}"\n'.format(self.variableu)) + rotate_namelist.write(' Vfilein = "{0}"\n'.format(vfile)) + rotate_namelist.write(' Vvarin = "{0}"\n'.format(self.variablev)) + rotate_namelist.write(' meshmask = "mask.nc"\n') + rotate_namelist.write(' Ufileout = "{0}"\n'.format(urotated)) + rotate_namelist.write('Vfileout = "{0}"\n'.format(vrotated)) + rotate_namelist.writelines("/\n") + rotate_namelist.close() + return namelist_file + + def _add_metadata_and_vars(self, reference_file_handler, rotaded_file, var_name): + rotated_handler = Utils.openCdf(rotaded_file) + self._copy_extra_variables(reference_file_handler, rotated_handler) + Utils.copy_attributes(rotated_handler.variables[var_name], reference_file_handler.variables[var_name], + ('_FillValue',)) + rotated_handler.close() + + def _copy_extra_variables(self, reference_file_handler, rotated_handler): + for var in reference_file_handler.variables.keys(): + if var not in rotated_handler.variables.keys() and var not in [self.variableu, self.variablev]: + Utils.copy_variable(reference_file_handler, rotated_handler, var, True, True) + + def _get_level_file(self, lev, direction): + if not self.tempTemplate: + self.tempTemplate = TempFile.get(suffix='_01.nc') + # self.tempTemplate = 'temp_01.nc' + return self.tempTemplate.replace('_01.nc', '_{1}_{0:02d}.nc'.format(lev + 1, direction)) + + + + diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py new file mode 100644 index 0000000000000000000000000000000000000000..d16295a458be9d6b8dec18a8a5a71720f07ba745 --- /dev/null +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -0,0 +1,157 @@ +# coding=utf-8 +import netCDF4 +import os + +from bscearth.utils.log import Log + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption +from earthdiagnostics.utils import Utils, TempFile +# noinspection PyUnresolvedReferences +import earthdiagnostics.cdftoolspython as cdftoolspython +import numpy as np + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.constants import Basins + + +class Siasiesiv(Diagnostic): + """ + Compute the sea ice extent , area and volume in both hemispheres or a specified region. + + + :original author: Virginie Guemas + :contributor: Neven Fuckar + :contributor: Ruben Cruz + :contributor: Javier Vegas-Regidor + + :created: April 2012 + :last modified: June 2016 + + """ + alias = 'siasiesiv' + "Diagnostic alias for the configuration file" + + e1t = None + e2t = None + gphit = None + + def __init__(self, data_manager, startdate, member, chunk, basin, mask): + """ + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param mask: mask to use + :type mask: numpy.array + """ + Diagnostic.__init__(self, data_manager) + self.basin = basin + self.startdate = startdate + self.member = member + self.chunk = chunk + self.mask = mask + self.generated = {} + + def __str__(self): + return 'Siasiesiv Startdate: {0} Member: {1} Chunk: {2} Basin: {3}'.format(self.startdate, self.member, + self.chunk, self.basin) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: basin + :type options: list[str] + :return: + """ + options_available = (DiagnosticBasinOption('basin', Basins().Global), ) + options = cls.process_options(options, options_available) + + if options['basin'] is None: + Log.error('Basin not recognized') + return () + + mask = np.asfortranarray(Utils.get_mask(options['basin'])) + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, options['basin'], mask)) + mesh_handler = Utils.openCdf('mesh_hgr.nc') + Siasiesiv.e1t = np.asfortranarray(mesh_handler.variables['e1t'][0, :]) + Siasiesiv.e2t = np.asfortranarray(mesh_handler.variables['e2t'][0, :]) + Siasiesiv.gphit = np.asfortranarray(mesh_handler.variables['gphit'][0, :]) + mesh_handler.close() + + return job_list + + def request_data(self): + self.sit = self.request_chunk(ModelingRealms.seaIce, 'sit', self.startdate, self.member, self.chunk) + self.sic = self.request_chunk(ModelingRealms.seaIce, 'sic', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self._declare_var('sivols') + self._declare_var('siareas') + self._declare_var('siextents') + + self._declare_var('sivoln') + self._declare_var('siarean') + self._declare_var('siextentn') + + def _declare_var(self, var_name): + self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name, self.startdate, self.member, + self.chunk, region=self.basin.fullname) + + def compute(self): + """ + Runs the diagnostic + """ + sit_handler = Utils.openCdf(self.sit.local_file) + sit = np.asfortranarray(sit_handler.variables['sit'][:]) + timesteps = sit_handler.dimensions['time'].size + sit_handler.close() + + sic_handler = Utils.openCdf(self.sic.local_file) + Utils.convert_units(sic_handler.variables['sic'], '1.0') + sic = np.asfortranarray(sic_handler.variables['sic'][:]) + sic_handler.close() + + result = np.empty((8, timesteps)) + for t in range(0, timesteps): + result[:, t] = cdftoolspython.icediag.icediags(Siasiesiv.e1t, Siasiesiv.e2t, self.mask, + Siasiesiv.gphit, sit[t, :], sic[t, :]) + + self._extract_variable_and_rename(result[4, :], 'sivols', '10^9 m3') + self._extract_variable_and_rename(result[5, :], 'siareas', '10^9 m2') + self._extract_variable_and_rename(result[7, :], 'siextents', '10^9 m2') + + self._extract_variable_and_rename(result[0, :], 'sivoln', '10^9 m3') + self._extract_variable_and_rename(result[1, :], 'siarean', '10^9 m2') + self._extract_variable_and_rename(result[3, :], 'siextentn', '10^9 m2') + + def _extract_variable_and_rename(self, values, cmor_name, units): + temp = TempFile.get() + reference_handler = Utils.openCdf(self.sit.local_file) + os.remove(temp) + handler = netCDF4.Dataset(temp, 'w') + + Utils.copy_variable(reference_handler, handler, 'time', add_dimensions=True) + Utils.copy_variable(reference_handler, handler, 'time_bnds', add_dimensions=True, must_exist=False) + Utils.copy_variable(reference_handler, handler, 'leadtime', add_dimensions=True, must_exist=False) + reference_handler.close() + + new_var = handler.createVariable(cmor_name, float, 'time', fill_value=1.0e20) + new_var.units = units + new_var.short_name = cmor_name + new_var.valid_min = 0.0 + new_var[:] = values + new_var.valid_max = np.max(values) + handler.close() + self.generated[cmor_name].set_local_file(temp) + diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py new file mode 100644 index 0000000000000000000000000000000000000000..9cd9ab6101875ea64c2f3c2a788ff423fa23fdae --- /dev/null +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -0,0 +1,117 @@ +# coding=utf-8 +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticVariableOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class VerticalGradient(Diagnostic): + """ + Chooses vertical level in ocean, or vertically averages between + 2 or more ocean levels + + :original author: Virginie Guemas + :contributor: Eleftheria Exarchou + :contributor: Javier Vegas-Regidor + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + :param box: box used to restrict the vertical mean + :type box: Box + """ + + alias = 'vgrad' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, variable, box): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.box = box + self.required_vars = [variable] + self.generated_vars = [variable + 'vmean'] + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.box == other.box and self.variable == other.variable + + def __str__(self): + return 'Vertical gradient Startdate: {0} Member: {1} Chunk: {2} Variable: {3} ' \ + 'Box: {4}'.format(self.startdate, self.member, self.chunk, self.variable, self.box) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (level), maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticIntOption('upper_level', 1), + DiagnosticIntOption('low_level', 2)) + options = cls.process_options(options, options_available) + + box = Box(False) + if options['upper_level'] >= 0: + box.min_depth = options['upper_level'] + if options['low_level'] >= 0: + box.max_depth = options['low_level'] + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(VerticalGradient(diags.data_manager, startdate, member, chunk, + options['variable'], box)) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, + self.member, self.chunk) + + def declare_data_generated(self): + self.gradient_file = self.declare_chunk(ModelingRealms.ocean, self.variable + 'vgrad', + self.startdate, self.member, self.chunk, box=self.box) + + def compute(self): + """ + Runs the diagnostic + """ + handler = Utils.openCdf(self.variable_file) + if 'lev' not in handler.dimensions: + raise Exception('Variable {0} does not have a level dimension') + var_handler = handler.variables[self.variable] + upper_level = var_handler[:, self.box.min_depth-1, ...] + lower_level = var_handler[:, self.box.max_depth-1, ...] + gradient = upper_level - lower_level + + temp = TempFile.get() + new_file = Utils.openCdf(temp, 'w') + for var in handler.variables.keys(): + if var in (self.variable, 'lev', 'lev_bnds'): + continue + Utils.copy_variable(handler, new_file, var, add_dimensions=True) + new_var = new_file.createVariable(self.variable + 'vgrad', var_handler.dtype, + dimensions=('time', 'j', 'i'), zlib=True) + Utils.copy_attributes(new_var, var_handler) + new_var[...] = gradient[...] + new_var.long_name += ' Vertical gradient' + new_var.standard_name += '_vertical_gradient' + + self.gradient_file.set_local_file(temp) + diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py new file mode 100644 index 0000000000000000000000000000000000000000..21bd4a88bc171a15fa6358364a23f99264b03328 --- /dev/null +++ b/earthdiagnostics/ocean/verticalmean.py @@ -0,0 +1,113 @@ +# coding=utf-8 +from earthdiagnostics import cdftools +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticVariableOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class VerticalMean(Diagnostic): + """ + Chooses vertical level in ocean, or vertically averages between + 2 or more ocean levels + + :original author: Virginie Guemas + :contributor: Eleftheria Exarchou + :contributor: Javier Vegas-Regidor + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + :param box: box used to restrict the vertical mean + :type box: Box + """ + + alias = 'vertmean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, variable, box): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.box = box + self.required_vars = [variable] + self.generated_vars = [variable + 'vmean'] + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.box == other.box and self.variable == other.variable + + def __str__(self): + return 'Vertical mean Startdate: {0} Member: {1} Chunk: {2} Variable: {3} ' \ + 'Box: {4}'.format(self.startdate, self.member, self.chunk, self.variable, self.box) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (level), maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticIntOption('min_depth', -1), + DiagnosticIntOption('max_depth', -1)) + options = cls.process_options(options, options_available) + + box = Box() + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(VerticalMean(diags.data_manager, startdate, member, chunk, + options['variable'], box)) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, + self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(ModelingRealms.ocean, self.variable + 'vmean', self.startdate, self.member, + self.chunk, box=self.box) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + handler = Utils.openCdf(self.variable_file.local_file) + if self.box.min_depth is None: + lev_min = handler.variables['lev'][0] + else: + lev_min = self.box.min_depth + + if self.box.max_depth is None: + lev_max = handler.variables['lev'][-1] + else: + lev_max = self.box.max_depth + handler.close() + + cdftools.run('cdfvertmean', input=self.variable_file.local_file, output=temp, + options=[self.variable, 'T', lev_min, lev_max, '-debug']) + Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) + self.results.set_local_file(temp, rename_var='{0}_vert_mean'.format(self.variable)) + diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py new file mode 100644 index 0000000000000000000000000000000000000000..8951f493229004fc2ba3da3d7274e3216c9dbb3f --- /dev/null +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -0,0 +1,114 @@ +# coding=utf-8 +from earthdiagnostics import cdftools +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticFloatOption, DiagnosticDomainOption, \ + DiagnosticVariableOption, DiagnosticChoiceOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class VerticalMeanMeters(Diagnostic): + """ + Averages vertically any given variable + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + :param box: box used to restrict the vertical mean + :type box: Box + + """ + + alias = 'vertmeanmeters' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, box, grid_point): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.domain = domain + self.variable = variable + self.box = box + self.grid_point = grid_point + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.box == other.box and self.variable == other.variable + + def __str__(self): + return 'Vertical mean meters Startdate: {0} Member: {1} Chunk: {2} Variable: {3}:{4} ' \ + 'Box: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, self.box) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (meters), maximum depth (meters) + :type options: list[str] + :return: + """ + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFloatOption('min_depth', -1), + DiagnosticFloatOption('max_depth', -1), + DiagnosticDomainOption(default_value=ModelingRealms.ocean), + DiagnosticChoiceOption('grid_point', ('T', 'U', 'V'), 'T')) + options = cls.process_options(options, options_available) + + box = Box(True) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(VerticalMeanMeters(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], box, options['grid_point'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, + self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(self.domain, self.variable + 'vmean', self.startdate, self.member, + self.chunk, box=self.box) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + handler = Utils.openCdf(self.variable_file.local_file) + if self.box.min_depth is None: + lev_min = handler.variables['lev'][0] + else: + lev_min = self.box.min_depth + + if self.box.max_depth is None: + lev_max = handler.variables['lev'][-1] + else: + lev_max = self.box.max_depth + handler.close() + + cdftools.run('cdfvertmean', input=self.variable_file.local_file, output=temp, + options=[self.variable, self.grid_point, lev_min, lev_max, '-debug']) + Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) + self.results.set_local_file(temp, rename_var='{0}_vert_mean'.format(self.variable)) diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py new file mode 100644 index 0000000000000000000000000000000000000000..4b318b2cafabbdac1ef3d3ed24e8e33f07b9ed79 --- /dev/null +++ b/earthdiagnostics/publisher.py @@ -0,0 +1,46 @@ +# coding=utf-8 +class Publisher(object): + """ + Base class to provide functionality to notify updates to other objects + """ + def __init__(self): + self._subscribers = dict() + + def subscribe(self, who, callback=None): + """ + Add a suscriber to the current publisher + + :param who: subscriber to add + :type who: object + :param callback: method to execute when publisher updates + :type callback: callable | NoneType + """ + if callback is None: + callback = getattr(who, 'update') + self._subscribers[who] = callback + + def unsubscribe(self, who): + """ + Removes a suscriber from the current publisher + + :param who: suscriber to remove + :type who: object + """ + del self._subscribers[who] + + def dispatch(self, *args): + """ + Notify update to all the suscribers + + :param args: arguments to pass + """ + for subscriber, callback in self._subscribers.items(): + # noinspection PyCallingNonCallable + callback(*args) + + @property + def suscribers(self): + """ + List of suscribers of this publisher + """ + return self._subscribers.keys() diff --git a/earthdiagnostics/singleton.py b/earthdiagnostics/singleton.py new file mode 100644 index 0000000000000000000000000000000000000000..2cd1bb8d91a945eace966863d1f49184232378bd --- /dev/null +++ b/earthdiagnostics/singleton.py @@ -0,0 +1,8 @@ +# coding=utf-8 +class SingletonType(type): + def __call__(cls, *args): + try: + return cls.__instance + except AttributeError: + cls.__instance = super(SingletonType, cls).__call__(*args) + return cls.__instance diff --git a/earthdiagnostics/statistics/__init__.py b/earthdiagnostics/statistics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2424b99360092efc33d5a92b8d03c888a3882ea1 --- /dev/null +++ b/earthdiagnostics/statistics/__init__.py @@ -0,0 +1,5 @@ +# coding=utf-8 +from monthlypercentile import MonthlyPercentile +from climatologicalpercentile import ClimatologicalPercentile +from daysoverpercentile import DaysOverPercentile +from discretize import Discretize diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..229ad02073a7f156fd3db0064c31b6e8508249ad --- /dev/null +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -0,0 +1,148 @@ +# coding=utf-8 +import iris +import iris.coord_categorisation +import iris.coords +import iris.exceptions +import numpy as np +import six +from bscearth.utils.log import Log + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, DiagnosticDomainOption, \ + DiagnosticIntOption, DiagnosticListIntOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.utils import TempFile +from earthdiagnostics.variable_type import VariableType + + +class ClimatologicalPercentile(Diagnostic): + """ + Calculates the climatological percentiles for the given leadtime + + :param data_manager: data management object + :type data_manager: DataManager + :param variable: variable to average + :type variable: str + :param experiment_config: + :type experiment_config: ExperimentConfig + """ + + alias = 'climpercent' + "Diagnostic alias for the configuration file" + + Percentiles = np.array([0.1, 0.25, 0.33, 0.5, 0.66, 0.75, 0.9]) + + def __init__(self, data_manager, domain, variable, start_year, end_year, + forecast_month, experiment_config): + Diagnostic.__init__(self, data_manager) + self.variable = variable + self.domain = domain + self.experiment_config = experiment_config + + self.start_year = start_year + self.end_year = end_year + self.forecast_month = forecast_month + self.distribution = None + + self.leadtime_files = {} + + def __eq__(self, other): + return self.domain == other.domain and self.variable == other.variable and \ + self.start_year == other.start_year and self.end_year == other.end_year and \ + self.forecast_month == other.forecast_month + + def __str__(self): + return 'Climatological percentile Variable: {0.domain}:{0.variable} Period: {0.start_year}-{0.end_year} ' \ + 'Forecast month: {0.forecast_month}'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain, variable, percentil number, maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticIntOption('start_year'), + DiagnosticIntOption('end_year'), + DiagnosticListIntOption('forecast_month'), + ) + options = cls.process_options(options, options_available) + + job_list = list() + for forecast_month in options['forecast_month']: + job_list.append(ClimatologicalPercentile(diags.data_manager, options['domain'], options['variable'], + options['start_year'], options['end_year'], + forecast_month, diags.config.experiment)) + return job_list + + def requested_startdates(self): + return ['{0}{1:02}01'.format(year, self.forecast_month) for year in range(self.start_year, self.end_year+1)] + + def request_data(self): + for startdate in self.requested_startdates(): + if startdate not in self.leadtime_files: + self.leadtime_files[startdate] = {} + Log.debug('Retrieving startdate {0}', startdate) + self.leadtime_files[startdate] = self.request_chunk(self.domain, '{0}_dis'.format(self.variable), startdate, + None, None, vartype=VariableType.STATISTIC) + + def declare_data_generated(self): + var_name = '{0.variable}prct{0.start_year}{0.forecast_month}-{0.end_year}{0.forecast_month:02d}'.format(self) + self.percentiles_file = self.declare_chunk(self.domain, var_name, None, None, None, + frequency=Frequencies.climatology, vartype=VariableType.STATISTIC) + + def compute(self): + """ + Runs the diagnostic + """ + iris.FUTURE.netcdf_promote = True + self._get_distribution() + percentile_values = self._calculate_percentiles() + self._save_results(percentile_values) + + def _save_results(self, percentile_values): + temp = TempFile.get() + iris.FUTURE.netcdf_no_unlimited = True + iris.save(percentile_values.merge_cube(), temp, zlib=True) + self.percentiles_file.set_local_file(temp, rename_var='percent') + + def _calculate_percentiles(self): + Log.debug('Calculating percentiles') + + bins = self.distribution.coord('bin').points + + def calculate(point_distribution): + cs = np.cumsum(point_distribution) + total = cs[-1] + percentile_values = ClimatologicalPercentile.Percentiles * total + index = np.searchsorted(cs, percentile_values) + return [bins[i] for i in index] + + results = iris.cube.CubeList() + percentile_coord = iris.coords.DimCoord(ClimatologicalPercentile.Percentiles, long_name='percentile') + print(self.distribution) + for leadtime_slice in self.distribution.slices_over('leadtime'): + result = iris.cube.Cube(np.apply_along_axis(calculate, 0, leadtime_slice.data), var_name='percent', + units=self.distribution.coord('bin').units) + result.add_dim_coord(percentile_coord, 0) + result.add_dim_coord(leadtime_slice.coord('latitude'), 1) + result.add_dim_coord(leadtime_slice.coord('longitude'), 2) + result.add_aux_coord(leadtime_slice.coord('leadtime')) + results.append(result) + return results + + def _get_distribution(self): + for startdate, startdate_file in six.iteritems(self.leadtime_files): + Log.info('Getting data for startdate {0}', startdate) + data_cube = iris.load_cube(startdate_file.local_file) + if self.distribution is None: + self.distribution = data_cube + else: + self.distribution += data_cube + if len(self.distribution.coords('leadtime')) == 0: + self.distribution.add_aux_coord(iris.coords.AuxCoord(1, var_name='leadtime', units='months')) diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..aad012a492728d7d2a5c320441dc88c398b79e64 --- /dev/null +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -0,0 +1,243 @@ +# coding=utf-8 +import os + +import iris +import iris.analysis +import iris.coord_categorisation +import iris.coords +import iris.exceptions +import numpy as np +from bscearth.utils.date import parse_date, add_months +from bscearth.utils.log import Log +from iris.time import PartialDateTime + +from earthdiagnostics.diagnostic import * +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile +from earthdiagnostics.utils import Utils, TempFile + + +class DaysOverPercentile(Diagnostic): + """ + Calculates the montlhy percentiles + + :param data_manager: data management object + :type data_manager: DataManager + :param variable: variable to average + :type variable: str + """ + + alias = 'daysover' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, domain, variable, start_year, end_year, startdate, forecast_month): + Diagnostic.__init__(self, data_manager) + self.variable = variable + self.domain = domain + self.start_year = start_year + self.end_year = end_year + self.forecast_month = forecast_month + self.startdate = startdate + + def __eq__(self, other): + return self.startdate == other.startdate and self.domain == other.domain and \ + self.variable == other.variable and self.start_year == other.start_year and \ + self.end_year == other.end_year + + def __str__(self): + return 'Days over percentile Startdate: {0.startdate} Variable: {0.domain}:{0.variable} ' \ + 'Climatology: {0.start_year}-{0.end_year}'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain, variable, percentil number, maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticOption('variable'), + DiagnosticIntOption('start_year'), + DiagnosticIntOption('end_year'), + DiagnosticListIntOption('forecast_month'),) + options = cls.process_options(options, options_available) + + job_list = list() + for startdate in diags.config.experiment.startdates: + for forecast_month in options['forecast_month']: + job_list.append(DaysOverPercentile(diags.data_manager, options['domain'], options['variable'], + options['start_year'], options['end_year'], + startdate, forecast_month)) + return job_list + + def request_data(self): + var_name = '{0.variable}prct{0.start_year}{0.forecast_month}-{0.end_year}{0.forecast_month:02d}'.format(self) + self.percentiles_file = self.request_chunk(self.domain, var_name, None, None, None, + frequency=Frequencies.climatology) + + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, None, None) + + def declare_data_generated(self): + var_over = self.variable + '_daysover_q{0}_{1.start_year}-{1.end_year}' + var_below = self.variable + '_daysbelow_q{0}_{1.start_year}-{1.end_year}' + self.days_over_file = {} + self.days_below_file = {} + for perc in ClimatologicalPercentile.Percentiles: + self.days_over_file[perc] = self.declare_chunk(self.domain, var_over.format(int(perc * 100), self), + self.startdate, None, + None, frequency=Frequencies.monthly, + vartype=VariableType.STATISTIC) + + self.days_below_file[perc] = self.declare_chunk(self.domain, var_below.format(int(perc * 100), self), + self.startdate, None, + None, frequency=Frequencies.monthly, + vartype=VariableType.STATISTIC) + + def compute(self): + """ + Runs the diagnostic + """ + iris.FUTURE.netcdf_promote = True + percentiles = iris.load_cube(self.percentiles_file.local_file) + + handler = Utils.openCdf(self.variable_file.local_file) + if 'realization' in handler.variables: + handler.variables[self.variable].coordinates = 'realization' + handler.close() + var = iris.load_cube(self.variable_file.local_file) + date = parse_date(self.startdate) + lead_date = add_months(date, 1, self.data_manager.config.experiment.calendar) + leadtimes = {1: PartialDateTime(lead_date.year, lead_date.month, lead_date.day)} + + def assign_leadtime(coord, x): + # noinspection PyBroadException + try: + leadtime_month = 1 + partial_date = leadtimes[leadtime_month] + while coord.units.num2date(x) >= partial_date: + leadtime_month += 1 + try: + partial_date = leadtimes[leadtime_month] + except KeyError: + new_date = add_months(date, leadtime_month, self.data_manager.config.experiment.calendar) + partial_date = PartialDateTime(new_date.year, new_date.month, new_date.day) + leadtimes[leadtime_month] = partial_date + return leadtime_month + except Exception: + pass + iris.coord_categorisation.add_categorised_coord(var, 'leadtime', 'time', assign_leadtime) + iris.coord_categorisation.add_year(var, 'time') + iris.coord_categorisation.add_day_of_year(var, 'time') + try: + realization_coord = var.coord('realization') + except iris.exceptions.CoordinateNotFoundError: + realization_coord = None + self.lat_coord = var.coord('latitude') + self.lon_coord = var.coord('longitude') + results_over = {perc: iris.cube.CubeList() for perc in ClimatologicalPercentile.Percentiles} + results_below = {perc: iris.cube.CubeList() for perc in ClimatologicalPercentile.Percentiles} + + var_daysover = 'days_over' + var_days_below = 'days_below' + long_name_days_over = 'Proportion of days over a given percentile for {0.start_year}-{0.end_year} ' \ + 'climatology'.format(self) + long_name_days_below = 'Proportion of days below a given percentile for {0.start_year}-{0.end_year} ' \ + 'climatology'.format(self) + + for leadtime in leadtimes.keys(): + Log.debug('Computing startdate {0} leadtime {1}', self.startdate, leadtime) + leadtime_slice = var.extract(iris.Constraint(leadtime=leadtime)) + if len(percentiles.coords('leadtime')) > 0: + percentiles_leadtime = percentiles.extract(iris.Constraint(leadtime=leadtime)) + else: + percentiles_leadtime = percentiles + time_coord = iris.coords.AuxCoord.from_coord(leadtime_slice.coord('time')) + first_time = time_coord.points[0] + last_time = time_coord.points[-1] + timesteps = leadtime_slice.coord('time').shape[0] + time_coord = time_coord.copy(first_time + (last_time - first_time) / 2, (first_time, last_time)) + for percentile_slice in percentiles_leadtime.slices_over('percentile'): + percentile = percentile_slice.coord('percentile').points[0] + + # noinspection PyTypeChecker + days_over = np.sum(leadtime_slice.data > percentile_slice.data, 0) / float(timesteps) + result = self.create_results_cube(days_over, percentile, realization_coord, + time_coord, var_daysover, long_name_days_over) + results_over[percentile].append(result) + + # noinspection PyTypeChecker + days_below = np.sum(leadtime_slice.data < percentile_slice.data, 0) / float(timesteps) + result = self.create_results_cube(days_below, percentile, realization_coord, + time_coord, var_days_below, long_name_days_below) + results_below[percentile].append(result) + + Log.debug('Saving percentiles startdate {0}', self.startdate) + for perc in ClimatologicalPercentile.Percentiles: + iris.FUTURE.netcdf_no_unlimited = True + self.days_over_file[perc].set_local_file(self.save_to_file(perc, results_over, var_daysover), + rename_var=var_daysover) + self.days_below_file[perc].set_local_file(self.save_to_file(perc, results_below, var_days_below), + rename_var=var_days_below) + + del self.days_over_file + del self.days_below_file + del self.lat_coord + del self.lon_coord + + def save_to_file(self, perc, results_over, var_daysover): + temp = TempFile.get() + iris.save(results_over[perc].merge_cube(), temp, zlib=True, unlimited_dimensions=['time']) + Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, + must_exist=False, rename_dimension=True) + handler = Utils.openCdf(temp) + if 'time' not in handler.dimensions: + new_file = TempFile.get() + new_handler = Utils.openCdf(new_file, 'w') + + new_handler.createDimension('time', 1) + for dimension in handler.dimensions: + Utils.copy_dimension(handler, new_handler, dimension) + + for variable in handler.variables.keys(): + if variable in (var_daysover, 'time', 'time_bnds'): + continue + Utils.copy_variable(handler, new_handler, variable) + old_var = handler.variables[var_daysover] + new_var = new_handler.createVariable(var_daysover, old_var.dtype, ('time',) + old_var.dimensions, + zlib=True, fill_value=1.0e20) + Utils.copy_attributes(new_var, old_var) + new_var[0, :] = old_var[:] + + old_var = handler.variables['time'] + new_var = new_handler.createVariable('time', old_var.dtype, ('time',)) + Utils.copy_attributes(new_var, old_var) + new_var[0] = old_var[0] + + old_var = handler.variables['time_bnds'] + new_var = new_handler.createVariable('time_bnds', old_var.dtype, ('time',) + old_var.dimensions) + Utils.copy_attributes(new_var, old_var) + new_var[0, :] = old_var[:] + + new_handler.close() + os.remove(temp) + temp = new_file + handler.close() + return temp + + def create_results_cube(self, days_over, percentile, realization_coord, time_coord, + var_name, long_name): + result = iris.cube.Cube(days_over.astype(np.float32), var_name=var_name, long_name=long_name, units=1.0) + if realization_coord is not None: + result.add_aux_coord(realization_coord, 0) + result.add_dim_coord(self.lat_coord, 1) + result.add_dim_coord(self.lon_coord, 2) + else: + result.add_dim_coord(self.lat_coord, 0) + result.add_dim_coord(self.lon_coord, 1) + result.add_aux_coord(iris.coords.AuxCoord(percentile, long_name='percentile')) + result.add_aux_coord(time_coord) + return result diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py new file mode 100644 index 0000000000000000000000000000000000000000..81ed8232b578f0e4a682b0c3972c1a1fb50fc485 --- /dev/null +++ b/earthdiagnostics/statistics/discretize.py @@ -0,0 +1,262 @@ +# coding=utf-8 +import math + +import iris +import iris.coord_categorisation +import iris.coords +import iris.exceptions +import iris.unit +import numpy as np +import psutil +import six +from bscearth.utils.date import parse_date, add_months, add_days +from bscearth.utils.log import Log +from iris.cube import Cube +from iris.time import PartialDateTime + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, DiagnosticDomainOption, \ + DiagnosticIntOption, DiagnosticFloatOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.variable_type import VariableType + + +class Discretize(Diagnostic): + """ + Discretizes a variable + + :param data_manager: data management object + :type data_manager: DataManager + :param variable: variable to average + :type variable: str + """ + + alias = 'discretize' + "Diagnostic alias for the configuration file" + + Percentiles = np.array([0.1, 0.25, 0.33, 0.5, 0.66, 0.75, 0.9]) + + def __init__(self, data_manager, startdate, domain, variable, num_bins, min_value, max_value): + Diagnostic.__init__(self, data_manager) + + self.startdate = startdate + self.variable = variable + self.domain = domain + + self.realizations = None + self.num_bins = num_bins + self._bins = None + self.cmor_var = data_manager.variable_list.get_variable(variable, silent=True) + + if not math.isnan(min_value): + self.min_value = min_value + self.check_min_value = False + elif self.cmor_var and self.cmor_var.valid_min: + self.min_value = float(self.cmor_var.valid_min) + self.check_min_value = False + else: + self.min_value = None + self.check_min_value = True + + if not math.isnan(max_value): + self.max_value = max_value + self.check_max_value = False + elif self.cmor_var and self.cmor_var.valid_min: + self.max_value = float(self.cmor_var.valid_max) + self.check_max_value = False + else: + self.max_value = None + self.check_max_value = True + + self.process = psutil.Process() + + def print_memory_used(self): + Log.debug('Memory: {0:.2f} GB'.format(self.process.memory_info().rss / 1024.0**3)) + + @property + def bins(self): + if self._bins is None: + return self.num_bins + return self._bins + + @bins.setter + def bins(self, value): + self._bins = value + + def __eq__(self, other): + return self.domain == other.domain and self.variable == other.variable and self.num_bins == other.num_bins and \ + self.min_value == other.min_value and self.max_value == other.max_value and \ + self.startdate == other.startdate + + def __str__(self): + return 'Discretizing variable: {0.domain}:{0.variable} Startdate: {0.startdate} ' \ + 'Bins: {0.num_bins} Range: [{0.min_value}, {0.max_value}]'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain, variable, percentil number, maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticIntOption('bins', 2000), + DiagnosticFloatOption('min_value', float('nan')), + DiagnosticFloatOption('max_value', float('nan')), + ) + options = cls.process_options(options, options_available) + + job_list = list() + for startdate in diags.config.experiment.startdates: + job_list.append(Discretize(diags.data_manager, startdate, options['domain'], options['variable'], + options['bins'], options['min_value'], options['max_value'])) + return job_list + + def request_data(self): + self.original_data = self.request_chunk(self.domain, self.variable, self.startdate, None, None) + + def declare_data_generated(self): + var_name = '{0.variable}_dis'.format(self) + self.discretized_data = self.declare_chunk(self.domain, var_name, self.startdate, None, None, + vartype=VariableType.STATISTIC) + + def compute(self): + """ + Runs the diagnostic + """ + self.print_memory_used() + iris.FUTURE.netcdf_promote = True + self._load_cube() + self.print_memory_used() + self._get_value_interval() + self.print_memory_used() + Log.info('Range: [{0}, {1}]', self.min_value, self.max_value) + self._get_distribution() + self.print_memory_used() + self._save_results() + self.print_memory_used() + del self.distribution + del self.data_cube + self.print_memory_used() + + def _load_cube(self): + + handler = Utils.openCdf(self.original_data.local_file) + if 'realization' in handler.variables: + handler.variables[self.variable].coordinates = 'realization' + handler.close() + data_cube = iris.load_cube(self.original_data.local_file) + + date = parse_date(self.startdate) + lead_date = add_months(date, 1, self.data_manager.config.experiment.calendar) + leadtimes = {1: PartialDateTime(lead_date.year, lead_date.month, lead_date.day)} + + def assign_leadtime(coord, x): + leadtime_month = 1 + partial_date = leadtimes[leadtime_month] + while coord.units.num2date(x) >= partial_date: + leadtime_month += 1 + try: + partial_date = leadtimes[leadtime_month] + except KeyError: + new_date = add_months(date, leadtime_month, self.data_manager.config.experiment.calendar) + partial_date = PartialDateTime(new_date.year, new_date.month, new_date.day) + leadtimes[leadtime_month] = partial_date + return leadtime_month + + iris.coord_categorisation.add_categorised_coord(data_cube, 'leadtime', 'time', assign_leadtime) + self.data_cube = data_cube + + def _save_results(self): + Log.debug('Saving results...') + + bins = np.zeros(self.num_bins) + bins_bounds = np.zeros((self.num_bins, 2)) + + for x in range(self.num_bins): + bins[x] = (self.bins[x+1] - self.bins[x]) / 2 + self.bins[x] + bins_bounds[x, 0] = self.bins[x] + bins_bounds[x, 1] = self.bins[x+1] + + bins_coord = iris.coords.DimCoord(bins, var_name='bin', units=self.data_cube.units, bounds=bins_bounds) + + cubes = iris.cube.CubeList() + + date = parse_date(self.startdate) + date = add_days(date, 14, self.data_manager.config.experiment.calendar) + + for leadtime, distribution in six.iteritems(self.distribution): + leadtime_cube = Cube(distribution.astype(np.uint32), var_name=self.data_cube.var_name, + standard_name=self.data_cube.standard_name, units='1') + leadtime_cube.add_dim_coord(bins_coord, 0) + leadtime_cube.add_dim_coord(self.data_cube.coord('latitude'), 1) + leadtime_cube.add_dim_coord(self.data_cube.coord('longitude'), 2) + leadtime_cube.add_aux_coord(iris.coords.AuxCoord(leadtime, + var_name='leadtime', + units='months')) + lead_date = add_months(date, leadtime - 1, self.data_manager.config.experiment.calendar) + leadtime_cube.add_aux_coord(iris.coords.AuxCoord(iris.unit.date2num(lead_date, + unit='days since 1950-01-01', + calendar="standard"), + var_name='time', + units='days since 1950-01-01')) + + cubes.append(leadtime_cube) + temp = TempFile.get() + iris.FUTURE.netcdf_no_unlimited = True + iris.save(cubes.merge_cube(), temp, zlib=True) + self.discretized_data.set_local_file(temp, rename_var=self.data_cube.var_name) + + def _get_distribution(self): + self.distribution = {} + Log.debug('Discretizing...') + for leadtime in set(self.data_cube.coord('leadtime').points): + Log.debug('Discretizing leadtime {0}', leadtime) + leadtime_cube = self.data_cube.extract(iris.Constraint(leadtime=leadtime)) + if 'realization' in leadtime_cube.coords(): + for realization_cube in self.data_cube.slices_over('realization'): + Log.debug('Discretizing realization {0}', realization_cube.coord('realization').points[0]) + self.print_memory_used() + if leadtime not in self.distribution: + self.distribution[leadtime] = self._calculate_distribution(realization_cube) + else: + self.distribution[leadtime] += self._calculate_distribution(realization_cube) + else: + self.print_memory_used() + self.distribution[leadtime] = self._calculate_distribution(leadtime_cube) + + # noinspection PyTypeChecker + def _get_value_interval(self): + if self.check_min_value or self.check_max_value: + Log.debug('Calculating max and min values...') + for time_slice in self.data_cube.slices_over('time'): + if self.check_min_value: + file_min = np.amin(time_slice.data) + if self.min_value is None: + self.min_value = file_min + self.min_value = min(self.min_value, file_min) + + if self.check_max_value: + file_max = np.amax(time_slice.data) + self.max_value = max(self.max_value, file_max) + + def _calculate_distribution(self, data_cube): + def calculate_histogram(time_series): + histogram, self.bins = np.histogram(time_series, bins=self.bins, + range=(self.min_value, self.max_value)) + return histogram + + return np.apply_along_axis(calculate_histogram, 0, data_cube.data) + + + + + + + + + diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..1ecb5c44276b68650216a0e79a7d239c13a499a6 --- /dev/null +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -0,0 +1,161 @@ +# coding=utf-8 +from bscearth.utils.log import Log + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.variable_type import VariableType +from calendar import monthrange + + +class MonthlyPercentile(Diagnostic): + """ + Calculates the montlhy percentiles + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + """ + + alias = 'monpercent' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, percentiles): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.percentiles = percentiles + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.percentiles == other.percentiles + + def __str__(self): + return 'Monthly percentile Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Percentiles: {5}'.format(self.startdate, self.member, self.chunk, + self.domain, self.variable, + ', '.join(str(i) for i in self.percentiles)) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain, variable, percentil number, maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticOption('variable'), + DiagnosticListIntOption('percentiles', [], 0, 100)) + options = cls.process_options(options, options_available) + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(MonthlyPercentile(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['percentiles'])) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.max_file = self.declare_chunk(self.domain, self.variable_max, self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, vartype=VariableType.STATISTIC) + self.min_file = self.declare_chunk(self.domain, self.variable_min, self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, vartype=VariableType.STATISTIC) + self.percentile_file = {} + for percentile in self.percentiles: + self.percentile_file[percentile] = self.declare_chunk(self.domain, self.percentile(percentile), + self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, + vartype=VariableType.STATISTIC) + + self.declare_chunk(self.domain, '{0}_q{1}'.format(self.variable, percentile), self.startdate, + self.member, self.chunk, frequency=Frequencies.monthly, vartype=VariableType.STATISTIC) + + @property + def variable_max(self): + return '{0}max'.format(self.variable) + + @property + def variable_min(self): + return '{0}min'.format(self.variable) + + def percentile(self, percentile): + return '{0}_q{1}'.format(self.variable, percentile) + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + handler = Utils.openCdf(self.variable_file.local_file) + datetimes = Utils.get_datetime_from_netcdf(handler) + handler.close() + + start_index = 0 + while datetimes[start_index].day != 1 and start_index < handler.size: + start_index += 1 + if start_index == datetimes.size: + raise Exception('No complete month for diagnostic {0}'.format(self)) + + end_index = datetimes.size - 1 + while datetimes[end_index].day != monthrange(datetimes[end_index].year, datetimes[end_index].month)[1] \ + and end_index >= 0: + end_index -= 1 + + if end_index < 0: + raise Exception('No complete month for diagnostic {0}'.format(self)) + + if start_index != 0 or end_index != datetimes.size - 1: + start_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[start_index]) + end_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[end_index]) + Utils.cdo.seldate('{0},{1}'.format(start_date, end_date), input=self.variable_file.local_file, output=temp) + Utils.rename_variable(temp, 'lev', 'ensemble', False, True) + else: + Utils.copy_file(self.variable_file.local_file, temp) + + Log.debug('Computing minimum') + monmin_file = TempFile.get() + Utils.cdo.monmin(input=temp, output=monmin_file) + + Log.debug('Computing maximum') + monmax_file = TempFile.get() + Utils.cdo.monmax(input=temp, output=monmax_file) + + for percentile in self.percentiles: + Log.debug('Computing percentile {0}', percentile) + Utils.cdo.monpctl(str(percentile), input=[temp, monmin_file, monmax_file], output=temp) + Utils.rename_variable(temp, 'lev', 'ensemble', False, True) + handler = Utils.openCdf(monmax_file) + handler.variables[self.variable].long_name += ' {0} Percentile'.format(percentile) + handler.close() + self.percentiles[percentile].set_local_file(temp, rename_var=self.variable) + + Utils.rename_variable(monmax_file, 'lev', 'ensemble', False, True) + handler = Utils.openCdf(monmax_file) + handler.variables[self.variable].long_name += ' Monthly Maximum' + handler.close() + self.max_file.set_local_file(monmax_file, rename_var=self.variable) + + Utils.rename_variable(monmin_file, 'lev', 'ensemble', False, True) + handler = Utils.openCdf(monmin_file) + handler.variables[self.variable].long_name += ' Monthly Minimum' + handler.close() + self.min_file.set_local_file(monmin_file, rename_var=self.variable) + + + diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py new file mode 100644 index 0000000000000000000000000000000000000000..8637121a55d0595b3e008cdf5c47675d0a7bbe29 --- /dev/null +++ b/earthdiagnostics/threddsmanager.py @@ -0,0 +1,392 @@ +# coding=utf-8 +import os +from time import strptime + +import iris +import netCDF4 +import numpy as np +from bscearth.utils.date import parse_date, add_months, chunk_start_date, chunk_end_date +from bscearth.utils.log import Log +from iris.coords import DimCoord +from cf_units import Unit + +from datafile import DataFile, StorageStatus, LocalStatus +from earthdiagnostics.datamanager import DataManager +from earthdiagnostics.utils import TempFile, Utils +from datetime import datetime + +from earthdiagnostics.variable_type import VariableType + + +class THREDDSManager(DataManager): + """ + Data manager class for CMORized experiments + """ + def __init__(self, config): + super(THREDDSManager, self).__init__(config) + self.server_url = config.thredds.server_url + data_folders = self.config.data_dir.split(':') + self.config.data_dir = None + for data_folder in data_folders: + if os.path.isdir(os.path.join(data_folder, self.config.data_type, self.experiment.institute.lower(), + self.experiment.model.lower())): + self.config.data_dir = data_folder + break + + if not self.config.data_dir: + raise Exception('Can not find model data') + + if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size != 1: + raise Exception('For obs and recon data chunk_size must be always 1') + + # noinspection PyUnusedLocal + def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, vartype=VariableType.MEAN): + + aggregation_path = self.get_var_url(variable, startdate, frequency, None, vartype) + startdate = parse_date(startdate) + start_chunk = chunk_start_date(startdate, self.experiment.num_chunks, self.experiment.chunk_size, + 'month', self.experiment.calendar) + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + + thredds_subset = THREDDSSubset(aggregation_path, "", variable, startdate, end_chunk) + selected_months = ','.join([str(add_months(startdate, i, self.experiment.calendar).month) for i in leadtimes]) + temp = TempFile.get() + if self.config.data_type == 'exp': + select_months = '-selmonth,{0} {1}'.format(selected_months, thredds_subset) + selected_years = ','.join([str(add_months(startdate, i, self.experiment.calendar).year) for i in leadtimes]) + Utils.cdo.selyear(selected_years, input=select_months, output=temp) + else: + Utils.cdo.selmonth(selected_months, input=thredds_subset, output=temp) + return temp + + # noinspection PyUnusedLocal + def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param domain: CMOR domain + :type domain: str + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) + + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', + self.experiment.calendar) + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + + thredds_subset = THREDDSSubset(aggregation_path, "", var, start_chunk, end_chunk) + return thredds_subset + + def get_file_path(self, startdate, domain, var, frequency, vartype, + box=None, grid=None): + """ + Returns the path to a concrete file + :param startdate: file's startdate + :type startdate: str + :param domain: file's domain + :type domain: str + :param var: file's var + :type var: str + :param frequency: file's frequency + :type frequency: Frequency + :param box: file's box + :type box: Box + :param grid: file's grid + :type grid: str + :return: path to the file + :rtype: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + """ + if not frequency: + frequency = self.config.frequency + var = self._get_final_var_name(box, var) + + folder_path = self._get_folder_path(frequency, domain, var, grid, vartype) + file_name = self._get_file_name(var, startdate) + + filepath = os.path.join(folder_path, file_name) + return filepath + + def _get_folder_path(self, frequency, domain, variable, grid, vartype): + + if self.config.data_type == 'exp': + var_folder = self.get_varfolder(domain, variable, grid) + else: + var_folder = variable + + folder_path = os.path.join(self.config.data_dir, self.config.data_type, + self.experiment.institute.lower(), + self.experiment.model.lower(), + frequency.folder_name(vartype), + var_folder) + return folder_path + + # noinspection PyUnusedLocal + def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN): + """ + Ge a file containing all the data for one year for one variable + :param domain: variable's domain + :type domain: str + :param var: variable's name + :type var: str + :param startdate: startdate to retrieve + :type startdate: str + :param member: member to retrieve + :type member: int + :param year: year to retrieve + :type year: int + :param grid: variable's grid + :type grid: str + :param box: variable's box + :type box: Box + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: + """ + aggregation_path = self.get_var_url(var, startdate, None, box, vartype) + thredds_subset = THREDDSSubset(aggregation_path, "", var, datetime(year, 1, 1), datetime(year+1, 1, 1)) + return thredds_subset.download() + + def get_var_url(self, var, startdate, frequency, box, vartype): + """ + Get url for dataset + :param var: variable to retrieve + :type var: str + :param startdate: startdate to retrieve + :type startdate: str + :param frequency: frequency to get: + :type frequency: Frequency | None + :param box: box to get + :type box: Box + :param vartype: type of variable + :type vartype: VariableType + :return: + """ + if not frequency: + frequency = self.config.frequency + var = self._get_final_var_name(box, var) + full_path = os.path.join(self.server_url, 'dodsC', self.config.data_type, self.experiment.institute, + self.experiment.model, frequency.folder_name(vartype)) + if self.config.data_type == 'exp': + full_path = os.path.join(full_path, var, self._get_file_name(var, startdate)) + else: + full_path = os.path.join(full_path, self._get_file_name(var, None)) + return full_path + + def _get_file_name(self, var, startdate): + if startdate: + if self.config.data_type != 'exp': + startdate = startdate[0:6] + return '{0}_{1}.nc'.format(var, startdate) + else: + return '{0}.nc'.format(var) + + def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, + frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): + """ + Creates the link of a given file from the CMOR repository. + + :param cmor_var: + :param move_old: + :param date_str: + :param year: if frequency is yearly, this parameter is used to give the corresponding year + :type year: int + :param domain: CMOR domain + :type domain: str + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + :param cmor_var: variable instance describing the selected variable + :type cmor_var: Variable + """ + # THREDDSManager does not require links + pass + + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param vartype: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :rtype: str + """ + aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) + file_path = self.get_file_path(startdate, domain, var, frequency, vartype, box=box) + + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', + self.experiment.calendar) + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + + thredds_subset = THREDDSSubset(aggregation_path, file_path, var, start_chunk, end_chunk) + thredds_subset.local_status = LocalStatus.PENDING + self.requested_files[file_path] = thredds_subset + return thredds_subset + + # noinspection PyUnusedLocal + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param diagnostic: + :param region: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) + file_path = self.get_file_path(startdate, domain, var, frequency, vartype, box=box) + + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', + self.experiment.calendar) + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + final_name = self._get_final_var_name(box, var) + if file_path in self.requested_files: + thredds_subset = self.requested_files[file_path] + else: + thredds_subset = THREDDSSubset(aggregation_path, file_path, var, start_chunk, end_chunk) + self.requested_files[file_path] = thredds_subset + + thredds_subset.final_name = final_name + thredds_subset.diagnostic = diagnostic + thredds_subset.storage_status = StorageStatus.PENDING + return thredds_subset + + +class THREDDSError(Exception): + pass + + +class THREDDSSubset(DataFile): + def __init__(self, thredds_path, file_path, var, start_time, end_time): + """ + + :param thredds_path: + :param file_path: + :param var: + :type var: str + :param start_time: + :param end_time: + """ + super(THREDDSSubset, self).__init__() + self.thredds_path = thredds_path + self.remote_file = file_path + if '_f' in var: + self.var = var[:var.index('_f')] + self.hourly = var[var.index('_f'):] + else: + self.var = var + self.hourly = '' + self.dimension_indexes = {} + self.handler = None + self.start_time = start_time + self.end_time = end_time + + def __str__(self): + return 'THREDDS {0.thredds_path} ({0.start_time}-{0.end_time})'.format(self) + + def download(self): + try: + Log.debug('Downloading thredds subset {0}...', self) + iris.FUTURE.netcdf_promote = True + iris.FUTURE.netcdf_no_unlimited = True + with iris.FUTURE.context(cell_datetime_objects=True): + time_constraint = iris.Constraint(time=lambda cell: self.start_time <= cell.point <= self.end_time) + var_cube = iris.load_cube(self.thredds_path, constraint=time_constraint, callback=self._correct_cube) + + if not self.local_file: + self.local_file = TempFile.get() + iris.save(var_cube, self.local_file, zlib=True) + if not Utils.check_netcdf_file(self.local_file): + raise Exception('netcdf check for downloaded file failed') + Log.info('Request {0} ready!', self) + self.local_status = LocalStatus.READY + except Exception as ex: + Log.error('Can not retrieve {0} from server: {1}'.format(self, ex)) + self.local_status = LocalStatus.FAILED + + # noinspection PyUnusedLocal,PyMethodMayBeStatic + def _correct_cube(self, cube, field, filename): + if not cube.coords('time'): + return + time = cube.coord('time') + if time.units.origin.startswith('month'): + ref = strptime(time.units.origin[time.units.origin.index(' since ') + 7:], '%Y-%m-%d %H:%M:%S') + helper = np.vectorize(lambda x: datetime(year=ref.tm_year + int(x) / 12, + month=int(x-1) % 12 + 1, + day=ref.tm_mday)) + times = np.round(time.points + ref.tm_mon) + dates = helper(times) + dates = netCDF4.date2num(dates, units='days since 1850-01-01', calendar=time.units.calendar) + new_time = DimCoord(dates, standard_name=time.standard_name, long_name=time.long_name, + var_name=time.var_name, attributes=time.attributes, + units=Unit('days since 1850-01-01', time.units.calendar)) + [dimension] = cube.coord_dims(time) + cube.remove_coord(time) + cube.add_dim_coord(new_time, dimension) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..42304aa02ed840edb038165a8b7a4fc8757d0990 --- /dev/null +++ b/earthdiagnostics/utils.py @@ -0,0 +1,770 @@ +# coding=utf-8 +import datetime +import os +import re +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +from contextlib import contextmanager + +import iris +import iris.exceptions +import netCDF4 +import numpy as np +import six +import xxhash +from bscearth.utils.log import Log +from cdo import Cdo +from cfunits import Units +from nco import Nco + +from earthdiagnostics.constants import Basins + + +@contextmanager +def suppress_stdout(): + with open(os.devnull, "w") as devnull: + old_stdout = sys.stdout + sys.stdout = devnull + try: + yield + finally: + sys.stdout = old_stdout + + +class File(object): + pass + + +class Utils(object): + """ + Container class for miscellaneous utility methods + """ + + nco = Nco() + """An instance of Nco class ready to be used""" + cdo = Cdo() + """An instance of Cdo class ready to be used""" + + @staticmethod + def get_mask(basin): + """ + Returns a numpy array containing the mask for the given basin + + :param basin: basin to retrieve + :type basin: Basin + :return: mask + :rtype: numpy.array + """ + + basin = Basins().parse(basin) + if basin != Basins().Global: + try: + mask_handler = Utils.openCdf('mask_regions.nc') + mask = mask_handler.variables[basin.name][:, 0, :] + mask_handler.close() + except IOError: + raise Exception('File mask.regions.nc is required for basin {0}'.format(basin)) + else: + mask_handler = Utils.openCdf('mask.nc') + mask = mask_handler.variables['tmask'][0, 0, :] + mask_handler.close() + return mask + + @staticmethod + def setminmax(filename, variable_list): + """ + Sets the valid_max and valid_min values to the current max and min values on the file + :param filename: path to file + :type filename: str + :param variable_list: list of variables in which valid_min and valid_max will be set + :type variable_list: str | list + """ + # noinspection PyTypeChecker + if isinstance(variable_list, six.string_types): + variable_list = variable_list.split() + + Log.info('Getting max and min values for {0}', ' '.join(variable_list)) + + handler = Utils.openCdf(filename) + for variable in variable_list: + var = handler.variables[variable] + values = [np.max(var), np.min(var)] + Utils.nco.ncatted(input=filename, output=filename, + options=('-h -a valid_max,{0},m,f,{1}'.format(variable, values[0]),)) + Utils.nco.ncatted(input=filename, output=filename, + options=('-h -a valid_min,{0},m,f,{1}'.format(variable, values[1]),)) + handler.close() + + @staticmethod + def rename_variable(filepath, old_name, new_name, must_exist=True, rename_dimension=False): + """ + Rename multiple variables from a NetCDF file + :param filepath: path to file + :type filepath: str + :param old_name: variable's name to change + :type old_name: str + :param new_name: new name + :type new_name: str + :param must_exist: if True, the function will raise an exception if the variable name does not exist + :type must_exist: bool + :param rename_dimension: if True, also rename dimensions with the same name + :type rename_dimension: bool + """ + Utils.rename_variables(filepath, {old_name: new_name}, must_exist, rename_dimension) + + @staticmethod + def rename_variables(filepath, dic_names, must_exist=True, rename_dimension=False): + """ + Rename multiple variables from a NetCDF file + :param filepath: path to file + :type filepath: str + :param dic_names: dictionary containing old names as keys and new names as values + :type dic_names: dict + :param must_exist: if True, the function will raise an exception if the variable name does not exist + :type must_exist: bool + :param rename_dimension: if True, also rename dimensions with the same name + :type rename_dimension: bool + """ + for old, new in six.iteritems(dic_names): + if old == new: + raise ValueError('{0} original name is the same as the new') + handler = Utils.openCdf(filepath) + + original_names = set(handler.variables.keys()).union(handler.dimensions.keys()) + if not any((True for x in dic_names.keys() if x in original_names)): + handler.close() + if must_exist: + raise Exception("Variables {0} does not exist in file {1}".format(','.join(dic_names.keys()), filepath)) + return + handler.close() + + temp = TempFile.get() + shutil.copyfile(filepath, temp) + + handler = Utils.openCdf(temp) + error = False + + try: + Utils._rename_vars_directly(dic_names, filepath, handler, must_exist, rename_dimension) + except RuntimeError: + error = True + + handler.close() + + if not Utils.check_netcdf_file(temp): + error = True + + if error: + Log.debug('First attemp to rename failed. Using secondary rename method for netCDF') + Utils._rename_vars_by_creating_new_file(dic_names, filepath, temp) + Log.debug('Rename done') + + Utils.move_file(temp, filepath) + + @staticmethod + def check_netcdf_file(filepath): + with suppress_stdout(): + try: + iris.FUTURE.netcdf_promote = True + iris.load(filepath) + except iris.exceptions.IrisError: + return False + return True + + @staticmethod + def get_file_variables(filename): + handler = Utils.openCdf(filename) + variables = handler.variables.keys() + handler.close() + return variables + + @staticmethod + def _rename_vars_by_creating_new_file(dic_names, filepath, temp): + original_handler = Utils.openCdf(filepath) + new_handler = Utils.openCdf(temp, 'w') + for attribute in original_handler.ncattrs(): + original = getattr(original_handler, attribute) + setattr(new_handler, attribute, Utils.convert_to_ASCII_if_possible(original)) + for dimension in original_handler.dimensions.keys(): + Utils.copy_dimension(original_handler, new_handler, dimension, new_names=dic_names) + for variable in original_handler.variables.keys(): + Utils.copy_variable(original_handler, new_handler, variable, new_names=dic_names) + original_handler.close() + new_handler.close() + + # noinspection PyPep8Naming + @staticmethod + def convert_to_ASCII_if_possible(string, encoding='ascii'): + # noinspection PyTypeChecker + if isinstance(string, six.string_types): + try: + return string.encode(encoding) + except UnicodeEncodeError: + if u'Bretonnière' in string: + string = string.replace(u'Bretonnière', 'Bretonniere') + return Utils.convert_to_ASCII_if_possible(string, encoding) + return string + + @staticmethod + def _rename_vars_directly(dic_names, filepath, handler, must_exist, rename_dimension): + for old_name, new_name in dic_names.items(): + if rename_dimension: + if old_name in handler.dimensions: + handler.renameDimension(old_name, new_name) + elif must_exist: + raise Exception("Dimension {0} does not exist in file {1}".format(old_name, filepath)) + + if old_name in handler.variables: + if new_name not in handler.variables: + handler.renameVariable(old_name, new_name) + elif must_exist: + raise Exception("Variable {0} does not exist in file {1}".format(old_name, filepath)) + handler.sync() + + @staticmethod + def copy_file(source, destiny, save_hash=False): + """ + Copies a file from source to destiny, creating dirs if necessary + + :param save_hash: if True, stores hash value in a file + :type save_hash: bool + :param source: path to source + :type source: str + :param destiny: path to destiny + :type destiny: str + """ + dirname_path = os.path.dirname(destiny) + if dirname_path and not os.path.exists(dirname_path): + try: + os.makedirs(dirname_path) + Utils.give_group_write_permissions(dirname_path) + except OSError as ex: + # This can be due to a race condition. If directory already exists, we don have to do nothing + if not os.path.exists(dirname_path): + raise ex + hash_destiny = None + Log.debug('Hashing original file... {0}', datetime.datetime.now()) + hash_original = Utils.get_file_hash(source, use_stored=True) + + retrials = 3 + while hash_original != hash_destiny: + if retrials == 0: + raise Exception('Can not copy {0} to {1}'.format(source, destiny)) + Log.debug('Copying... {0}', datetime.datetime.now()) + shutil.copyfile(source, destiny) + Log.debug('Hashing copy ... {0}', datetime.datetime.now()) + hash_destiny = Utils.get_file_hash(destiny, save=save_hash) + retrials -= 1 + Log.info('Finished {0}', datetime.datetime.now()) + + @staticmethod + def move_file(source, destiny, save_hash=False): + """ + Moves a file from source to destiny, creating dirs if necessary + + :param source: path to source + :type source: str + :param destiny: path to destiny + :type destiny: str + :param save_hash: if True, stores hash value in a file + :type save_hash: bool + """ + Utils.copy_file(source, destiny, save_hash) + os.remove(source) + + @staticmethod + def remove_file(path): + """ + Removes a file, checking before if its exists + + :param path: path to file + :type path: str + """ + if os.path.isfile(path): + os.remove(path) + + @staticmethod + def copy_tree(source, destiny): + if not os.path.exists(destiny): + os.makedirs(destiny) + shutil.copystat(source, destiny) + lst = os.listdir(source) + for item in lst: + item_source = os.path.join(source, item) + item_destiny = os.path.join(destiny, item) + if os.path.isdir(item_source): + Utils.copy_tree(item_source, item_destiny) + else: + shutil.copy2(item_source, item_destiny) + + @staticmethod + def move_tree(source, destiny): + Utils.copy_tree(source, destiny) + shutil.rmtree(source) + + @staticmethod + def get_file_hash(filepath, use_stored=False, save=False): + """ + Returns the xxHash hash for the given filepath + :param filepath: path to the file to compute hash on + :type filepath:str + :param use_stored: if True, try to read the hash value from file + :type use_stored: bool + :param save: if True, stores hash value in a file + :type save: bool + :return: file's xxHash hash + :rtype: str + """ + if use_stored: + hash_file = Utils._get_hash_filename(filepath) + if os.path.isfile(hash_file): + hash_value = open(hash_file, 'r').readline() + return hash_value + + blocksize = 104857600 + hasher = xxhash.xxh64() + with open(filepath, 'rb') as afile: + buf = afile.read(blocksize) + while len(buf) > 0: + hasher.update(buf) + buf = afile.read(blocksize) + hash_value = hasher.hexdigest() + if save: + hash_file = open(Utils._get_hash_filename(filepath), 'w') + hash_file.write(hash_value) + hash_file.close() + + return hash_value + + @staticmethod + def _get_hash_filename(filepath): + folder = os.path.dirname(filepath) + filename = os.path.basename(filepath) + hash_file = os.path.join(folder, '.{0}.xxhash64.hash'.format(filename)) + return hash_file + + @staticmethod + def execute_shell_command(command, log_level=Log.DEBUG): + """ + Executes a sheel commandsi + :param command: command to execute + + Log.info('Detailed time for diagnostic class') + :param log_level: log level to use for command output + :type log_level: int + :return: command output + :rtype: list + """ + # noinspection PyTypeChecker + if isinstance(command, six.string_types): + command = command.split() + process = subprocess.Popen(command, stdout=subprocess.PIPE) + output = list() + comunicate = process.communicate() + for line in comunicate: + if not line: + continue + if log_level != Log.NO_LOG: + Log.log.log(log_level, line) + output.append(line) + if process.returncode != 0: + raise Utils.ExecutionError('Error executing {0}\n Return code: {1}'.format(' '.join(command), + process.returncode)) + return output + + _cpu_count = None + + @staticmethod + def available_cpu_count(): + """ + Number of available virtual or physical CPUs on this systemx + """ + if Utils._cpu_count is None: + try: + m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', + open('/proc/self/status').read()) + if m: + res = bin(int(m.group(1).replace(',', ''), 16)).count('1') + if res > 0: + Utils._cpu_count = res + except IOError: + try: + import multiprocessing + Utils._cpu_count = multiprocessing.cpu_count() + return Utils._cpu_count + except (ImportError, NotImplementedError): + Utils._cpu_count = -1 + return Utils._cpu_count + + @staticmethod + def convert2netcdf4(filetoconvert): + """ + Checks if a file is in netCDF4 format and converts to netCDF4 if not + + :param filetoconvert: file to convert + :type filetoconvert: str + """ + + if Utils._is_compressed_netcdf4(filetoconvert): + return + + Log.debug('Reformatting to netCDF-4') + temp = TempFile.get() + Utils.execute_shell_command(["nccopy", "-4", "-d4", "-s", filetoconvert, temp]) + shutil.move(temp, filetoconvert) + + @classmethod + def _is_compressed_netcdf4(cls, filetoconvert): + is_compressed = True + handler = Utils.openCdf(filetoconvert) + if not handler.file_format == 'NETCDF4': + is_compressed = False + else: + ncdump_result = Utils.execute_shell_command('ncdump -hs {0}'.format(filetoconvert), Log.NO_LOG) + ncdump_result = ncdump_result[0].replace('\t', '').split('\n') + for var in handler.variables: + if not '{0}:_DeflateLevel = 4 ;'.format(var) in ncdump_result: + is_compressed = False + break + if not '{0}:_Shuffle = "true" ;'.format(var) in ncdump_result: + is_compressed = False + break + + handler.close() + return is_compressed + + +# noinspection PyPep8Naming + @staticmethod + def openCdf(filepath, mode='a'): + """ + Opens a netCDF file and returns a handler to it + + :param filepath: path to the file + :type filepath: str + :param mode: mode to open the file. By default, a (append) + :type mode: str + :return: handler to the file + :rtype: netCDF4.Dataset + """ + return netCDF4.Dataset(filepath, mode) + + @staticmethod + def get_datetime_from_netcdf(handler, time_variable='time'): + """ + Gets a datetime array from a netCDF file + + :param handler: file to read + :type handler: netCDF4.Dataset + :param time_variable: variable to read, by default 'time' + :type time_variable: str + :return: Datetime numpy array created from the values stored at the netCDF file + :rtype: np.array + """ + var_time = handler.variables[time_variable] + nctime = var_time[:] # get values + units = var_time.units + + try: + cal_temps = var_time.calendar + except AttributeError: + cal_temps = u"standard" + return netCDF4.num2date(nctime, units=units, calendar=cal_temps) + + @staticmethod + def copy_variable(source, destiny, variable, must_exist=True, add_dimensions=False, new_names=None): + """ + Copies the given variable from source to destiny + + :param add_dimensions: if it's true, dimensions required by the variable will be automatically added to the + file. It will also add the dimension variable + :type add_dimensions: bool + :param source: origin file + :type source: netCDF4.Dataset + :param destiny: destiny file + :type destiny: netCDF4.Dataset + :param variable: variable to copy + :type variable: str + :param must_exist: if false, does not raise an error uf variable does not exist + :type must_exist: bool + :param new_names: dictionary containing variables to rename and new name as key-value pairs + :type new_names: dict + :return: + """ + if not must_exist and variable not in source.variables.keys(): + return + + if not new_names: + new_names = dict() + if variable in new_names: + new_name = new_names[variable] + else: + new_name = variable + + if new_name in destiny.variables.keys(): + return + + translated_dimensions = Utils._translate(source.variables[variable].dimensions, new_names) + if not set(translated_dimensions).issubset(destiny.dimensions): + if not add_dimensions: + raise Exception('Variable {0} can not be added because dimensions does not match: ' + '{1} {2}'.format(variable, translated_dimensions, destiny.dimensions)) + for dimension in source.variables[variable].dimensions: + Utils.copy_dimension(source, destiny, dimension, must_exist, new_names) + if new_name in destiny.variables.keys(): + # Just in case the variable we are copying match a dimension name + return + original_var = source.variables[variable] + new_var = destiny.createVariable(new_name, original_var.datatype, translated_dimensions) + Utils.copy_attributes(new_var, original_var) + new_var[:] = original_var[:] + + @staticmethod + def copy_attributes(new_var, original_var, omitted_attributtes=None): + if omitted_attributtes is None: + omitted_attributtes = [] + new_var.setncatts({k: Utils.convert_to_ASCII_if_possible(original_var.getncattr(k)) + for k in original_var.ncattrs() if k not in omitted_attributtes}) + + @staticmethod + def copy_dimension(source, destiny, dimension, must_exist=True, new_names=None): + """ + Copies the given dimension from source to destiny, including dimension variables if present + + :param new_names: dictionary containing variables to rename and new name as key-value pairs + :type new_names: dict + :param source: origin file + :type source: netCDF4.Dataset + :param destiny: destiny file + :type destiny: netCDF4.Dataset + :param dimension: variable to copy + :type dimension: str + :param must_exist: if false, does not raise an error uf variable does not exist + :type must_exist: bool + + :return: + """ + if not must_exist and dimension not in source.dimensions.keys(): + return + if not new_names: + new_names = dict() + if dimension in new_names: + new_name = new_names[dimension] + else: + new_name = dimension + if new_name in destiny.dimensions.keys(): + return + if not new_name: + new_name = dimension + destiny.createDimension(new_name, source.dimensions[dimension].size) + if dimension in source.variables: + Utils.copy_variable(source, destiny, dimension, new_names=new_names) + + @staticmethod + def concat_variables(source, destiny, remove_source=False): + """ + Add variables from a nc file to another + :param source: path to source file + :type source: str + :param destiny: path to destiny file + :type destiny: str + :param remove_source: if True, removes source file + :type remove_source: bool + """ + if os.path.exists(destiny): + handler_total = Utils.openCdf(destiny) + handler_variable = Utils.openCdf(source) + concatenated = dict() + for var in handler_variable.variables: + if var not in handler_total.variables: + Utils.copy_variable(handler_variable, handler_total, var, add_dimensions=True) + else: + variable = handler_variable.variables[var] + if 'time' not in variable.dimensions: + continue + concatenated[var] = np.concatenate((handler_total.variables[var][:], variable[:]), + axis=variable.dimensions.index('time')) + + for var, array in six.iteritems(concatenated): + handler_total.variables[var][:] = array + handler_total.close() + handler_variable.close() + if remove_source: + os.remove(source) + else: + if remove_source: + Utils.move_file(source, destiny) + else: + shutil.copy(source, destiny) + Utils.convert2netcdf4(destiny) + + class ExecutionError(Exception): + """ + Exception to raise when a command execution fails + """ + pass + + @classmethod + def _translate(cls, dimensions, new_names): + translated = list() + for dim in dimensions: + if dim in new_names: + translated.append(new_names[dim]) + else: + translated.append(dim) + return translated + + @staticmethod + def create_folder_tree(path): + """ + Createas a fodle path will and parent directories if needed. + :param path: folder's path + :type path: str + """ + if not os.path.exists(path): + # noinspection PyBroadException + try: + os.makedirs(path) + except Exception: + # Here we can have a race condition. Let's check again for existence and rethrow if still not exists + if not os.path.isdir(path): + raise + + @staticmethod + def give_group_write_permissions(path): + st = os.stat(path) + if st.st_mode & stat.S_IWGRP: + return + os.chmod(path, st.st_mode | stat.S_IWGRP) + + @staticmethod + def convert_units(var_handler, new_units): + if new_units == var_handler.units: + return + new_unit = Units(new_units) + old_unit = Units(var_handler.units) + var_handler[:] = Units.conform(var_handler[:], old_unit, new_unit, inplace=True) + if 'valid_min' in var_handler.ncattrs(): + var_handler.valid_min = Units.conform(float(var_handler.valid_min), old_unit, new_unit, + inplace=True) + if 'valid_max' in var_handler.ncattrs(): + var_handler.valid_max = Units.conform(float(var_handler.valid_max), old_unit, new_unit, + inplace=True) + var_handler.units = new_units + + @staticmethod + def untar(files, destiny_path): + """ + Untar files to a given destiny + :param files: files to unzip + :type files: list[Any] | Tuple[Any] + :param destiny_path: path to destination folder + :type destiny_path: str + """ + for filepath in files: + Log.debug('Unpacking {0}', filepath) + tar = tarfile.open(filepath) + for file_compressed in tar.getmembers(): + if file_compressed.isdir(): + if os.path.isdir(os.path.join(destiny_path, file_compressed.name)): + continue + else: + if os.path.exists(os.path.join(destiny_path, file_compressed.name)): + os.remove(os.path.join(destiny_path, file_compressed.name)) + tar.extract(file_compressed, destiny_path) + tar.close() + + @staticmethod + def unzip(files, force=False): + """ + Unzip a list of files + :param files: files to unzip + :type files: list | str + :param force: if True, it will overwrite unzipped files + :type force: bool + """ + # noinspection PyTypeChecker + if isinstance(files, six.string_types): + files = [files] + for filepath in files: + Log.debug('Unzipping {0}', filepath) + if force: + option = ' -f' + else: + option = '' + try: + Utils.execute_shell_command('gunzip{1} {0}'.format(filepath, option)) + except Exception as ex: + raise Utils.UnzipException('Can not unzip {0}: {1}'.format(filepath, ex)) + + class UnzipException(Exception): + """ + Excpetion raised when unzip fails + """ + pass + + +class TempFile(object): + """ + Class to manage temporal files + """ + + autoclean = True + """ + If True, new temporary files are added to the list for future cleaning + """ + files = list() + """ + List of files to clean automatically + """ + scratch_folder = '' + """ + Scratch folder to create temporary files on it + """ + prefix = 'temp' + """ + Prefix for temporary filenames + """ + + @staticmethod + def get(filename=None, clean=None, suffix='.nc'): + """ + Gets a new temporal filename, storing it for automated cleaning + + :param suffix: + :param filename: if it is not none, the function will use this filename instead of a random one + :type filename: str + :param clean: if true, stores filename for cleaning + :type clean: bool + :return: path to the temporal file + :rtype: str + """ + if clean is None: + clean = TempFile.autoclean + + if filename: + path = os.path.join(TempFile.scratch_folder, filename) + else: + fd, path = tempfile.mkstemp(dir=TempFile.scratch_folder, prefix=TempFile.prefix, suffix=suffix) + path = str(path) + os.close(fd) + + if clean: + TempFile.files.append(path) + + return path + + @staticmethod + def clean(): + """ + Removes all temporary files created with Tempfile until now + """ + for temp_file in TempFile.files: + if os.path.exists(temp_file): + os.remove(temp_file) + TempFile.files = list() + diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py new file mode 100644 index 0000000000000000000000000000000000000000..9a623469694438ea5bf71afd49c703fd868cb448 --- /dev/null +++ b/earthdiagnostics/variable.py @@ -0,0 +1,439 @@ +# coding=utf-8 +import csv +import glob +import json +import os + +import openpyxl +from bscearth.utils.log import Log + +from earthdiagnostics.constants import Basins +from earthdiagnostics.frequency import Frequency +from earthdiagnostics.modelingrealm import ModelingRealms +from singleton import SingletonType + + +class VariableJsonException(Exception): + pass + + +class VariableManager(object): + def __init__(self): + self._cmor_tables_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cmor_tables') + self._aliases_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'variable_alias') + self._dict_variables = {} + + def get_variable(self, original_name, silent=False): + """ + Returns the cmor variable instance given a variable name + + :param original_name: original variable's name + :type original_name: str + :param silent: if True, omits log warning when variable is not found + :type silent: bool + :return: CMOR variable + :rtype: Variable + """ + try: + return self._dict_aliases[original_name.lower()][1] + except KeyError: + if not silent: + Log.warning('Variable {0} is not defined in the CMOR table. Please add it'.format(original_name)) + return None + + def get_all_variables(self): + """ + Returns all variables + + :return: CMOR variable list + :rtype: set[Variable] + """ + all_vars = set(self._dict_variables.values()) + return sorted(all_vars, key=lambda var: var.short_name) + + def get_variable_and_alias(self, original_name, silent=False): + """ + Returns the cmor variable instance given a variable name + + :param original_name: original variable's name + :type original_name: str + :param silent: if True, omits log warning when variable is not found + :type silent: bool + :return: CMOR variable + :rtype: Variable + """ + try: + return self._dict_aliases[original_name.lower()] + except KeyError: + if not silent: + Log.warning('Variable {0} is not defined in the CMOR table. Please add it'.format(original_name)) + return None, None + + def load_variables(self, table_name): + """ + Loads the CMOR csv and creates the variables dictionary + """ + self._dict_variables = dict() + self._load_variable_list(table_name) + self._load_missing_defaults() + self._load_known_aliases(table_name) + self._construct_aliases_dict() + + def _load_variable_list(self, table_name): + + xlsx_path = self._get_xlsx_path(table_name) + if xlsx_path: + self._load_xlsx(xlsx_path) + return + + json_folder = self._get_json_folder(table_name) + if os.path.isdir(json_folder): + self._load_json(json_folder) + return + + csv_path = self._get_csv_path(table_name) + if os.path.isfile(csv_path): + self._load_file(table_name) + return + + raise Exception('Data convention {0} unknown'.format(table_name)) + + def _get_csv_path(self, table_name): + csv_table_path = os.path.join(self._cmor_tables_folder, '{0}.csv'.format(table_name)) + return csv_table_path + + def _get_json_folder(self, table_name): + json_folder = os.path.join(self._cmor_tables_folder, '{0}/Tables'.format(table_name)) + return json_folder + + def _load_file(self, csv_table_path, default=False): + with open(self._get_csv_path(csv_table_path), 'rb') as csvfile: + reader = csv.reader(csvfile, dialect='excel') + for line in reader: + if line[0] == 'Variable': + continue + + var = Variable() + var.parse_csv(line) + if not var.short_name or var.short_name.lower() in self._dict_variables: + continue + var.default = default + + self._dict_variables[var.short_name.lower()] = var + + def _load_json(self, json_folder): + for file_name in os.listdir(json_folder): + if file_name in ('CMIP6_grids.json', 'CMIP6_formula_terms.json'): + continue + json_data = open(os.path.join(json_folder, file_name)).read() + try: + data = json.loads(json_data) + except ValueError: + continue + if 'variable_entry' in data: + Log.debug('Parsing file {0}'.format(file_name)) + table = CMORTable(data['Header']['table_id'][6:], + Frequency(data['Header']['frequency']), + data['Header']['table_date']) + self._load_json_variables(data['variable_entry'], table) + + def _load_json_variables(self, json_data, table): + for short_name in json_data.keys(): + if short_name == 'ta19': + pass + short_name = str.strip(str(short_name)) + if short_name.lower() in self._dict_variables: + self._dict_variables[short_name.lower()].tables.append(table) + continue + variable = Variable() + try: + variable.parse_json(json_data[short_name], short_name) + variable.add_table(table) + self._dict_variables[variable.short_name.lower()] = variable + except VariableJsonException: + Log.error('Could not read variable {0}'.format(short_name)) + + def _load_known_aliases(self, table_name): + self._load_alias_csv('default') + self._load_alias_csv(table_name) + + def _load_alias_csv(self, filename): + file_path = self._get_aliases_csv_path(filename) + if not os.path.isfile(file_path): + return + + with open(file_path, 'rb') as csvfile: + reader = csv.reader(csvfile, dialect='excel') + for line in reader: + if line[0] == 'Aliases': + continue + + aliases = line[0].split(':') + if line[1] not in aliases: + aliases.append(line[1]) + + cmor_vars = [] + for alias in aliases: + alias = str.strip(alias) + if alias.lower() in self._dict_variables: + cmor_vars.append(self._dict_variables[alias.lower()]) + if len(cmor_vars) == 0: + Log.error('Aliases {0} could not be mapped to any variable'.format(aliases)) + continue + elif len(cmor_vars) > 1: + non_default = [var for var in cmor_vars if not var.default] + if len(non_default) == 1: + for default in [var for var in cmor_vars if var not in non_default]: + del self._dict_variables[default.short_name.lower()] + cmor_vars = non_default + + else: + Log.error('Aliases {0} can be be mapped to multiple variables ' + '[{1}]'.format(aliases, ', '.join(map(str, cmor_vars)))) + continue + cmor_var = cmor_vars[0] + + for alias in aliases: + if alias != cmor_var.short_name and alias in self._dict_variables: + Log.error('Alias {0} for variable {1} is already a different ' + 'variable!'.format(alias, cmor_var.short_name)) + continue + alias_object = VariableAlias(alias) + if line[2]: + alias_object.basin = Basins().parse(line[2]) + if line[3]: + alias_object.grid = line[3] + cmor_var.known_aliases.append(alias_object) + + def _get_aliases_csv_path(self, filename): + csv_table_path = os.path.join(self._aliases_folder, '{0}.csv'.format(filename)) + return csv_table_path + + def _construct_aliases_dict(self): + self._dict_aliases = {} + for cmor_var_name in self._dict_variables: + cmor_var = self._dict_variables[cmor_var_name] + if cmor_var_name not in cmor_var.known_aliases: + cmor_var.known_aliases.append(VariableAlias(cmor_var_name)) + for alias in cmor_var.known_aliases: + self._dict_aliases[alias.alias] = (alias, cmor_var) + + def _get_xlsx_path(self, table_name): + xlsx_table_path = os.path.join(self._cmor_tables_folder, '{0}.xlsx'.format(table_name)) + + if os.path.isfile(xlsx_table_path): + return xlsx_table_path + xlsx_table_path = os.path.join(self._cmor_tables_folder, table_name, 'etc', '*.xlsx') + xlsx_table_path = glob.glob(xlsx_table_path) + if len(xlsx_table_path) == 1: + return xlsx_table_path[0] + return None + + def _load_xlsx(self, xlsx_table_path): + excel = openpyxl.load_workbook(xlsx_table_path, True) + + table_data = {} + data_sheet = excel.worksheets[0] + for row in data_sheet.rows: + if row[1].value in excel.sheetnames: + table_data[row[1].value] = (Frequency(row[2].value), 'Date missing') + + for sheet_name in excel.sheetnames: + try: + sheet = excel.get_sheet_by_name(sheet_name) + if sheet.title == 'Primday': + pass + if sheet['A1'].value != 'Priority': + continue + table_frequency, table_date = table_data[sheet.title] + table = CMORTable(sheet.title, table_frequency, table_date) + for row in sheet.rows: + if row[0].value == 'Priority' or not row[5].value: + continue + + cmor_name = row[11].value + if not cmor_name: + cmor_name = row[5].value + + priority = int(row[0].value) + bsc_commitment = row[30].value + if bsc_commitment is not None and bsc_commitment.strip().lower() == 'false': + priority = priority + 3 + + if cmor_name.lower() in self._dict_variables: + var = self._dict_variables[cmor_name.lower()] + else: + var = Variable() + var.short_name = cmor_name + var.standard_name = row[6].value + var.long_name = row[1].value + + var.domain = self._process_modelling_realm(var, row[12].value) + + var.units = row[2].value + self._dict_variables[var.short_name.lower()] = var + var.add_table(table, priority) + + except Exception as ex: + Log.error('Table {0} can not be loaded: {1}', sheet_name, ex) + import traceback + traceback.print_exc() + + @staticmethod + def _process_modelling_realm(var, value): + if value is None: + value = '' + modelling_realm = value.split(' ') + return var.get_modelling_realm(modelling_realm) + + def _load_missing_defaults(self): + self._load_file('default', True) + + +class Variable(object): + """ + Class to characterize a CMOR variable. It also contains the static method to make the match between thje original + name and the standard name. Requires data _convetion to be available in cmor_tables to work. + """ + + def __str__(self): + return '{0} ({1})'.format(self.standard_name, self.short_name) + + def __repr__(self): + return '{0} ({1})'.format(self.standard_name, self.short_name) + + def __init__(self): + self.short_name = None + self.standard_name = None + self.long_name = None + self.units = None + self.valid_min = None + self.valid_max = None + self.grid = None + self.default = False + self.domain = None + self.known_aliases = [] + self.tables = [] + + def add_table(self, table, priority=None): + self.tables.append((table, priority)) + + def parse_json(self, json_var, key): + + if 'out_name' in json_var: + self.short_name = json_var['out_name'].strip() + else: + raise VariableJsonException('Variable has no out name defined'.format(key)) + self.standard_name = json_var['standard_name'].strip() + self.long_name = json_var['long_name'].strip() + + domain = json_var['modeling_realm'].split(' ') + self.domain = self.get_modelling_realm(domain) + + self.valid_min = json_var['valid_min'].strip() + self.valid_max = json_var['valid_max'].strip() + self.units = json_var['units'].strip() + if 'priority' in json_var: + self.priority = int(json_var['priority'].strip()) + elif 'primavera_priority' in json_var: + self.priority = int(json_var['primavera_priority'].strip()) + else: + self.priority = 1 + + def get_modelling_realm(self, domains): + if len(domains) > 1: + Log.warning('Multiple modeling realms assigned to variable {0}: {1}. ', self, domains) + parsed = [] + for domain in domains: + parsed.append(ModelingRealms.parse(domain)) + + selected = self._select_most_specific(parsed) + if selected: + Log.warning('We will use {0} as it is the most specific', selected) + return selected + + Log.warning('We will use {0} as it is the first on the list and there is no one that is more specific', + parsed[0]) + return parsed[0] + + elif len(domains) == 0: + Log.warning('Variable {0} has no modeling realm defined'.format(self.short_name)) + return None + else: + return ModelingRealms.parse(domains[0]) + + def parse_csv(self, var_line): + self.short_name = var_line[1].strip() + self.standard_name = var_line[2].strip() + self.long_name = var_line[3].strip() + self.domain = ModelingRealms.parse(var_line[4].strip()) + self.basin = Basins().parse(var_line[5]) + self.units = var_line[6].strip() + self.valid_min = var_line[7].strip() + self.valid_max = var_line[8].strip() + self.grid = var_line[9].strip() + + def get_table(self, frequency, data_convention): + for table, priority in self.tables: + if table.frequency == frequency: + return table + if self.domain: + table_name = self.domain.get_table_name(frequency, data_convention) + return CMORTable(table_name, frequency, 'December 2013') + raise ValueError('Can not get table for {0} and frequency {1}'.format(self, frequency)) + + @staticmethod + def _select_most_specific(parsed): + parsed = set(parsed) + if {ModelingRealms.land, ModelingRealms.landIce} == parsed: + return ModelingRealms.landIce + + if {ModelingRealms.seaIce, ModelingRealms.ocean} == parsed: + return ModelingRealms.seaIce + + if {ModelingRealms.atmos, ModelingRealms.atmosChem} == parsed: + return ModelingRealms.atmosChem + + if {ModelingRealms.ocean, ModelingRealms.ocnBgchem} == parsed: + return ModelingRealms.ocnBgchem + + return None + + +class VariableAlias(object): + """ + Class to characterize a CMOR variable. It also contains the static method to make the match between thje original + name and the standard name. Requires data _convetion to be available in cmor_tables to work. + """ + + def __init__(self, alias): + self.alias = alias + self.basin = None + self.grid = None + + def __str__(self): + string = self.alias + if self.basin: + string += ' Basin: {0}'.format(self.basin) + if self.grid: + string += ' Grid: {0}'.format(self.grid) + return string + + +class CMORTable(object): + def __init__(self, name, frequency, date): + self.name = name + self.frequency = Frequency.parse(frequency) + self.date = date + + def __str__(self): + return self.name + + def __repr__(self): + return '{0.name} ({0.frequency}, {0.date}'.format(self) + + def __lt__(self, other): + return self.name < other.name + + + diff --git a/earthdiagnostics/variable_alias/cmip6.csv b/earthdiagnostics/variable_alias/cmip6.csv new file mode 100644 index 0000000000000000000000000000000000000000..e6c8a679f5ea3fc1969546c0f8a6de6d71769da4 --- /dev/null +++ b/earthdiagnostics/variable_alias/cmip6.csv @@ -0,0 +1,81 @@ +Aliases,Shortname,Basin,Grid +iiceconc:soicecov:ileadfra:sic,siconc,, +alk,talk,, +oxygen,o2,, +calcite,calc,, +po4,po4,, +poc,poc,, +silicate,si,, +nanophy,nanophy,, +microzoo,zmicro,, +doc,dissoc,, +diaphy,phydiat,, +mesozoo,zmeso,, +dsi,dsi,, +dissfe,dfe,, +bfe,bfe,, +goc,goc,, +sfe,sfe,, +dfe,dfe,, +micrzoo,zmicro,, +nfe,nfe,, +nchl,nchl,, +dchl,chldiat,, +nitrate,no3,, +ammonium,nh4,, +pno3tot,pno3tot,, +psiltot,psiltot,, +palktot,palktot,, +pfertot,pfertot,, +tcflx,tcflx,, +tcflxcum,tcflxcum,, +c-export,c-export,, +tintpp,tintpp,, +tnfix,tnfix,, +tdenit,tdenit,, +intppnew,intppnew,, +inttppnew,inttppnew,, +inttpbfe,pbfe,, +intdic,intdic,, +o2min,o2min,, +zo2min,zo2min,, +intnfix,intpn2,, +intppphy,intppphy,, +intppphy2,intppdiat,, +ppphy ,ppphy ,, +ppphy2 ,pdi,, +intpp,intpp,, +inttpp,inttpp,, +intpbfe,intpbfe,, +intpbsi,intpbsi,, +intpbcal,intpbcal,, +cflx,cflx,, +remin,remin,, +denit,denit,, +nfix,nfix,, +sdenit,sdenit,, +dpco2,dpco2,, +epc100,epc100,, +expc,expc,, +par,par,, +lnnut,lnnut,, +ldnut,ldnut,, +lnfe,lnfe,, +ldfe,limfediat,, +lnlight,lnlight,, +ldlight,ldlight,, +graz1,graz1,, +graz2,graz2,, +mumax,mumax,, +mun,mun,, +mud,mud,, +ppnewn,ppnewn,, +ppnewd,ppnewd,, +alb_ice,sialb,, +qsr3d,rsdo,, +hflx_rnf_cea,hfrunoffds2d,, +hflx_rain_cea,hfrainds,, +hflx_cal_cea,hfibthermds2d,, +rain,prra,, +calving,ficeberg2d,, + diff --git a/earthdiagnostics/variable_alias/default.csv b/earthdiagnostics/variable_alias/default.csv new file mode 100644 index 0000000000000000000000000000000000000000..1baafce74f5d88a9bcefb2ef5aa17152fa7d28b2 --- /dev/null +++ b/earthdiagnostics/variable_alias/default.csv @@ -0,0 +1,302 @@ +Aliases,Shortname,Basin,Grid +iiceages:siage:iice_otd,ageice,, +al,al,, +bgfrcsal,bgfrcsal,, +bgfrctem,bgfrctem,, +bgfrcvol,bgfrcvol,, +bgheatco,bgheatco,, +bgsaline,bgsaline,, +bgsaltco,bgsaltco,, +bgtemper,bgtemper,, +bgvole3t,bgvole3t,, +bgvolssh,bgvolssh,, +bld,bld,, +iicebome:iocewflx,bmelt,, +sobowlin,bowlin,, +cc,cl,, +hcc,clh,, +lcc,cll,, +mcc,clm,, +ciwc,cli,, +tcc,clt,, +tcw,clwvi,, +iicedive:sidive,divice,, +e,evspsbl,, +fal,fal,, +sowaflep,fatmosocean,, +sowaflcd,fdilution,, +sophtldf,fhbasindif,, +iowaflup,ficeocean,, +sorunoff,friver,, +sowaflup,fupward,, +gwd,gwd,, +ibgheatco,hcicega,, +sbgheatco,hcsnga,, +heatc,heatc,, +sohtatl:hfnortha,hfbasin,Atl, +sohtind,hfbasin,Ind, +sohtipc,hfbasin,IndPac, +sohtpac,hfbasin,Pac, +sophtadv,hfbasinadv,, +sophteiv,hfbasinba,, +qt_oce:sohefldo:qt,hfds,, +slhf,hfls,, +sshf,hfss,, +zqla,hflso,, +zqsb,hfsso,, +zqlw,rlntds,, +sophtove,htovovrt,, +q,hus,, +soicealb,ialb,, +ibgfrcsfx,ibgfrcsfx,, +ibgfrcvol,ibgfrcvol,, +ibghfxbog,ibghfxbog,, +ibghfxbom,ibghfxbom,, +ibghfxdhc,ibghfxdhc,, +ibghfxdif,ibghfxdif,, +ibghfxdyn,ibghfxdyn,, +ibghfxin,ibghfxin,, +ibghfxopw,ibghfxopw,, +ibghfxout,ibghfxout,, +ibghfxres,ibghfxres,, +ibghfxsnw,ibghfxsnw,, +ibghfxspr,ibghfxspr,, +ibghfxsub,ibghfxsub,, +ibghfxsum,ibghfxsum,, +ibghfxthd,ibghfxthd,, +ibgsfxbog,ibgsfxbogga,, +ibgsfxbom,ibgsfxbomga,, +ibgsfxbri,ibgsfxbriga,, +ibgsfxdyn,ibgsfxdynga,, +ibgsfx,ibgsfxga,, +ibgsfxopw,ibgsfxopwga,, +ibgsfxres,ibgsfxresga,, +ibgsfxsni,ibgsfxsniga,, +ibgsfxsum,ibgsfxsumga,, +ibgvfxbog,ibgvfxbogga,, +ibgvfxbom,ibgvfxbomga,, +ibgvfxdyn,ibgvfxdynga,, +ibgvfx,ibgvfxga,, +ibgvfxopw,ibgvfxopwga,, +ibgvfxres,ibgvfxresga,, +ibgvfxsni,ibgvfxsniga,, +ibgvfxsnw,ibgvfxsnwga,, +ibgvfxspr,ibgvfxsprga,, +ibgvfxsub,ibgvfxsubga,, +ibgvfxsum,ibgvfxsumga,, +ibgvolgrm,ibgvolgrm,, +ibrinvol,ibrinvol,, +sibricat,ibrinvolcat,, +iicebopr,iicebopr,, +iicecolf,iicecolf,, +iicedypr,iicedypr,, +iice_etd,iiceetd,, +iicelapr,iicelapr,, +iicenflx,iicenflx,, +iicesflx,iicesflx,, +iiceshea,iiceshea,, +iicesipr,iicesipr,, +iicfsbri,iicfsbri,, +iicfseqv,iicfseqv,, +ioceflxb,ioceflxb,, +iocehebr,iocehebr,, +iocesafl,iocesafl,, +iocesflx,iocesflx,, +iocetflx,iocetflx,, +iocwnsfl,iocwnsfl,, +isstempe,isstempe,, +scmastot,masso,, +mldkz5,mldkz5,, +somxl010:mldr10_1,mlotst,, +swvl1,mrlsl1,, +swvl2,mrlsl2,, +swvl3,mrlsl3,, +swvl4,mrlsl4,, +ro,mrro,, +tp:precip,pr,, +cp,prc,, +lsp,prs,, +isnowpre,prsn,, +sf:snowpre,prsn,, +tcwv,prw,, +msl,psl,, +qns_ice,qnsice,, +qt_ice,qtice,, +strd,rlds,, +strc:str,rls,, +ttr,rlut,, +ttrc,rlutcs,, +ssrd,rsds,, +tsr,rsdt,, +soshfldo,rsntds,, +ssr,rss,, +ssrc,rsscs,, +tsrc,rsut,, +saltc,saltc,, +sosalflx,sfs,, +NArea,siarean,, +SArea,siareas,, +iice_itd:siconc_cat:siconcat,siccat,, +ibgarea,sicga,, +NExnsidc,siextentn,, +SExnsidc,siextents,, +iiceprod,sigr,, +iiceheco,siheco,, +ibgsaltco,sisaltcga,, +iicethic:sithic::sithick,sit,, +iice_hid:sithic_cat:sithicat,sitcat,, +iicetemp,sitemp,, +ibgtemper,sitempga,, +iicevelo:sivelo,sivelo,, +iicevelu:sivelu,sivelu,, +iicevelv:sivelv,sivelv,, +ibgvoltot,sivolga,, +sivoln:NVolume,sivoln,, +sivols:SVolume,sivols,, +sivolu,sivolu,, +sostatl,sltbasin,, +sostind,sltbasin,, +sostipc,sltbasin,, +sostpac,sltbasin,, +sopstadv,sltbasinadv,, +sopsteiv,sltbasinba,, +sopstldf,sltbasindif,, +sltnortha,sltnortha,, +sopstove,sltovovrt,, +zosalatl,sltzmean,Atl, +zosalglo,sltzmean,Glob, +zosalind,sltzmean,Ind, +zosalipc,sltzmean,IndPac, +zosalpac,sltzmean,Pac, +asn,snal,, +iice_hsd:snthicat,sndcat,, +isnoheco,snheco,, +sd,snld,, +smlt,snm,, +isnowthi,snld,, +sbgvoltot,snvolga,, +snvolu,snvolu,, +vosaline:mean_3Dsosaline,so,, +scsaltot,soga,, +soleaeiw,soleaeiw,, +soleahtw,soleahtw,, +somixhgt,somixhgt,, +sosaline:isssalin:mean_sosaline,sos,, +sothedep,sothedep,, +src,src,, +zosrfatl,srfzmean,Atl, +zosrfglo,srfzmean,Glob, +zosrfind,srfzmean,Ind, +zosrfipc,srfzmean,IndPac, +zosrfpac,srfzmean,Pac, +rsn,srho,, +iicesali:iice_std,ssi,, +salincat,ssicat,, +ibgsaline,ssiga,, +iicestre,streng,, +so20chgt,t20d,, +t,ta,, +t2m,tas,, +mx2t,tasmax,, +mn2t,tasmin,, +ewss,tauu,, +utau_ice:iocestru:iicestru,strairx,, +sozotaux,tauuo,, +nsss,tauv,, +vtau_ice:iocestrv:iicestrv,strairy,, +sozotauy:sometauy,tauvo,, +d2m,tdps,, +votemper:mean_3Dsosstsst,thetao,, +sctemtot,thetaoga,, +iicesume,tmelt,, +sosstsst:mean_sosstsst,tos,, +sstk,tos,,ifs +tossq,tossq,, +zotematl,toszmean,Atl, +zotemglo,toszmean,Glob, +zotemind,toszmean,Ind, +zotemipc,toszmean,IndPac, +zotempac,toszmean,Pac, +skt,ts,, +iicesurt:soicetem:sistem,tsice,, +istl1,tsice,, +stl1,tsl1,, +stl2,tsl2,, +stl3,tsl3,, +stl4,tsl4,, +tsn,tsn,, +u,ua,, +u10m,uas,, +vozocrtx,uo,, +uos,uos,, +v,va,, +v10m,vas,, +vomecrty,vo,, +vos,vos,, +voddmavs,voddmavs,, +vozoeivu,voeivu,, +vomeeivv,voeivv,, +voveeivw,voeivz,, +scvoltot,volo,, +votkeavm,votkeavm,, +votkeavt,votkeavt,, +votkeevd,votkeevd,, +votkeevm,votkeevm,, +sobarstf,vsftbarot,, +zomsfatl,vsftmyz,Atl, +zomsfglo,vsftmyz,Glob, +zomsfind,vsftmyz,Ind, +zomsfipc:zomsfinp,vsftmyz,IndPac, +zomsfpac,vsftmyz,Pac, +zomsfeiv,vsftmyzba,, +w,wa,, +z,zg,, +vovecrtz,zo,, +sossheigh:sossheig:mean_sossheig,zos,, +scsshtot,zosga,, +scsshste,zossga,, +zossq,zossq,, +scsshtst,zostoga,, +ohc,heatc,, +ohcsum,heatcsum,, +ohcvmean,heatcvmean,, +transix,transix,, +transiy,transiy,, +windsp,sfcWind,, +vsfsit,vsfsit,, +sfdsi,sfdsi,, +hfsithermds,hfsithermds,, +u2o,uosq,, +v2o,vosq,, +vozomatr,umo,, +vomematr,vmo,, +sozohetr,hfx,, +somehetr,hfy,, +uto,uothetao,, +vto,vothetao,, +uso,uoso,, +vso,voso,, +wfo,wfo,, +emp_oce,evsmpr,, +emp_ice,evsmpr,, +qsr_oce,rsntds,, +qns_oce,rlds,, +qsr_ice,rsdssi,, +qns_ice,rldssi,, +sfx,sfx,, +taum,taum,, +zfull,zfull,, +zhalf,zhalf,, +pbo,pbo,, +thkcello,thkcello,, +ficeberg,ficeberg,, +wo,wo,, +w2o,wosq,, +difvho,difvho,, +vovematr,wmo,, +qtr_ice,qtr,, +var78,tclw,, +var79,tciw,, +rho,rhopoto,, +qsr,rsntds,, +runoffs,friver,, \ No newline at end of file diff --git a/earthdiagnostics/variable_alias/preface.csv b/earthdiagnostics/variable_alias/preface.csv new file mode 100644 index 0000000000000000000000000000000000000000..07aef840283521d855e3b6e9c4fad60f400ed56d --- /dev/null +++ b/earthdiagnostics/variable_alias/preface.csv @@ -0,0 +1,4 @@ +Aliases,Shortname,Basin,Grid +iiceconc:siconc:soicecov:ileadfra,sic,, +ci,sic,,ifs +es,sbl,, \ No newline at end of file diff --git a/earthdiagnostics/variable_alias/primavera.csv b/earthdiagnostics/variable_alias/primavera.csv new file mode 100644 index 0000000000000000000000000000000000000000..db38cd5c1c7448696e8583c0d94739cce1790e54 --- /dev/null +++ b/earthdiagnostics/variable_alias/primavera.csv @@ -0,0 +1,9 @@ +Aliases,Shortname,Basin,Grid +iiceconc:siconc:soicecov:ileadfra:sic,siconc,, +alb_ice,sialb,, +qsr3d,rsdo,, +hflx_rnf_cea,hfrunoffds2d,, +hflx_rain_cea,hfrainds,, +hflx_cal_cea,hfibthermds2d,, +rain,prra,, +calving,ficeberg2d,, diff --git a/earthdiagnostics/variable_alias/specs.csv b/earthdiagnostics/variable_alias/specs.csv new file mode 100644 index 0000000000000000000000000000000000000000..07aef840283521d855e3b6e9c4fad60f400ed56d --- /dev/null +++ b/earthdiagnostics/variable_alias/specs.csv @@ -0,0 +1,4 @@ +Aliases,Shortname,Basin,Grid +iiceconc:siconc:soicecov:ileadfra,sic,, +ci,sic,,ifs +es,sbl,, \ No newline at end of file diff --git a/earthdiagnostics/variable_type.py b/earthdiagnostics/variable_type.py new file mode 100644 index 0000000000000000000000000000000000000000..4b3f17daa0b515582f1870cbbc7f05dacfa3379a --- /dev/null +++ b/earthdiagnostics/variable_type.py @@ -0,0 +1,13 @@ +# coding=utf-8 +class VariableType(object): + MEAN = 1 + STATISTIC = 2 + + @staticmethod + def to_str(vartype): + if vartype == VariableType.MEAN: + return 'mean' + elif vartype == VariableType.STATISTIC: + return 'statistics' + else: + raise ValueError('Variable type {0} not supported'.format(vartype)) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..ef3a62ea119458e629c15ecc752d412f837be9f0 --- /dev/null +++ b/earthdiagnostics/work_manager.py @@ -0,0 +1,285 @@ +# coding=utf-8 +import datetime +import operator +import threading +import time + +from bscearth.utils.log import Log +# noinspection PyCompatibility +from concurrent.futures import ThreadPoolExecutor + +from earthdiagnostics.datafile import StorageStatus, LocalStatus +from earthdiagnostics.diagnostic import DiagnosticStatus, Diagnostic, DiagnosticOptionError +from earthdiagnostics.general import * +from earthdiagnostics.ocean import * +from earthdiagnostics.statistics import * +from earthdiagnostics.utils import Utils, TempFile + + +class WorkManager(object): + + def __init__(self, config, data_manager): + self.jobs = None + self.config = config + self.time = {} + self.had_errors = False + self.data_manager = data_manager + + def prepare_job_list(self): + self._register_diagnostics() + list_jobs = list() + for fulldiag in self.config.get_commands(): + Log.info("Adding {0} to diagnostic list", fulldiag) + diag_options = fulldiag.split(',') + + diag_class = Diagnostic.get_diagnostic(diag_options[0]) + if diag_class: + try: + for job in diag_class.generate_jobs(self, diag_options): + list_jobs.append(job) + continue + except DiagnosticOptionError as ex: + Log.error('Can not configure diagnostic {0}: {1}', diag_options[0], ex) + self.had_errors = True + else: + Log.error('{0} is not an available diagnostic', diag_options[0]) + self.had_errors = True + self.jobs = list_jobs + + def run(self): + time = datetime.datetime.now() + Log.info("Starting to compute at {0}", time) + self.threads = Utils.available_cpu_count() + if 0 < self.config.max_cores < self.threads: + self.threads = self.config.max_cores + Log.info('Using {0} threads', self.threads) + + self.downloader = Downloader() + self.uploader = ThreadPoolExecutor(self.config.parallel_uploads) + self.executor = ThreadPoolExecutor(self.threads) + + for job in self.jobs: + job.request_data() + job.declare_data_generated() + job.subscribe(self, self._job_status_changed) + for subjob in job.subjobs: + subjob.subscribe(self, self._job_status_changed) + job.check_is_ready() + + for file_object in self.data_manager.requested_files.values(): + file_object.subscribe(self, self._file_object_status_changed) + if file_object.download_required(): + self.downloader.submit(file_object) + + self.downloader.start() + self.lock = threading.Lock() + self.lock.acquire() + + self.check_completion() + self.lock.acquire() + + self.downloader.shutdown() + self.executor.shutdown() + self.uploader.shutdown(True) + + TempFile.clean() + finish_time = datetime.datetime.now() + Log.result("Diagnostics finished at {0}", finish_time) + Log.result("Elapsed time: {0}\n", finish_time - time) + self.print_errors() + self.print_stats() + return not self.had_errors + + def _job_status_changed(self, job): + if job.status == DiagnosticStatus.READY: + self.executor.submit(self._run_job, job) + self.check_completion() + + def _file_object_status_changed(self, file_object): + if file_object.download_required(): + self.downloader.submit(file_object) + return + if file_object.upload_required(): + self.uploader.submit(file_object.upload) + return + self.check_completion() + + def check_completion(self): + for job in self.jobs: + if job.status in (DiagnosticStatus.READY, DiagnosticStatus.RUNNING): + return False + + if job.status == DiagnosticStatus.WAITING: + if job.all_requests_in_storage(): + return False + + for request in self.data_manager.requested_files.values(): + if request.storage_status == StorageStatus.UPLOADING: + return + if request.local_status == LocalStatus.DOWNLOADING: + return + if request.upload_required(): + return + if request.download_required(): + return + self.lock.release() + return True + + def print_stats(self): + Log.info('Time consumed by each diagnostic class') + Log.info('--------------------------------------') + + times = {} + for job in self.jobs: + job_type = job.alias + if job_type in times.keys(): + times[job_type] += job.consumed_time + else: + times[job_type] = job.consumed_time + + for diag in sorted(times, key=operator.itemgetter(1)): + Log.info('{0:23} {1:}', diag, times[diag]) + + def print_errors(self): + failed = [job for job in self.jobs if job.status == DiagnosticStatus.FAILED] + if len(failed) == 0: + return + self.had_errors = True + Log.error('Failed jobs') + Log.error('-----------') + for job in failed: + Log.error('{0}: {0.message}', job) + Log.error('Total wasted time: {0}', sum([job.consumed_time for job in failed], datetime.timedelta())) + Log.info('') + + @staticmethod + def _run_job(job): + time = datetime.datetime.now() + try: + Log.info('Starting {0}', job) + job.status = DiagnosticStatus.RUNNING + job.compute() + except Exception as ex: + job.consumed_time = datetime.datetime.now() - time + job.message = str(ex) + Log.error('Job {0} failed: {1}', job, ex) + job.status = DiagnosticStatus.FAILED + return False + + job.consumed_time = datetime.datetime.now() - time + Log.result('Finished {0}', job) + job.status = DiagnosticStatus.COMPLETED + return True + count = 0 + failed_jobs = list() + + @staticmethod + def _register_diagnostics(): + WorkManager._register_ocean_diagnostics() + WorkManager._register_general_diagnostics() + WorkManager._register_stats_diagnostics() + + @staticmethod + def _register_stats_diagnostics(): + Diagnostic.register(MonthlyPercentile) + Diagnostic.register(ClimatologicalPercentile) + Diagnostic.register(DaysOverPercentile) + Diagnostic.register(Discretize) + + @staticmethod + def _register_general_diagnostics(): + Diagnostic.register(DailyMean) + Diagnostic.register(MonthlyMean) + Diagnostic.register(YearlyMean) + Diagnostic.register(Rewrite) + Diagnostic.register(Relink) + Diagnostic.register(RelinkAll) + Diagnostic.register(Scale) + Diagnostic.register(Attribute) + Diagnostic.register(Module) + Diagnostic.register(VerticalMeanMetersIris) + Diagnostic.register(SimplifyDimensions) + + @staticmethod + def _register_ocean_diagnostics(): + Diagnostic.register(MixedLayerSaltContent) + Diagnostic.register(Siasiesiv) + Diagnostic.register(VerticalMean) + Diagnostic.register(VerticalMeanMeters) + Diagnostic.register(Interpolate) + Diagnostic.register(InterpolateCDO) + Diagnostic.register(Moc) + Diagnostic.register(AreaMoc) + Diagnostic.register(MaxMoc) + Diagnostic.register(Psi) + Diagnostic.register(Gyres) + Diagnostic.register(ConvectionSites) + Diagnostic.register(CutSection) + Diagnostic.register(AverageSection) + Diagnostic.register(MixedLayerHeatContent) + Diagnostic.register(HeatContentLayer) + Diagnostic.register(HeatContent) + Diagnostic.register(RegionMean) + Diagnostic.register(Rotation) + Diagnostic.register(VerticalGradient) + + +class Downloader(object): + def __init__(self): + self._downloads = [] + self._lock = threading.Lock() + self._wait = threading.Semaphore() + self.stop = False + + def start(self): + self._thread = threading.Thread(target=self.downloader) + self._thread.start() + + def submit(self, datafile): + self._lock.acquire() + self._downloads.append(datafile) + self._lock.release() + + def downloader(self): + try: + def suscribers_waiting(datafile): + waiting = 0 + for diag in datafile.suscribers: + if not isinstance(diag, Diagnostic): + continue + if diag.pending_requests() == 1: + waiting += 1 + return waiting + + def prioritize(datafile1, datafile2): + waiting = suscribers_waiting(datafile1) - suscribers_waiting(datafile2) + if waiting: + return -waiting + + suscribers = len(datafile1.suscribers) - len(datafile2.suscribers) + if suscribers: + return -suscribers + + size = datafile1.size - datafile2.size + if size: + return -size + return 0 + + while True: + with self._lock: + if len(self._downloads) == 0: + if self.stop: + return + time.sleep(0.01) + break + self._downloads.sort(prioritize) + datafile = self._downloads[0] + self._downloads.remove(datafile) + datafile.download() + except Exception as ex: + Log.critical('Unhandled error at downloader: {0}', ex) + + def shutdown(self): + self.stop = True + self._thread.join() + diff --git a/fix.py b/fix.py new file mode 100644 index 0000000000000000000000000000000000000000..b5444a530a0369baa12eb15ea9c03409abd817f9 --- /dev/null +++ b/fix.py @@ -0,0 +1,32 @@ +import os +import shutil + +base_path = '/esarchive/exp/PREFACE/ctrl/cmorfiles/Cerfacs' +for experiment in os.listdir(base_path): + exp_path = os.path.join(base_path, experiment) + if not os.path.isdir(exp_path): + continue + for startdate in os.listdir(exp_path): + if not os.path.isdir(os.path.join(exp_path, startdate)): + continue + for freq in os.listdir(os.path.join(exp_path, startdate)): + for domain in os.listdir(os.path.join(exp_path, startdate, freq)): + for var in os.listdir(os.path.join(exp_path, startdate, freq, domain)): + for member in os.listdir(os.path.join(exp_path, startdate, freq, domain, var)): + for version in os.listdir(os.path.join(exp_path, startdate, freq, domain, var, member)): + for filename in os.listdir(os.path.join(exp_path, startdate, freq, domain, var, member, version)): + print(os.path.join(exp_path, startdate, freq, domain, var, member, version, filename)) + print(os.path.join(exp_path, startdate, freq, domain, var, member, version, + filename.replace('_CNRM-CM-HR_', '_CNRM-CM-HR_{0}_'.format(experiment)))) + print('') + shutil.move(os.path.join(exp_path, startdate, freq, domain, var, member, version, filename), + os.path.join(exp_path, startdate, freq, domain, var, member, version, + filename.replace('_CNRM-CM-HR_', '_CNRM-CM-HR_{0}_'.format(experiment)))) + + + # original_tos_path = os.path.join(exp_path, startdate, 'mon/atmos/tos') + # if os.path.isdir(original_tos_path): + # new_tos_path = os.path.join(exp_path, startdate, 'mon/ocean') + # if not os.path.isdir(new_tos_path): + # os.makedirs(new_tos_path) + # shutil.move(original_tos_path, new_tos_path) \ No newline at end of file diff --git a/launch_diags.sh b/launch_diags.sh new file mode 100755 index 0000000000000000000000000000000000000000..477a1ca4069f72efad559e9052a14227487b5318 --- /dev/null +++ b/launch_diags.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +#SBATCH -n 4 +#SBATCH --time 7-00:00:00 +#SBATCH --error=job.%J.err +#SBATCH --output=job.%J.out + + + +PATH_TO_CONF_FILE=~jvegas/earthdiagnostics/diags.conf +PATH_TO_DIAGNOSTICS=~jvegas/earthdiagnostics +PATH_TO_CONDAENV=diags + +module purge +module load NCO/4.5.4-foss-2015a +module load CDO/1.7.2-foss-2015a +module load CDFTOOLS/3.0a8-foss-2015a +module load Miniconda2 + +set -xv + +source activate diags + +export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} +cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ +./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} diff --git a/model_diags.conf b/model_diags.conf new file mode 100644 index 0000000000000000000000000000000000000000..e97f27731821d14e8013b25c3782f0de06a793fc --- /dev/null +++ b/model_diags.conf @@ -0,0 +1,182 @@ +[DIAGNOSTICS] + +# Temporary folder for the calculations. Final results will never be stored here. +SCRATCH_DIR = /scratch/Earth/$USER + +# Common scratch folder for the ocean masks. This is useful to avoid replicating them for each run at the fat nodes. +# By default is '/scratch/Earth/ocean_masks' +# SCRATCH_MASKS = + +# By default, Earth Diagnostics only copies the mask files if they are not present in the scratch folder. If this +# option is set to true, Earth Diagnostics will copy them regardless of existence. Default is False. +# RESTORE_MESHES = + +# ':' separated list of folders to look for data in. It will look for file in the path $DATA_FOLDER/$EXPID and +# $DATA_FOLDER/$DATA_TYPE/$MODEL/$EXPID +DATA_DIR = /esnas:/esarchive + +# Folder containing mask and mesh files for the dataset. +CON_FILES = /esnas/autosubmit/con_files/ + +# Default data frequency to be used by the diagnostics. Some diagnostics can override this configuration or even +# ignore it completely. +FREQUENCY = mon + +# Type of the dataset to use. It can be exp, obs or recon. Default is exp. +# DATA_TYPE = exp + +# This is used to choose the mechanism for storing and retrieving data. Options are CMOR (for our own experiments) or +# THREDDS (for anything else). Default value is CMOR +# DATA_ADAPTOR = CMOR + +# Convention to use for file paths and names and variable naming among other things. Can be SPECS, PRIMAVERA or CMIP6. +# Default is SPECS. +DATA_CONVENTION = CMIP6 + +# Path to the folder containing CDFTOOLS executables. By default is empty, so CDFTOOLS binaries must be added to the +# system path. +# CDFTOOLS_PATH = + +# Maximum number of cores to use. By default the diagnostics will use all cores available to them. It is not +# necessary when launching through a scheduler, as Earthdiagnostics can detect how many cores the scheduler has +# allocated to it. +# MAX_CORES = 1 + +# Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or +# an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it +# empty +DIAGS = siasie + + +[EXPERIMENT] + +# Institute that made the experiment, observation or reconstruction +INSTITUTE = BSC + +# Name of the model used for the experiment. +MODEL = EC-EARTH + +# Model version. Used to get the correct mask and mesh files. +# Available versions: +# Ec2.3_O1L42 +# Ec3.0_O1L46 Ec3.0_O25L46 Ec3.0_O25L75 +# Ec3.1_O1L46 Ec3.1_O25L75 +# Ec3.2beta_O1L75 Ec3.2_O1L75 Ec3.2_O25L75 +# N3.2_O1L42 N3.3_O1L46 N3.6_O025L75 N3.6_O1L75 N3.6_O25L75 +# nemovar_O1L42 CNRM_O1L42.nc glorys2v1_O25L75 ucl_O2L31 +# ORCA12L75 +MODEL_VERSION =Ec3.2_O1L75 + +# Time between outputs from the atmosphere. This is not the model simulation timestep! Default is 6 +ATMOS_TIMESTEP = 6 + +# Time between outputs from the ocean. This is not the model simulation timestep! Default is 6 +OCEAN_TIMESTEP = 6 + +# Unique identifier for the experiment +EXPID = + +# Startdates to run as a space separated list +STARTDATES = + +# Members to run as a space separated list. You can just provide the number or also add the prefix +MEMBERS = + +# Number of minimum digits to compose the member name. By default it is 1. For example, for member 1 member name +# will be fc1 if MEMBER_DIGITS is 1 or fc01 if MEMBER_DIGITS is 2 +MEMBER_DIGITS = + +# Prefix to use for the member names. By default is 'fc' +MEMBER_PREFIX = + +# Number corresponding to the first member. For example, if your first member is 'fc1', it should be 1. +# If it is 'fc0', it should be 0. By default is 0 +MEMBER_COUNT_START = + +# Length of the chunks in months +CHUNK_SIZE = + +# Number of chunks to run +CHUNKS = + +# Atmospheric grid definition. Will be used as a default target for interpolation diagnostics. +# ATMOS_GRID = + +# Experiment's name. By default it is the EXPID. +# NAME = + +# Calendar to use for date calculation. All calendars supported by Autosubmit are available. Default is 'standard' +# CALENDAR = + + +[CMOR] +# If true, recreates CMOR files regardless of presence. Default = False +# FORCE = False + +# If true, CMORizes ocean files. Default = True +# OCEAN_FILES = True + +# FILTER_FILES = + +# If true, CMORizes atmosphere files. Default = True +# ATMOSPHERE_FILES = True + +# You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var, i.e ocean:thetao atmos:tas +# VARIABLE_LIST = + +# Variables to be CMORized from the grib atmospheric files, separated by comma. +# You can also specify the levels to extract using the following syntax +# VARIABLE_CODE, VARIABLE_CODE:LEVEL, VARIABLE_CODE:LEVEL1-LEVEL2, VARIABLE_CODE:MIN_LEVEL:MAX_LEVEL:STEP +# Examples: +# Variable with code 129 at level 30000: 129:30000 +# Variable with code 129 at levels 30000, 40000 and 60000: 129:30000-40000-60000 +# Variable with code 129 at levels between 30000 and 600000 with 10000 intervals: +# 129:30000:60000:10000 equivalent to 129:30000-40000-50000-60000 + +# Hourly vars +ATMOS_HOURLY_VARS = 129:30000:90000:5000, 130, 131:30000:90000:5000, 132:30000:90000:5000, 151, 167, 168, 164, 165, 166 +# Daily vars +ATMOS_DAILY_VARS = 167, 165, 166, 151, 164, 168, 169, 177, 179, 228, 201, 202, 130:85000 +# Monthly vars +ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, 147, 176, 169, 177, 175, 212, 141, 180, 181, 179, 168, 243, 129:5000-20000-50000-85000, 130:5000-20000-50000-85000, 131:5000-20000-50000-85000, 132:5000-20000-50000-85000, 133:5000-20000-50000-85000 + +# The next bunch of parameters are used to provide metadata for the CMOR files +# ASSOCIATED_EXPERIMENT = +# INITIALIZATION_METHOD = 1 +# INITIALIZATION_DESCRIPTION = ocean: ECMWF system4, ice: DFS4.3 , atmosphere: +# PHYSICS_VERSION = 1 +# PHYSICS_DESCRIPTION = +# ASSOCIATED_MODEL = +# SOURCE = 'EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 + +[THREDDS] +SERVER_URL = https://earth.bsc.es/thredds + +# This ALIAS section is a bit different +# Inside this, you can provide alias for frequent diagnostics calls. +# By default, there are some of the diagnostics available at the previous version. +# You can define an alias for one or moraa90a1ee diagnostic calls + +[ALIAS] +MAX_MOC = mocmax,38,50,500,2000 mocmax,40,40,0,10000 +AREA_MOC = mocarea,40,55,1000,2000,atl mocarea,30,40,1000,2000,atl +STC = mocarea,0,25,0,200,Pac mocarea,-25,0,0,200,Pac mocarea,0,25,0,200,Atl mocarea,-25,0,0,200,Atl +HEAT_SAL_MXL = mlotstsc mlotsthc +LMSALC = vertmeanmeters,so,300,5400 +USALC = vertmeanmeters,so,0,300 +OHC = ohc,glob,0,0,2000 +XOHC = ohc,glob,1,0,0 +LOHC = ohc,glob,0,700,2000 +MOHC = ohc,glob,0,300,700 +UOHC = ohc,glob,0,0,300 +OHC_SPECIFIED_LAYER = ohclayer,0,300 ohclayer,300,800 +3DTEMP = interp,thetao +3DSAL = interp,so +TSEC_AVE190-220E =avgsection,thetao,190,220,-90,90 +SSEC_AVE190-220E =avgsection,so,190,220,-90,90 +VERT_SSECTIONS = cutsection,so,Z,0 cutsection,so,Z,45 cutsection,so,Z,-45 cutsection,so,M,-30 cutsection,so,M,180 cutsection,so,M,80 +VERT_TSECTIONS = cutsection,thetao,Z,0 cutsection,thetao,Z,45 cutsection,thetao,Z,-45 cutsection,thetao,M,-30 cutsection,thetao,M,180 cutsection,thetao,M,80 +SIASIESIV = siasiesiv,glob + + + diff --git a/model_launch_diags.sh b/model_launch_diags.sh new file mode 100755 index 0000000000000000000000000000000000000000..7a226961832a92029c8617ede91486743f48ef07 --- /dev/null +++ b/model_launch_diags.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +#SBATCH -n 1 +#SBATCH --time 7-00:00:00 +#SBATCH --error=earthdiags.%J.err +#SBATCH --output=earthdiags.%J.out + +PATH_TO_CONF_FILE=~jvegas/earthdiagnostics/diags.conf + +module purge +module load earthdiagnostics + +set -xv + +earthdiags -lc DEBUG -f ${PATH_TO_CONF_FILE} diff --git a/ocean_pp.bash b/ocean_pp.bash deleted file mode 100755 index ddabc08c7125d2c700bfc8c49706b3939c9f15a3..0000000000000000000000000000000000000000 --- a/ocean_pp.bash +++ /dev/null @@ -1,646 +0,0 @@ -#!/bin/bash -set -evx - -#module load CDFTOOLS/2.1-foss-2015a CDO NCO - -function delete { -at now +7 days << EOF -rm -rf $WORKDIR -EOF -} - - -msg='Your experiment crashed! Your workdir \($WORKDIR\) will be kept for one week from now and then deleted' -trap "echo $msg ; delete ; exit" SIGINT SIGHUP SIGTERM SIGSEGV SIGKILL EXIT - -################################# -#### User Defined Funtions #### -################################# - - - - -# check if args are ok and read options in config_file - - if [ $# -ne 1 ] ; then - echo - echo "USAGE: config_file " - echo "For example: ./ocean_pp.new.bash /home/Earth/$user/es_git/ocean_diagnostics/config_file " - echo - exit 1 - fi - -config_file=$1 -. ${config_file} - -list_files='grid_T' -if [[ ${listpost[@]} =~ "psi" ]] ; then - echo "The list of diags require grid_U" - list_files=$(echo ${list_files} grid_U) -fi - -if [[ ${listpost[@]} =~ "moc" ]] ; then - echo "The list of diags require grid_V" - list_files=$(echo ${list_files} grid_V) -fi -if [[ ${listpost[@]} =~ "ice" ]] || [[ ${listpost[@]} =~ "siasiesiv" ]]; then - echo "The list of diags contains ice" - list_files=$(echo ${list_files} icemod) -fi - -############################################################################### -# -# moc needs to be computed before max_moc and area_moc -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -if [[ ${listpost[@]##*moc*} != ${listpost[@]} ]] || [[ ${listpost[@]##*stc*} != ${listpost[@]} ]] ; then - if [[ ${listpost[@]#moc} != ${listpost[@]:1} ]] ; then - listpost=( 'moc' "${listpost[@]#moc}" ) - fi -fi -# -# psi needs to be computed before gyres -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -if [[ ${listpost[@]##*gyres*} != ${listpost[@]} ]] ; then - if [[ ${listpost[@]#psi} != ${listpost[@]:1} ]] ; then - listpost=( 'psi' "${listpost[@]#psi}" ) - fi -fi -# -# oce raw outputs need to be extracted before convection option -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -if [[ ${listpost[@]##*convection*} != ${listpost[@]} ]] ; then - if [[ ${listpost[@]#ext_raw_oce} != ${listpost[@]:1} ]] ; then - listpost=( 'ext_raw_oce' "${listpost[@]#ext_raw_oce}" ) - fi -fi -# -# 3d interpolation required before average T sections over longitudes -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -if [[ ${listpost[@]##TSec_ave*} != ${listpost[@]} ]] ; then - if [[ ${listpost[@]#3dtemp} != ${listpost[@]:1} ]] ; then - listpost=( '3dtemp' "${listpost[@]#3dtemp}" ) - warning_T=.true. - fi -fi -if [[ ${listpost[@]##SSec_ave*} != ${listpost[@]} ]] ; then - if [[ ${listpost[@]#3dsal} != ${listpost[@]:1} ]] ; then - listpost=( '3dsal' "${listpost[@]#3dsal}" ) - warning_S=.true. - fi -fi -# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -# You have created a function ? If your new diagnostic relies on an already -# existing diagnotics, you might need similar lignes to the above ones -# Any doubt ---> vguemas@ic3.cat -# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -# -# Preparing WORKDIR and set of available functions -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -export WORKDIR=/scratch/Earth/${USER}/tmp/post_ocean/$$ -mkdir -p $WORKDIR -cd $WORKDIR -source $PATHCOMMONOCEANDIAG/common_ocean_post.txt -# -# Interval of lead months be post-processed -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -case $expid in - 'nemovar_s4'|'nemovar_combine') moni=09 ; syeari=1957 ; syearf=1957 ; insdate=1 ; typeoutput='MMO' ; NEMOVERSION='nemovar_O1L42' ;; - 'glorys2v1') moni=01 ; syeari=1993 ; syearf=1993 ; insdate=1 ; typeoutput='MMO' ;; -esac -case $expid in - 'nemovar_s4') rootout='/esnas/exp/ECMWF/NEMOVAR_S4/monthly_mean' ;; - 'nemovar_combine') rootout='/esnas/exp/ECMWF/NEMOVAR_COMBINE/monthly_mean' ;; - 'glorys2v1') rootout='/esnas/exp/MERCATOR/GLORYS2V1/monthly_mean';; -esac -if [[ ${listpost[@]##max_moc} != ${listpost[@]} ]] || [[ -z "$ltimef" ]] || [[ -z "$ltime0" ]] ; then - if [[ ! -z "$year0" ]] && [[ ! -z "$yearf" ]] ; then - ltime0=$(((${year0}-${syeari})*12+1)) - ltimef=$(((${yearf}-${syeari}+1-(10#$moni+10)/12)*12)) - fi -fi -mon0=$(( (10#$moni+$ltime0-2)%12+1 )) -monf=$(( (10#$moni+$ltimef-2)%12+1 )) -# -# Check on the consistency between the chunk length and the leadtimes -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -if [[ $((($ltimef-$ltime0+1)%$chunklen)) != 0 || $((($ltime0-1)%$chunklen)) != 0 ]] ; then - echo "This a safety stop because we think you might have made a mistake in your configuration file" - echo "Unless you have run your experiment with a variable chunk length, you should have" - echo "a number of leadtimes to post-process that is a multiple of the chunken and the first" - echo "leadtime should be right after the end of a chunk" - echo "If you have run your experiment with a variable chunk length, please remove l.85-93 of ocean_pp.bash" - exit 1 -fi -# -# Loop on start dates -# ~~~~~~~~~~~~~~~~~~~~~ -if [[ $intsdate -eq 0 ]] ; then intsdate=1 ; fi # if only one start date, user might set -# intsdates to 0 which leads to an infinite loop below -for ((yeari=$syeari;yeari<=$syearf;yeari=$(($yeari+intsdate)))) ; do - # - # Interval of years to be post-processed - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - year0=$(($yeari+(10#$moni+$ltime0-2)/12)) - yearf=$(($yeari+(10#$moni+$ltimef-2)/12)) - - for memb in ${listmemb[@]} ; do - # - # Fetching the files on esnas - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - case $expid in - 'nemovar_s4'|'nemovar_combine') get_nemovar ${expid} ${memb} ${year0} ${yearf} ${mon0} ${monf} "${list_files}" - ;; - 'glorys2v1') get_glorys ${year0} ${yearf} ${mon0} ${monf} ;; - *) freqout=${rootout:${#rootout}-12} ; freqout=${freqout/_mean} ; freqout=${freqout/*\/} - get_diagsMMO ${yeari}${moni}01 ${expid} ${memb} $ltime0 $ltimef $chunklen $mod $typeoutput $freqout "${list_files}" - esac - # - # Ready for the post-processing - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - for post in ${listpost[@]} ; do - - case $post in -# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -# You have created a function ? Enter its call right here under the flag chosen -# Remember to consider both 'MMO' and 'diags' cases -# Any doubt ---> vguemas@ic3.cat -# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - 'ext_raw_oce') - if [[ $typeoutput == 'MMO' ]] ; then - lstvars=`cdo showvar grid_T_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc` - if [[ $raw_vars_ocean == '' ]] ; then - lstext=`echo $lstvars | sed s/\ /,/g` - else - if [[ $raw_vars_ocean == 'default' ]] ; then - lstextvar=( 'sosstsst' 'sosaline' 'somixhgt' 'somxl010' ) - lstext='' - for varex in ${lstextvar[@]} ; do - if [[ ${lstvars/${varex}/} != ${lstvars} ]] ; then - lstext=`echo ${lstext} ${varex}|sed s/\ /,/g` - fi - done - else - lstext=`echo ${raw_vars_ocean[@]} |sed s/\ /,/g` - fi - fi - if [ -z "$lstext" ] ; then - echo "The list of variables you wish to extract is not in your outputs" - exit 1 - else - ncks -O -v ${lstext} grid_T_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc oce_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc -# ncks -O -v ${lstext[@]} grid_T_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc oce_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - 'ext_raw_ice') - if [[ $typeoutput == 'MMO' ]] ; then - lstvars=`cdo showvar icemod_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc` - if [[ $raw_vars_ice == '' ]] ; then - lstext=`echo $lstvars | sed s/\ /,/g` - else - if [[ $raw_vars_ice == 'default' ]] ; then - lstextvar=( 'isnowthi' 'iicethic' 'ileadfra' 'iicetemp' 'ice_pres' ) - lstext='' - for varex in ${lstextvar[@]} ; do - if [[ ${lstvars/${varex}/} != ${lstvars} ]] ; then - lstext=`echo ${lstext} ${varex}|sed s/\ /,/g` - fi - done - else - lstext=`echo $raw_vars_ice |sed s/\ /,/g` - fi - fi - if [ -z "$lstext" ] ; then - echo "The list of variables you wish to extract is not in your outputs" - exit 1 - else - ncks -O -v ${lstext} icemod_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ice_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - 'heat_sal_mxl') - if [[ $typeoutput == 'MMO' ]] ; then - if [ ! -f heat_sal_mxl_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] ; then - heat_sal_mxl grid_T_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc heat_sal_mxl_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - 'psi') - if [[ $typeoutput == 'MMO' ]] ; then - if [ ! -f psi_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] ; then - psi grid_U_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc grid_V_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc psi_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - 'gyres') - gyres psi_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc $NEMOVERSION gyres_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ;; - - 'lmsalc') - if [[ $typeoutput == 'MMO' ]] ; then - if [ ! -f sal_300-5400m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] ; then - vertmeansal grid_T_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 300 5400 sal_300-5400m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - 'usalc') - if [[ $typeoutput == 'MMO' ]] ; then - if [ ! -f sal_0-300m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] ; then - vertmeansal grid_T_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 0 300 sal_0-300m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - - 'temp_lev') - if [[ $typeoutput == 'MMO' ]] ; then - if [ ! -f temp_lev${level1}-${level2}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] ; then - vertmeanvar grid_T_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc votemper $level1 $level2 temp_lev${level1}-${level2}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - - 'sal_lev') - if [[ $typeoutput == 'MMO' ]] ; then - if [ ! -f sal_lev${level1}-${level2}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] ; then - vertmeanvar grid_T_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc vosaline $level1 $level2 sal_lev${level1}-${level2}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - 'ohc_specified_layer') - if [ ! -f ohc_2d_avg_0-300m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ];then - case $typeoutput in - 'MMO' ) pref='grid_T' ;; - 'diags') pref='t3d' ;; - esac - ohc_specified_layer ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 0.0 300.0 ohc_2d_avg_0-300m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ohc_specified_layer ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 300.0 800.0 ohc_2d_avg_300-800m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - ;; - - 'vert_Tsections') - case $typeoutput in - 'MMO' ) pref='grid_T' ;; - 'diags') pref='t3d' ;; - esac - for coord in 0 45 -45 -30 180 80 - do - if [[ $coord == '0' ]] || [[ $coord == '45' ]] || [[ $coord == '-45' ]] ; then - [[ ` echo $coord | cut -b 1 ` == '-' ]] && direction=S || direction=N - z_m=Z - else - [[ ` echo $coord | cut -b 1 ` == '-' ]] && direction=W || direction=E - z_m=M - fi - coord=`echo $coord | sed -e s/-//g` - [ ! -f temp_${coord}${direction}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] && cutsection ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc votemper $z_m $coord temp_${coord}${direction}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - done - - ;; - - 'vert_Ssections') - if [[ $typeoutput == 'MMO' ]] ; then - pref='grid_T' - for coord in 0 45 -45 -30 180 80 - do - if [[ $coord == '0' ]] || [[ $coord == '45' ]] || [[ $coord == '-45' ]] ; then - [[ ` echo $coord | cut -b 1 ` == '-' ]] && direction=S || direction=N - z_m=Z - else - [[ ` echo $coord | cut -b 1 ` == '-' ]] && direction=W || direction=E - z_m=M - fi - coord=`echo $coord | sed -e s/-//g` - [ ! -f sal_${coord}${direction}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] && cutsection ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc vosaline $z_m $coord sal_${coord}${direction}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - done - fi - ;; - '3dtemp') - case $typeoutput in - 'MMO' ) pref='grid_T' ;; - 'diags') pref='t3d' ;; - esac - if [ ! -f regular3dT_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ]; then - echo " Warning: you are about to perform a 3d interpolation " - [ $warning_T ] && echo "(because you asked for cross sections calculations)" - echo "this might take time to complete (~days), be sure you really need/want to do this..." - interp3d ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc votemper regular3dT_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - ;; - '3dsal') - if [[ $typeoutput == 'MMO' ]] ; then - pref='grid_T' - if [ ! -f regular3dS_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ]; then - echo " Warning: you are about to perform a 3d interpolation " - [ $warning_S ] && echo "(because you asked for cross sections calculations)" - echo "this might take time to complete (~days), be sure you really need/want to do this..." - interp3d ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc vosaline regular3dS_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - 'TSec_ave190-220E') - [ ! -f TSec_ave190-220E_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] && cdo zonmean -sellonlatbox,190,220,-90,90 regular3dT_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc TSec_ave190-220E_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ;; - 'SSec_ave190-220E') - if [[ $typeoutput == 'MMO' ]] ; then - [ ! -f SSec_ave190-220E_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] && cdo zonmean -sellonlatbox,190,220,-90,90 regular3dS_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc SSec_ave190-220E_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - ;; - - 'moc') - if [[ $typeoutput == 'MMO' ]] ; then - if [ ! -f moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] ; then - moc grid_V_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - fi - ;; - - 'max_moc') - max_moc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 38 50 500 2000 max_moc_38N50N_500m-2km_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - max_moc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 40 40 0 10000 max_moc_40N_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ;; - - 'stc') - area_moc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 0.0 25.0 NPac_stc_0N25N_0-200m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 0.0 200.0 zomsfpac - area_moc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc -25.0 0.0 SPac_stc_25S0S_0-200m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 0.0 200.0 zomsfpac - area_moc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 0.0 25.0 NAtl_stc_0N25N_0-200m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 0.0 200.0 - area_moc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc -25.0 0.0 SAtl_stc_25S0S_0-200m_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 0.0 200.0 - ;; - - 'area_moc') - if [ ! -f moc_40N55N_1-2km_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ];then - area_moc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 40.0 55.0 moc_40N55N_1-2km_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - if [ ! -f moc_30N40N_1-2km_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ];then - area_moc moc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc 30.0 40.0 moc_30N40N_1-2km_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - ;; - - 'convection') - convection oce_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc $NEMOVERSION convection_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ;; - - 'siasiesiv' ) - - if [ ! -f siasiesiv_N_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ]||[ ! -f siasiesiv_S_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ];then #check if ? instead of N or S works - case $typeoutput in - 'MMO' ) pref='icemod' ;; - 'diags') pref='ice' ;; - esac - siasiesiv ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc tmp.nc - mv ice_N_tmp.nc siasiesiv_N_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - mv ice_S_tmp.nc siasiesiv_S_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - fi - ;; - - 'siasiesivsit_Arcticreg1') - - case $typeoutput in - 'MMO' ) pref='icemod' ;; - 'diags') pref='ice' ;; - esac - - if [[ $raw_regions_ice == '' ]] ; then - lstseas=$( cdo showvar mask_regions.nc ) - else - if [[ $raw_regions_ice == 'default' ]] ; then - lstseas="Baffin_Bay Baltic_Sea Barents_Sea Beaufort_Sea Bering CanArch Chukchi_Sea CntArctic CntArcticRing1 CntArcticRing2 CntArcticRing3 CntArcticRing4 CntArcticRing5 CntArcticRing6 CntArcticRing7_Lincoln_Sea CntArcticRing8 CntArcticPrf1 CntArcticPrf2r CntArcticPrf3 CntArcticPrf4 East_Siberian_Sea1 Greenland_Sea Hudson Icelandic_Sea Irminger Japan1 Kara_Sea Laptev_Sea Labrador_Sea1 Norwegian_Sea Okhotsk StLawr" - else - lstseas=$( echo ${raw_regions_ice[@]} ) - fi - fi - - for sea in $lstseas ; do - - siasiesiv ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc tmp.nc $sea - - ncks -O -v sia ice_N_tmp.nc sia_N_tmp.nc - ncks -O -v sie ice_N_tmp.nc sie_N_tmp.nc - ncks -O -v siv ice_N_tmp.nc siv_N_tmp.nc - ncks -O -v sit ice_N_tmp.nc sit_N_tmp.nc - - ncrename -h -v sia,sia_$sea sia_N_tmp.nc - ncrename -h -v sie,sie_$sea sie_N_tmp.nc - ncrename -h -v siv,siv_$sea siv_N_tmp.nc - ncrename -h -v sit,sit_$sea sit_N_tmp.nc - - if [ -e sia_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc ] ; then - ncks -A sia_N_tmp.nc sia_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ncks -A sie_N_tmp.nc sie_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ncks -A siv_N_tmp.nc siv_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ncks -A sit_N_tmp.nc sit_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - rm -f sia_N_tmp.nc sie_N_tmp.nc siv_N_tmp.nc sit_N_tmp.nc ice_N_tmp.nc ice_S_tmp.nc - else - mv sia_N_tmp.nc sia_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - mv sie_N_tmp.nc sie_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - mv siv_N_tmp.nc siv_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - mv sit_N_tmp.nc sit_Arcticreg1_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - rm -f ice_N_tmp.nc ice_S_tmp.nc - fi - - done - ;; - - esac - - case `echo $post|cut -c$((${#post}-2))-${#post}` in - 'ohc') - case `echo $post | cut -c1` in - 'x') kmin=0 ; kmax=0 ; start=2 ; mxl=1 ;; - 'l') start=2 ; mxl=0 - case $NEMOVERSION in - 'Ec2.3_O1L42'|'N3.2_O1L42'|'nemovar_O1L42') kmin=25 ; kmax=42 ;; - 'Ec3.0_O1L46'|'Ec3.0_O25L46'|'N3.3_O1L46') kmin=23 ; kmax=46 ;; - 'Ec3.0_O1L75'|'Ec3.0_O25L75'|'glorys2v1_O25L75') kmin=45; kmax=75;; - esac - ;; - 'm') start=2 ; mxl=0 - case $NEMOVERSION in - 'Ec2.3_O1L42'|'N3.2_O1L42'|'nemovar_O1L42') kmin=21 ; kmax=24 ;; - 'Ec3.0_O1L46'|'Ec3.0_O25L46'|'N3.3_O1L46') kmin=18 ; kmax=22 ;; - 'Ec3.0_O1L75'|'Ec3.0_O25L75'|'glorys2v1_O25L75') kmin=35; kmax=44;; - esac - ;; - 'u') kmin=1 ; start=2 ; mxl=0 - case $NEMOVERSION in - 'Ec2.3_O1L42'|'N3.2_O1L42'|'nemovar_O1L42') kmax=20 ;; - 'Ec3.0_O1L46'|'Ec3.0_O25L46'|'N3.3_O1L46') kmax=17 ;; - 'Ec3.0_O1L75'|'Ec3.0_O25L75'|'glorys2v1_O25L75') kmax=34;; - esac - ;; - *) kmin="" ; kmax="" ; start=1 ; mxl=0 ;; - esac - case `echo $post | cut -c${start}-$((start+3))` in - 'ohc') basin='Glob' ;; - *) basin=`echo $post | cut -c${start}-$((start+3))` - esac - case $typeoutput in - 'MMO' ) pref='grid_T' ;; - 'diags') - pref='t3d' - ncks -A -v somxl010,somixhgt oce_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc t3d_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - ;; - esac - ohc ${pref}_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc heatc_${expid}_${yeari}${moni}01_fc${memb}_${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc $basin $mxl $kmin $kmax - ;; - esac - - done - - # Removing the raw output from this start dates and this member - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - clean_diagsMMO ${yeari}${moni}01 ${expid} ${memb} $ltime0 $ltimef $typeoutput "${list_files}" - done - - # Prepare storage : choose output directory and file name - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - for post in ${listpost[@]} ; do - case $post in -# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -# You have created a function ? Enter the output directory and the prefix -# or your(s) output files under the flag chosen -# Any doubt ---> vguemas@ic3.cat -# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - 'area_moc') dirout='moc'; files=('moc_40N55N_1-2km' 'moc_30N40N_1-2km') ;; - 'stc') dirout='moc' ; files=( 'NPac_stc_0N25N_0-200m' 'SPac_stc_25S0S_0-200m' 'NAtl_stc_0N25N_0-200m' 'SAtl_stc_25S0S_0-200m' ) ;; - 'max_moc') dirout='moc' ; files=('max_moc_38N50N_500m-2km' 'max_moc_40N' ) ;; - 'siasiesiv' ) dirout='ice' ; files=('siasiesiv_N' 'siasiesiv_S') ;; - 'siasiesivsit_Arcticreg1' ) dirout='ice' ; files=('sia_Arcticreg1' 'sie_Arcticreg1' 'siv_Arcticreg1' 'sit_Arcticreg1') ;; - 'moc') dirout='moc' ; files=('moc') ;; - 'convection') dirout='moc' ; files=('convection') ;; -# 'ext_raw_ice') dirout='ice' ; files=('ice_raw') ;; -# 'ext_raw_oce') dirout='oce_raw' ; files=('oce_raw') ;; - 'ext_raw_ice') dirout='ice' ; files=('ice') ;; - 'ext_raw_oce') dirout='oce' ; files=('oce') ;; - 'heat_sal_mxl') dirout='heatc' ; files=('heat_sal_mxl') ;; - 'psi') dirout='psi' ; files=('psi') ;; - 'gyres') dirout='psi' ; files=('gyres') ;; - 'usalc') dirout='saltc' ; files=('sal_0-300m') ;; - 'temp_lev') dirout='temp_lev'${level1}-${level2} ; files=('temp_lev'${level1}-${level2}) ;; - 'sal_lev') dirout='sal_lev'${level1}-${level2} ; files=('sal_lev'${level1}-${level2}) ;; - 'lmsalc') dirout='saltc' ; files=('sal_300-5400m') ;; - 'ohc_specified_layer') dirout='heatc' ; files=('ohc_2d_avg_0-300m' 'ohc_2d_avg_300-800m') ;; - 'vert_Tsections') dirout='sections' ; files=('temp_0N' 'temp_45N' 'temp_45S' 'temp_30W' 'temp_80E' 'temp_180E') ;; - 'vert_Ssections') dirout='sections' ; files=('sal_0N' 'sal_45N' 'sal_45S' 'sal_30W' 'sal_80E' 'sal_180E') ;; - '3dtemp') dirout='InterpT' ; files=('regular3dT') ;; - '3dsal') dirout='InterpS' ; files=('regular3dS') ;; - 'TSec_ave190-220E') dirout='sections' ; files=('TSec_ave190-220E') ;; - 'SSec_ave190-220E') dirout='sections' ; files=('SSec_ave190-220E') ;; - esac - case `echo $post|cut -c$((${#post}-2))-${#post}` in - 'ohc') - dirout='heatc' - file='heatc' - case `echo $post | cut -c1` in - 'x') mxl=1 ; start=2 ;; - 'l') start=2 ; mxl=0 - case $NEMOVERSION in - 'Ec2.3_O1L42'|'N3.2_O1L42'|'nemovar_O1L42') file='800-5350_'${file} ;; - 'Ec3.0_O1L46'|'Ec3.0_O25L46'|'N3.3_O1L46') file='855-5875_'${file} ;; - 'Ec3.0_O1L75'|'Ec3.0_O25L75'|'glorys2v1_O25L75') file='857-5902_'${file};; - esac - ;; - 'm') start=2 ; mxl=0 - case $NEMOVERSION in - 'Ec2.3_O1L42'|'N3.2_O1L42'|'nemovar_O1L42') file='373-657_'${file} ;; - 'Ec3.0_O1L46'|'Ec3.0_O25L46'|'N3.3_O1L46') file='382-735_'${file} ;; - 'Ec3.0_O1L75'|'Ec3.0_O25L75'|'glorys2v1_O25L75') file='301-773_'${file};; - esac - ;; - 'u') start=2 ; mxl=0 - case $NEMOVERSION in - 'Ec2.3_O1L42'|'N3.2_O1L42'|'nemovar_O1L42') file='0-315_'${file} ;; - 'Ec3.0_O1L46'|'Ec3.0_O25L46'|'N3.3_O1L46') file='0-322_'${file} ;; - 'Ec3.0_O1L75'|'Ec3.0_O25L75'|'glorys2v1_O25L75') file='0-271_'${file};; - esac - ;; - *) mxl=0 ; start=1 ;; - esac - - case `echo $post | cut -c${start}-$((start+3))` in - 'NAtl') file='NAtl_10N65N_'${file} ;; - 'TAtl') file='TAtl_30S30N_'${file};; - 'NPac') file='NPac_10N70N_'${file} ;; - 'TPac') file='TPac_30S30N_'${file} ;; - 'Arct') file='Arc_65N90N_'${file} ;; - 'Anta') file='Ant_90S60S_'${file} ;; - 'TInd') file='TInd_30S30N_'${file} ;; - esac - if [[ $mxl == 1 ]] ; then - file='mxl_'$file - fi - files=( $file ) - esac - pathout=${rootout}/${dirout} - mkdir -m ug+w -m o-w -p $pathout - for file in ${files[@]} ; do - prefix=${file}_${expid}_${yeari}${moni}01_fc - lsmbso=0-${listmemb[${#listmemb[@]}-1]} - # - # Merging the post-processed members together and with the previous members if existing - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - lsyrsh=${year0}$(printf "%02d" ${mon0})_${yearf}$(printf "%02d" ${monf}).nc - lsmbsh=${listmemb[0]}-${listmemb[${#listmemb[@]}-1]} - lsmbsb=0-$((${listmemb[0]}-1)) - if [ -e ${pathout}/${prefix}${lsmbsb}_${lsyrsh} ] ; then - cp ${pathout}/${prefix}${lsmbsb}_${lsyrsh} . - lsmbsh=0-${listmemb[${#listmemb[@]}-1]} - fi - gather_memb ${prefix} _${lsyrsh} ${prefix}${lsmbsh}_${lsyrsh} - for jmemb in ${listmemb[@]} ; do - rm -f ${prefix}${jmemb}_${lsyrsh} - done - # - # Concatenating the result with the previous years if existing - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -# You have created a function ? If your diagnostic provides yearly output -# you need to use the concat option rather than the ncrcat one below. -# Any doubt ---> vguemas@ic3.cat -# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - lsyrsb=${yeari}${moni}_$((year0-(1-(10#$mon0+10)/12)))$(printf "%02d" $(((mon0-13)%12+12)) ).nc - lsyrso=${yeari}${moni}_${yearf}$(printf "%02d" ${monf}).nc - if [ -e ${pathout}/${prefix}${lsmbsh}_${lsyrsb} ] ; then - case $post in - 'max_moc' ) concat ${pathout}/${prefix}${lsmbsh}_${lsyrsb} ${prefix}${lsmbsh}_${lsyrsh} $(printf "%02d" ${monf}) ${prefix}${lsmbsh}_${lsyrso} ;; - *) ncrcat -O ${pathout}/${prefix}${lsmbsh}_${lsyrsb} ${prefix}${lsmbsh}_${lsyrsh} ${prefix}${lsmbsh}_${lsyrso} ;; - esac - else - lsyrso=$lsyrsh - fi - # - # Merging the result with the previous members if existing - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - if [[ $lsyrsh != $lsyrso ]] && [[ -e ${pathout}/${prefix}${lsmbsb}_${lsyrso} ]] ; then - cp ${pathout}/${prefix}${lsmbsb}_${lsyrso} . - gather_memb ${prefix} _${lsyrso} ${prefix}${lsmbso}_${lsyrso} - else - lsmbso=$lsmbsh - fi - # - # Storing and cleaning - # ~~~~~~~~~~~~~~~~~~~~~ - cp ${prefix}${lsmbso}_${lsyrso} ${pathout}/. || { if [ -e ${pathout}/${prefix}${lsmbso}_${lsyrso} ]; - then - echo "${prefix}${lsmbso}_${lsyrso} already exists in ${pathout}" - sleep 5 - else - echo " problem writing file in ${pathout} directory" - exit - fi - } - rm -f ${pathout}/${prefix}${lsmbsh}_${lsyrsb} ${prefix}${lsmbsh}_${lsyrso} ${prefix}${lsmbsb}_${lsyrso} ${pathout}/${prefix}${lsmbsb}_${lsyrso} ${prefix}${lsmbso}_${lsyrso} ${pathout}/${prefix}${lsmbsb}_${lsyrsh} ${prefix}${lsmbsb}_${lsyrsh} - done - done -done - -trap - EXIT -rm -rf $WORKDIR diff --git a/patch.bash b/patch.bash new file mode 100755 index 0000000000000000000000000000000000000000..60e7b8f885c9e7462391aee55c33bd030ec2869c --- /dev/null +++ b/patch.bash @@ -0,0 +1,18 @@ +#!/bin/bash +set -o nounset +set -o errexit + +exp=a0fj +sd=19580101 +root=esarchive +mod=nemo + +if [ -d /${root}/exp/${mod}/${exp}/original_files ] +then + echo "folder already exists" + exit +fi + +mkdir -p /${root}/exp/${mod}/${exp}/original_files +mv /${root}/exp/${mod}/${exp}/${sd} /${root}/exp/${mod}/${exp}/original_files +ln -sf /${root}/exp/${mod}/${exp}/original_files/${sd} /${root}/exp/${mod}/${exp}/${sd} diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..0e68a0892e2f5df5df0cd7afc2a14c8a7475dd62 --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Installation script for EarthDiagnostics package +""" + +from os import path + +from setuptools import find_packages +from setuptools import setup + +here = path.abspath(path.dirname(__file__)) + +# Get the version number from the relevant file +with open(path.join(here, 'VERSION')) as f: + version = f.read().strip() + +setup( + name='earthdiagnostics', + license='GNU GPL v3', + platforms=['GNU/Linux Debian'], + version=version, + description='EarthDiagnostics', + author='BSC-CNS Earth Sciences Department', + author_email='javier.vegas@bsc.es', + url='http://www.bsc.es/projects/earthsciences/autosubmit/', + keywords=['climate', 'weather', 'diagnostic'], + setup_requires=['pyproj'], + install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo>=1.3.4', 'nco>=0.0.3', 'iris>=1.12.0', 'coverage', + 'pygrib', 'openpyxl', 'mock', 'futures', 'cf_units', 'cfunits', 'xxhash', 'six', 'psutil', + 'exrex'], + packages=find_packages(), + include_package_data=True, + scripts=['bin/earthdiags'] +) diff --git a/setup_development.bash b/setup_development.bash deleted file mode 100755 index 9591a894ed1b6aed6278021684a07ed2bc4fcfac..0000000000000000000000000000000000000000 --- a/setup_development.bash +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# -# This script intends to support the development of new functions in -# common_ocean_post.txt of new options to existing funtions. It allows to -# test quickly these developments -# -# History : Virginie Guemas - Initial version - 2012 -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -set -evx -# This does not need to be changed : -CON_FILES='/esnas/autosubmit/con_files' -# This option is compatible with the testing on i00k but if you want to test -# your developments on another experiment you would need to change this : -NEMOVERSION=Ec2.3_O1L42 -# This lines needs to be replaced by the path were you stored your modifed -# version of common_ocean_post.txt : -source /home/Earth/rcruzgar/ocean_diagnostics/common_ocean_post.txt -# Here we only fetch one random chunk of outputs from i00k : -cp /esnas/exp/ecearth/i00k/19601101/fc0/outputs/MMO_i00k_19601101_fc0_19601101-19610228.tar . -# Here we untar and gunzip these files : -tar -xvf MMO_i00k_19601101_fc0_19601101-19610228.tar -gunzip *.gz -# The lines below might need to be changed depending on which function you need -# to test. In the case below, the grid_T files are needed because they contain -# the 3d temperature = input to ohc function. If you test siasiesiv for exemple, -# you would need to replace grid_T by icemod. -filein=`ls *grid_T*` -cdo cat *grid_T* tmp.nc -# The lines below are essential because the files have a time dimension named -# time in all the functions from common_ocean_post.txt (this is handled in -# ocean_pp.bash and in the templates) -timevar=`ncdump -h tmp.nc | grep UNLIMITED | awk '{print $1}'` -if [[ $timevar == 'time_counter' ]] ; then ncrename -v time_counter,time -d time_counter,time tmp.nc ; fi -# Some cleaning -rm -f *grid* *icemod* -# This is the final testing line. You need to replace that line by the function -# you want to test (here it is ohc) followed by all its arguments (here we have -# only the input file tmp.nc and the output file tmpout.nc) -ohc tmp.nc tmppout.nc 'Glob' diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb --- /dev/null +++ b/test/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/test/run_test.py b/test/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..59eec7e98a18694833f660a50431d2ea696de630 --- /dev/null +++ b/test/run_test.py @@ -0,0 +1,26 @@ +# coding=utf-8 +""" +Script to run the tests for EarthDiagnostics and generate the code coverage report +""" + +import coverage +import unittest +import os +work_path = os.path.abspath('.') +source_path = os.path.join(work_path, '..', 'earthdiagnostics', '*') +print(source_path) +cov = coverage.Coverage(include=source_path) +cov.set_option("run:branch", True) +cov.set_option("html:title", 'Coverage report for EarthDiagnostics') + +cov.start() +suite = unittest.TestLoader().discover('.') +unittest.TextTestRunner(verbosity=2).run(suite) +cov.stop() + +cov.save() +cov.report() +cov.html_report() + + + diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb --- /dev/null +++ b/test/unit/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/test/unit/general/__init__.py b/test/unit/general/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb --- /dev/null +++ b/test/unit/general/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/test/unit/general/test_attribute.py b/test/unit/general/test_attribute.py new file mode 100644 index 0000000000000000000000000000000000000000..da6d4146b090a3e529678a332f1ba9fe705a8eb1 --- /dev/null +++ b/test/unit/general/test_attribute.py @@ -0,0 +1,54 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.attribute import Attribute +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestAttribute(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = Attribute.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'att', 'value']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', '', + 'att', 'value')) + self.assertEqual(jobs[1], Attribute(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', '', + 'att', 'value')) + + jobs = Attribute.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', 'att', 'value', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', 'grid', + 'att', 'value')) + self.assertEqual(jobs[1], Attribute(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, 'var', 'grid', + 'att', 'value')) + + with self.assertRaises(Exception): + Attribute.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + Attribute.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + mixed = Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'att', 'value') + self.assertEquals(str(mixed), 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 ' + 'Variable: atmos:var Attributte: att:value Grid: grid') diff --git a/test/unit/general/test_dailymean.py b/test/unit/general/test_dailymean.py new file mode 100644 index 0000000000000000000000000000000000000000..ec85d9dcea51e903b8be04b8f3c57b4a9769eb2b --- /dev/null +++ b/test/unit/general/test_dailymean.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.dailymean import DailyMean +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestDailyMean(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = DailyMean.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '6hr']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], DailyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + Frequencies.six_hourly, '')) + self.assertEqual(jobs[1], DailyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + Frequencies.six_hourly, '')) + + jobs = DailyMean.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', '3h', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], DailyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', + Frequencies.three_hourly, 'grid')) + self.assertEqual(jobs[1], DailyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, 'var', + Frequencies.three_hourly, 'grid')) + + with self.assertRaises(Exception): + DailyMean.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + DailyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + mixed = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + self.assertEquals(str(mixed), 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_module.py b/test/unit/general/test_module.py new file mode 100644 index 0000000000000000000000000000000000000000..e01ecf8c277db292cc8c786f399f473caa826229 --- /dev/null +++ b/test/unit/general/test_module.py @@ -0,0 +1,54 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.module import Module +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestModule(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = Module.generate_jobs(self.diags, ['diagnostic', 'atmos', 'varu', 'varv', 'varmodule']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Module(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, + 'varu', 'varv', 'varmodule', '')) + self.assertEqual(jobs[1], Module(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, + 'varu', 'varv', 'varmodule', '')) + + jobs = Module.generate_jobs(self.diags, ['diagnostic', 'seaIce', 'varu', 'varv', 'varmodule', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Module(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, + 'varu', 'varv', 'varmodule', 'grid')) + self.assertEqual(jobs[1], Module(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, + 'varu', 'varv', 'varmodule', 'grid')) + + with self.assertRaises(Exception): + Module.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + Module.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + mixed = Module(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'varu', 'varv', 'varmodule', 'grid') + self.assertEquals(str(mixed), 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' + 'Variables: atmos:varu,varv,varmodule Grid: grid') diff --git a/test/unit/general/test_monthlymean.py b/test/unit/general/test_monthlymean.py new file mode 100644 index 0000000000000000000000000000000000000000..b31561ac1fbfe9a9d8e40e80a9329b141cb27ed8 --- /dev/null +++ b/test/unit/general/test_monthlymean.py @@ -0,0 +1,63 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.monthlymean import MonthlyMean +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestMonthlyMean(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + self.mixed = MonthlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + Frequencies.daily, '')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + Frequencies.daily, '')) + + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'monthly']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + Frequencies.monthly, '')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + Frequencies.monthly, '')) + + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', 'mon', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', + Frequencies.monthly, 'grid')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, 'var', + Frequencies.monthly, 'grid')) + + with self.assertRaises(Exception): + MonthlyMean.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + MonthlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var') diff --git a/test/unit/general/test_relink.py b/test/unit/general/test_relink.py new file mode 100644 index 0000000000000000000000000000000000000000..835b0bcd6f944847430720abade1e52dc95357b3 --- /dev/null +++ b/test/unit/general/test_relink.py @@ -0,0 +1,61 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.relink import Relink +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestRelink(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = Relink.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, + 'var', True, '')) + self.assertEqual(jobs[1], Relink(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, + 'var', True, '')) + + jobs = Relink.generate_jobs(self.diags, ['diagnostic', 'seaIce', 'var', 'False']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, + 'var', False, '')) + self.assertEqual(jobs[1], Relink(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, + 'var', False, '')) + + jobs = Relink.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'True', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, + 'var', True, 'grid')) + self.assertEqual(jobs[1], Relink(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, + 'var', True, 'grid')) + + with self.assertRaises(Exception): + Relink.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + Relink.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + mixed = Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', True, 'grid') + self.assertEquals(str(mixed), 'Relink output Startdate: 20010101 Member: 0 Chunk: 0 Move old: True ' + 'Variable: ocean:var Grid: grid') diff --git a/test/unit/general/test_relinkall.py b/test/unit/general/test_relinkall.py new file mode 100644 index 0000000000000000000000000000000000000000..cf8c9a163437d5d868401a54b6df62195d0e1f7a --- /dev/null +++ b/test/unit/general/test_relinkall.py @@ -0,0 +1,39 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.relinkall import RelinkAll +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestRelinkAll(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = RelinkAll.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0], RelinkAll(self.data_manager, '20010101')) + + with self.assertRaises(Exception): + RelinkAll.generate_jobs(self.diags, ['diagnostic', '0']) + + def test_str(self): + mixed = RelinkAll(self.data_manager, '20010101') + self.assertEquals(str(mixed), 'Relink all output Startdate: 20010101') diff --git a/test/unit/general/test_rewrite.py b/test/unit/general/test_rewrite.py new file mode 100644 index 0000000000000000000000000000000000000000..202e2c39afddfaae8e23f3edd2be7ff3f74a336d --- /dev/null +++ b/test/unit/general/test_rewrite.py @@ -0,0 +1,51 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.rewrite import Rewrite +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestRewrite(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + self.mixed = Rewrite(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', 'grid') + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = Rewrite.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'original')) + self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'original')) + + jobs = Rewrite.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'grid')) + self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'grid')) + + with self.assertRaises(Exception): + Rewrite.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + Rewrite.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: atmos:var Grid: grid') diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py new file mode 100644 index 0000000000000000000000000000000000000000..e7697cc26076c55e7a8d50ea65031a74305d3ac0 --- /dev/null +++ b/test/unit/general/test_scale.py @@ -0,0 +1,71 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.general.scale import Scale +from earthdiagnostics.frequency import Frequencies +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestScale(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', '', + float('nan'), float('nan'), Frequencies.monthly)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', '', + float('nan'), float('nan'), Frequencies.monthly)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + float('nan'), float('nan'), Frequencies.monthly)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + float('nan'), float('nan'), Frequencies.monthly)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.monthly)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.monthly)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly)) + + with self.assertRaises(DiagnosticOptionError): + Scale.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', + 'extra']) + + def test_str(self): + mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, + Frequencies.three_hourly) + self.assertEquals(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' + 'Variable: atmos:var Frequency: 3hr') diff --git a/test/unit/general/test_select_levels.py b/test/unit/general/test_select_levels.py new file mode 100644 index 0000000000000000000000000000000000000000..32e7424da69948ecf986eb8295b3e4a0b03bb168 --- /dev/null +++ b/test/unit/general/test_select_levels.py @@ -0,0 +1,66 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.general.select_levels import SelectLevels +from earthdiagnostics.frequency import Frequencies +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestSelectLevels(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value.split('-') + + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '20']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + '', 0, 20)) + self.assertEqual(jobs[1], SelectLevels(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + '', 0, 20)) + + jobs = SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var1-var2', '0', '20']) + self.assertEqual(len(jobs), 4) + self.assertEqual(jobs[0], SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var1', + '', 0, 20)) + self.assertEqual(jobs[1], SelectLevels(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var1', + '', 0, 20)) + self.assertEqual(jobs[2], SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var2', + '', 0, 20)) + self.assertEqual(jobs[3], SelectLevels(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var2', + '', 0, 20)) + + jobs = SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '20', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + 'grid', 0, 20)) + self.assertEqual(jobs[1], SelectLevels(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + 'grid', 0, 20)) + + with self.assertRaises(DiagnosticOptionError): + SelectLevels.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '20', 'grid', 'extra']) + + def test_str(self): + mixed = SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 20) + self.assertEquals(str(mixed), 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Levels: 0-20 Grid: grid') diff --git a/test/unit/general/test_simplify_dimensions.py b/test/unit/general/test_simplify_dimensions.py new file mode 100644 index 0000000000000000000000000000000000000000..429ad6f2344ed5d5dd4f67d38c89ed53b1ffd807 --- /dev/null +++ b/test/unit/general/test_simplify_dimensions.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.general.simplify_dimensions import SimplifyDimensions +from earthdiagnostics.frequency import Frequencies +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestSimplifyDimensions(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value.split('-') + + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], SimplifyDimensions(self.data_manager, '20010101', 0, 0, + ModelingRealms.atmos, 'var', '')) + self.assertEqual(jobs[1], SimplifyDimensions(self.data_manager, '20010101', 0, 1, + ModelingRealms.atmos, 'var', '')) + + jobs = SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], SimplifyDimensions(self.data_manager, '20010101', 0, 0, + ModelingRealms.atmos, 'var', 'grid')) + self.assertEqual(jobs[1], SimplifyDimensions(self.data_manager, '20010101', 0, 1, + ModelingRealms.atmos, 'var', 'grid')) + + with self.assertRaises(DiagnosticOptionError): + SimplifyDimensions.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid', 'extra']) + + def test_str(self): + mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid') + self.assertEquals(str(mixed), 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Grid: grid') diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py new file mode 100644 index 0000000000000000000000000000000000000000..cd2876fe9f2a62b91ee28616a5dee862b8f79dbf --- /dev/null +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -0,0 +1,74 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris +from earthdiagnostics.frequency import Frequencies +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestVerticalMeanMetersIris(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + box = Box(True) + + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, + ModelingRealms.ocean, 'var', box)) + self.assertEqual(jobs[1], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 1, + ModelingRealms.ocean, 'var', box)) + + box = Box(True) + box.min_depth = 0 + box.max_depth = 100 + + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, + ModelingRealms.ocean, 'var', box)) + self.assertEqual(jobs[1], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 1, + ModelingRealms.ocean, 'var', box)) + + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100', 'seaIce']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, + ModelingRealms.seaIce, 'var', box)) + self.assertEqual(jobs[1], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 1, + ModelingRealms.seaIce, 'var', box)) + + with self.assertRaises(DiagnosticOptionError): + VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100', 'seaIce', 'extra']) + + def test_str(self): + box = Box(True) + box.min_depth = 0 + box.max_depth = 100 + mixed = VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', box) + self.assertEquals(str(mixed), 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Box: 0-100m') diff --git a/test/unit/general/test_yearlymean.py b/test/unit/general/test_yearlymean.py new file mode 100644 index 0000000000000000000000000000000000000000..dcf5ad75ea16b60f061c44d17e8eada524cdca56 --- /dev/null +++ b/test/unit/general/test_yearlymean.py @@ -0,0 +1,63 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.yearlymean import YearlyMean +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestYearlyMean(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + self.mixed = YearlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = YearlyMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], YearlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + Frequencies.monthly, '')) + self.assertEqual(jobs[1], YearlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, '')) + + jobs = YearlyMean.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'day']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], YearlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + Frequencies.daily, '')) + self.assertEqual(jobs[1], YearlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + Frequencies.daily, '')) + + jobs = YearlyMean.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', 'mon', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], YearlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', + Frequencies.monthly, 'grid')) + self.assertEqual(jobs[1], YearlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, 'var', + Frequencies.monthly, 'grid')) + + with self.assertRaises(Exception): + YearlyMean.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + YearlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/ocean/__init__.py b/test/unit/ocean/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb --- /dev/null +++ b/test/unit/ocean/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/test/unit/ocean/test_areamoc.py b/test/unit/ocean/test_areamoc.py new file mode 100644 index 0000000000000000000000000000000000000000..c1361b8393676bc6fee3106ebf1bd451b34abd92 --- /dev/null +++ b/test/unit/ocean/test_areamoc.py @@ -0,0 +1,57 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.box import Box +from earthdiagnostics.constants import Basins, Basin +from earthdiagnostics.ocean.areamoc import AreaMoc +from mock import Mock, patch + + +class TestAreaMoc(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + self.basins = Mock() + self.basins.Global = Basin('Global') + self.basins.Atlantic = Basin('Atlantic') + + self.box = Box() + self.box.min_lat = 0 + self.box.max_lat = 0 + self.box.min_depth = 0 + self.box.max_depth = 0 + + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.psi = AreaMoc(self.data_manager, '20000101', 1, 1, self.basins.Atlantic, self.box) + + def fake_parse(self, value): + if type(value) is Basin: + return value + if value == 'atl': + value = 'Atlantic' + else: + value = 'Global' + + return Basin(value) + + @patch.object(Basins, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], AreaMoc(self.data_manager, '20010101', 0, 0, self.basins.Global, self.box)) + self.assertEqual(jobs[1], AreaMoc(self.data_manager, '20010101', 0, 1, self.basins.Global, self.box)) + + jobs = AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', 'atl']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], AreaMoc(self.data_manager, '20010101', 0, 0, self.basins.Atlantic, self.box)) + self.assertEqual(jobs[1], AreaMoc(self.data_manager, '20010101', 0, 1, self.basins.Atlantic, self.box)) + + with self.assertRaises(Exception): + AreaMoc.generate_jobs(self.diags, ['diagnostic']) + with self.assertRaises(Exception): + AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.psi), 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0 ' + 'Basin: Atlantic') diff --git a/test/unit/ocean/test_averagesection.py b/test/unit/ocean/test_averagesection.py new file mode 100644 index 0000000000000000000000000000000000000000..d3be4b2bba95b3f66515ee07af45eab51d42b1d5 --- /dev/null +++ b/test/unit/ocean/test_averagesection.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.box import Box +from earthdiagnostics.ocean.averagesection import AverageSection +from mock import Mock, patch +from earthdiagnostics.diagnostic import DiagnosticVariableOption + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestAverageSection(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + + self.box = Box() + self.box.min_lat = 0 + self.box.max_lat = 0 + self.box.min_lon = 0 + self.box.max_lon = 0 + + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def fake_parse(self, value): + return value + + # noinspection PyUnresolvedReferences + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = AverageSection.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', '0', '0', '0', '0']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + self.box, '')) + self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + self.box, '')) + + jobs = AverageSection.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', '0', '0', '0', '0', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + self.box, 'grid')) + self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + self.box, 'grid')) + + with self.assertRaises(Exception): + AverageSection.generate_jobs(self.diags, ['diagnostic']) + with self.assertRaises(Exception): + AverageSection.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', '0', '0', '0', '0', 'grid', + 'extra']) + + def test_str(self): + diag = AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', self.box, 'grid') + self.assertEquals(str(diag), 'Average section Startdate: 20010101 Member: 0 Chunk: 0 Box: 0N0E ' + 'Variable: ocean:var Grid: grid') diff --git a/test/unit/ocean/test_convectionsites.py b/test/unit/ocean/test_convectionsites.py new file mode 100644 index 0000000000000000000000000000000000000000..63710d054e6a4f937cb9c22e8e78d659c80f3c73 --- /dev/null +++ b/test/unit/ocean/test_convectionsites.py @@ -0,0 +1,28 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.ocean.convectionsites import ConvectionSites +from mock import Mock + + +class TestConvectionSites(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.psi = ConvectionSites(self.data_manager, '20000101', 1, 1, 'model_version') + + def test_generate_jobs(self): + jobs = ConvectionSites.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], ConvectionSites(self.data_manager, '20010101', 0, 0, 'model_version')) + self.assertEqual(jobs[1], ConvectionSites(self.data_manager, '20010101', 0, 1, 'model_version')) + + with self.assertRaises(Exception): + ConvectionSites.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.psi), 'Convection sites Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_cutsection.py b/test/unit/ocean/test_cutsection.py new file mode 100644 index 0000000000000000000000000000000000000000..a658b06d5524c96b2e2bf4f73e32958fcffcef4d --- /dev/null +++ b/test/unit/ocean/test_cutsection.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.ocean.cutsection import CutSection +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestCutSection(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + + self.box = Box() + self.box.min_lat = 0 + self.box.max_lat = 0 + self.box.min_lon = 0 + self.box.max_lon = 0 + + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.psi = CutSection(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', True, 0) + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = CutSection.generate_jobs(self.diags, ['diagnostic', 'var', 'true', '10']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + True, 10)) + self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + True, 10)) + + jobs = CutSection.generate_jobs(self.diags, ['diagnostic', 'var', 'false', '0', 'atmos']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + False, 0)) + self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + False, 0)) + + with self.assertRaises(DiagnosticOptionError): + CutSection.generate_jobs(self.diags, ['diagnostic']) + with self.assertRaises(DiagnosticOptionError): + CutSection.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.psi), 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' + 'Zonal: True Value: 0') diff --git a/test/unit/ocean/test_gyres.py b/test/unit/ocean/test_gyres.py new file mode 100644 index 0000000000000000000000000000000000000000..8a0255835c0373f317d9885864a2e5df14a10610 --- /dev/null +++ b/test/unit/ocean/test_gyres.py @@ -0,0 +1,29 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.gyres import Gyres +from mock import Mock + + +class TestGyres(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.gyres = Gyres(self.data_manager, '20000101', 1, 1, 'model_version') + + def test_generate_jobs(self): + jobs = Gyres.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Gyres(self.data_manager, '20010101', 0, 0, 'model_version')) + self.assertEqual(jobs[1], Gyres(self.data_manager, '20010101', 0, 1, 'model_version')) + + with self.assertRaises(Exception): + Gyres.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.gyres), 'Gyres Startdate: 20000101 Member: 1 Chunk: 1 Model version: model_version') diff --git a/test/unit/ocean/test_heatcontent.py b/test/unit/ocean/test_heatcontent.py new file mode 100644 index 0000000000000000000000000000000000000000..c58d1217a9338e0dd422a21a3d3d3fb6926ef124 --- /dev/null +++ b/test/unit/ocean/test_heatcontent.py @@ -0,0 +1,47 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.box import Box +from earthdiagnostics.constants import Basins +from earthdiagnostics.ocean.heatcontent import HeatContent +from mock import Mock, patch + + +# noinspection PyUnusedLocal +def _get_levels_from_meters_mock(cls, box): + return 20, 10 + + +class TestHeatContent(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box(True) + self.box.min_depth = 0 + self.box.max_depth = 100 + + @patch('earthdiagnostics.ocean.heatcontent.HeatContent._get_levels_from_meters') + def test_generate_jobs(self, levels_mock): + levels_mock.return_value = (1, 20) + jobs = HeatContent.generate_jobs(self.diags, ['diagnostic', 'Global', '-1', '0', '100']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], HeatContent(self.data_manager, '20010101', 0, 0, Basins().Global, -1, + self.box, 0, 0)) + self.assertEqual(jobs[1], HeatContent(self.data_manager, '20010101', 0, 1, Basins().Global, -1, + self.box, 0, 0)) + + with self.assertRaises(Exception): + HeatContent.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + HeatContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + diag = HeatContent(self.data_manager, '20010101', 0, 0, Basins().Global, -1, self.box, 1, 20) + self.assertEquals(str(diag), 'Heat content Startdate: 20010101 Member: 0 Chunk: 0 Mixed layer: -1 Box: 0-100m ' + 'Basin: Global') diff --git a/test/unit/ocean/test_heatcontentlayer.py b/test/unit/ocean/test_heatcontentlayer.py new file mode 100644 index 0000000000000000000000000000000000000000..bf8135c65ed2a39b633660b410e14874f058766c --- /dev/null +++ b/test/unit/ocean/test_heatcontentlayer.py @@ -0,0 +1,25 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.box import Box +from earthdiagnostics.ocean.heatcontentlayer import HeatContentLayer +from mock import Mock + + +class TestHeatContentLayer(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.weight = Mock() + + self.box = Box(True) + self.box.min_depth = 0 + self.box.max_depth = 100 + + def test_str(self): + diag = HeatContentLayer(self.data_manager, '20000101', 1, 1, self.box, self.weight, 0, 10) + self.assertEquals(str(diag), 'Heat content layer Startdate: 20000101 Member: 1 Chunk: 1 Box: 0-100m') diff --git a/test/unit/ocean/test_interpolate.py b/test/unit/ocean/test_interpolate.py new file mode 100644 index 0000000000000000000000000000000000000000..4d19949ff4b0d09a33057008119c9e14c33f7493 --- /dev/null +++ b/test/unit/ocean/test_interpolate.py @@ -0,0 +1,76 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.interpolate import Interpolate +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.diagnostic import DiagnosticVariableListOption + + +class TestInterpolate(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.model_version = 'model_version' + + def fake_parse(self, value): + return value.split('-') + + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'grid', + 'model_version', False, '')) + + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var1-var2']) + self.assertEqual(len(jobs), 4) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var1', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var1', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[2], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var2', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[3], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var2', 'grid', + 'model_version', False, '')) + + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', + 'model_version', False, '')) + + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, '')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, '')) + + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true', 'original_grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, 'original_grid')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, 'original_grid')) + + with self.assertRaises(Exception): + Interpolate.generate_jobs(self.diags, ['interp']) + + with self.assertRaises(Exception): + Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true', 'original_grid', 'extra']) + + def test_str(self): + diag = Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, 'original_grid') + self.assertEquals(str(diag), 'Interpolate Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Target grid: grid Invert lat: True Model: model_version ' + 'Original grid: original_grid') diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py new file mode 100644 index 0000000000000000000000000000000000000000..04b085521407b34572a6edc95f238906de21b8b0 --- /dev/null +++ b/test/unit/ocean/test_interpolatecdo.py @@ -0,0 +1,89 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.interpolatecdo import InterpolateCDO +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError + + +class TestInterpolate(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.model_version = 'model_version' + self.diags.config.experiment.atmos_grid = 'atmos_grid' + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value.split('-') + + @patch('earthdiagnostics.ocean.interpolatecdo.InterpolateCDO._compute_weights') + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + def test_generate_jobs(self, mock_weights): + mock_weights.return_value = None + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'atmos_grid', 'model_version', True, '', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'atmos_grid', 'model_version', True, '', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', True, '', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', True, '', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid', 'bicubic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', True, '', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', True, '', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid', 'bicubic', + 'false']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, '', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, '', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid', 'bicubic', + 'false', 'orig']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, 'orig', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, 'orig', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid', 'bicubic', + 'false', 'orig', 'false']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, 'orig', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, 'orig', None)) + + with self.assertRaises(DiagnosticOptionError): + InterpolateCDO.generate_jobs(self.diags, ['interp']) + + with self.assertRaises(DiagnosticOptionError): + InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'bicubic', 'false', 'orig', 'false', + 'extra']) + + def test_str(self): + diag = InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'atmos_grid', 'model_version', False, 'orig', None) + self.assertEquals(str(diag), 'Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var ' + 'Target grid: atmos_grid Original grid: orig Mask ocean: False ' + 'Model: model_version') diff --git a/test/unit/ocean/test_maskland.py b/test/unit/ocean/test_maskland.py new file mode 100644 index 0000000000000000000000000000000000000000..ede009114fda18e2582a6ce7ba7372520d436192 --- /dev/null +++ b/test/unit/ocean/test_maskland.py @@ -0,0 +1,65 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.ocean.mask_land import MaskLand +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestMaskLand(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + + self.box = Box() + self.box.min_lat = 0 + self.box.max_lat = 0 + self.box.min_lon = 0 + self.box.max_lon = 0 + + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value.split('-') + + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + @patch('earthdiagnostics.ocean.mask_land.MaskLand._get_mask') + def test_generate_jobs(self, get_mask_mock): + get_mask_mock.return_value = None + jobs = MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 't', '')) + self.assertEqual(jobs[1], MaskLand(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 't', '')) + + for mask in ('t', 'u', 'v', 'f', 'w'): + jobs = MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', mask]) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + mask, '')) + self.assertEqual(jobs[1], MaskLand(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + mask, '')) + + with self.assertRaises(DiagnosticOptionError): + MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'BAD']) + + jobs = MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 't', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 't', 'grid')) + self.assertEqual(jobs[1], MaskLand(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 't', 'grid')) + + with self.assertRaises(DiagnosticOptionError): + MaskLand.generate_jobs(self.diags, ['diagnostic']) + with self.assertRaises(DiagnosticOptionError): + MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 't', 'grid', 'extra']) + + def test_str(self): + diag = MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 't', 'grid') + self.assertEquals(str(diag), 'Land mask Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var Grid: grid') diff --git a/test/unit/ocean/test_maxmoc.py b/test/unit/ocean/test_maxmoc.py new file mode 100644 index 0000000000000000000000000000000000000000..f6f12b3468a050c5ad118ff86664b664ceae29c9 --- /dev/null +++ b/test/unit/ocean/test_maxmoc.py @@ -0,0 +1,66 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.box import Box +from earthdiagnostics.constants import Basins, Basin +from earthdiagnostics.ocean.maxmoc import MaxMoc +from mock import Mock, patch + + +class TestMaxMoc(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.basins = Mock() + self.basins.Global = Basin('Global') + self.basins.Atlantic = Basin('Atlantic') + + self.box = Box(True) + self.box.min_lat = 0.0 + self.box.max_lat = 0.0 + self.box.min_depth = 0.0 + self.box.max_depth = 0.0 + + self.maxmoc = MaxMoc(self.data_manager, '20000101', 1, 2000, self.basins.Global, self.box) + + def fake_parse(self, value): + if type(value) is Basin: + return value + if value == 'atl': + value = 'Atlantic' + else: + value = 'Global' + + return Basin(value) + + @patch.object(Basins, 'parse', fake_parse) + def test_generate_jobs(self): + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.startdates = ('20010101',) + self.diags.config.experiment.members = (0,) + self.diags.config.experiment.get_full_years.return_value = (2000, 2001) + + jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MaxMoc(self.data_manager, '20010101', 0, 2000, self.basins.Global, self.box)) + self.assertEqual(jobs[1], MaxMoc(self.data_manager, '20010101', 0, 2001, self.basins.Global, self.box)) + + jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', 'atl']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MaxMoc(self.data_manager, '20010101', 0, 2000, self.basins.Atlantic, self.box)) + self.assertEqual(jobs[1], MaxMoc(self.data_manager, '20010101', 0, 2001, self.basins.Atlantic, self.box)) + + self.diags.config.experiment.get_full_years.return_value = list() + jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) + self.assertEqual(len(jobs), 0) + + with self.assertRaises(Exception): + MaxMoc.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.maxmoc), 'Max moc Startdate: 20000101 Member: 1 Year: 2000 ' + 'Box: 0.0N0.0m Basin: Global') diff --git a/test/unit/ocean/test_mixedlayerheatcontent.py b/test/unit/ocean/test_mixedlayerheatcontent.py new file mode 100644 index 0000000000000000000000000000000000000000..b7cf564bc9cdfabe924327ea0602ea3bc6d84478 --- /dev/null +++ b/test/unit/ocean/test_mixedlayerheatcontent.py @@ -0,0 +1,29 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.mixedlayerheatcontent import MixedLayerHeatContent +from mock import Mock + + +class TestMixedLayerHeatContent(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.mixed = MixedLayerHeatContent(self.data_manager, '20000101', 1, 1) + + def test_generate_jobs(self): + jobs = MixedLayerHeatContent.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MixedLayerHeatContent(self.data_manager, '20010101', 0, 0)) + self.assertEqual(jobs[1], MixedLayerHeatContent(self.data_manager, '20010101', 0, 1)) + + with self.assertRaises(Exception): + MixedLayerHeatContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'Mixed layer heat content Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_mixedlayersaltcontent.py b/test/unit/ocean/test_mixedlayersaltcontent.py new file mode 100644 index 0000000000000000000000000000000000000000..7aa42f6691eff2d80c4290b1fa5d1505794543d2 --- /dev/null +++ b/test/unit/ocean/test_mixedlayersaltcontent.py @@ -0,0 +1,29 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.mixedlayersaltcontent import MixedLayerSaltContent +from mock import Mock + + +class TestMixedLayerSaltContent(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.mixed = MixedLayerSaltContent(self.data_manager, '20000101', 1, 1) + + def test_generate_jobs(self): + jobs = MixedLayerSaltContent.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MixedLayerSaltContent(self.data_manager, '20010101', 0, 0)) + self.assertEqual(jobs[1], MixedLayerSaltContent(self.data_manager, '20010101', 0, 1)) + + with self.assertRaises(Exception): + MixedLayerSaltContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'Mixed layer salt content Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_moc.py b/test/unit/ocean/test_moc.py new file mode 100644 index 0000000000000000000000000000000000000000..0c05f8c4fe7dd3632cc5b662aacda78c95dbfe95 --- /dev/null +++ b/test/unit/ocean/test_moc.py @@ -0,0 +1,29 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.moc import Moc +from mock import Mock + + +class TestMoc(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.mixed = Moc(self.data_manager, '20000101', 1, 1) + + def test_generate_jobs(self): + jobs = Moc.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Moc(self.data_manager, '20010101', 0, 0)) + self.assertEqual(jobs[1], Moc(self.data_manager, '20010101', 0, 1)) + + with self.assertRaises(Exception): + Moc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'MOC Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_mxl.py b/test/unit/ocean/test_mxl.py new file mode 100644 index 0000000000000000000000000000000000000000..ead3fbbccfb2bfc40b6075089b8dc166326dcba8 --- /dev/null +++ b/test/unit/ocean/test_mxl.py @@ -0,0 +1,28 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.mxl import Mxl +from mock import Mock + + +class TestMxl(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def test_generate_jobs(self): + jobs = Mxl.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Mxl(self.data_manager, '20010101', 0, 0)) + self.assertEqual(jobs[1], Mxl(self.data_manager, '20010101', 0, 1)) + + with self.assertRaises(Exception): + Mxl.generate_jobs(self.diags, ['diagnostic', 'extra']) + + def test_str(self): + diag = Mxl(self.data_manager, '20010101', 0, 0) + self.assertEquals(str(diag), 'Mixed layer Startdate: 20010101 Member: 0 Chunk: 0') diff --git a/test/unit/ocean/test_psi.py b/test/unit/ocean/test_psi.py new file mode 100644 index 0000000000000000000000000000000000000000..019e9339ca281274d59ba15b07bf190ee9b457da --- /dev/null +++ b/test/unit/ocean/test_psi.py @@ -0,0 +1,25 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.ocean.psi import Psi +from mock import Mock + + +class TestPsi(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.psi = Psi(self.data_manager, '20000101', 1, 1) + + def test_generate_jobs(self): + jobs = Psi.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Psi(self.data_manager, '20010101', 0, 0)) + self.assertEqual(jobs[1], Psi(self.data_manager, '20010101', 0, 1)) + + with self.assertRaises(Exception): + Psi.generate_jobs(self.diags, ['diagnostic', 'badoption']) + + def test_str(self): + self.assertEquals(str(self.psi), 'PSI Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py new file mode 100644 index 0000000000000000000000000000000000000000..e527feda328b3103615680a4387d155ef29d3f63 --- /dev/null +++ b/test/unit/ocean/test_region_mean.py @@ -0,0 +1,100 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.ocean.regionmean import RegionMean +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.constants import Basins +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption +from mock import Mock, patch + + +class TestRegionMean(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + box = Box() + box.min_depth = 0 + box.max_depth = 0 + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'T', + box, True, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'T', + box, True, Basins().Global, False, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + + box = Box() + box.min_depth = 1 + box.max_depth = 10 + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, False, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', + 'True']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, True, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, True, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', + 'True', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, True, 'grid')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, True, 'grid')) + + with self.assertRaises(DiagnosticOptionError): + RegionMean.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', + 'True', 'grid', 'extra']) + + def test_str(self): + box = Box() + box.min_depth = 1 + box.max_depth = 10 + + diag = RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', box, False, + Basins().Global, True, 'grid') + self.assertEquals(str(diag), 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Grid point: U ' + 'Box: 1-10 Save 3D: False Save variance: True Original grid: grid') diff --git a/test/unit/ocean/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py new file mode 100644 index 0000000000000000000000000000000000000000..00f7c68ca53e22c4ca2f10d90b1d4d89e4176236 --- /dev/null +++ b/test/unit/ocean/test_siasiesiv.py @@ -0,0 +1,21 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.constants import Basins +from earthdiagnostics.ocean.siasiesiv import Siasiesiv +from mock import Mock + + +class TestSiasiesiv(TestCase): + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.mask = Mock() + self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask) + + def test_str(self): + self.assertEquals(str(self.psi), 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global') diff --git a/test/unit/ocean/test_vertical_gradient.py b/test/unit/ocean/test_vertical_gradient.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d9d22d32358059c73899c1c7aee834400dddab --- /dev/null +++ b/test/unit/ocean/test_vertical_gradient.py @@ -0,0 +1,65 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.ocean.verticalgradient import VerticalGradient +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.constants import Basins +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption +from mock import Mock, patch + + +class TestVerticalGradient(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + box = Box() + box.min_depth = 1 + box.max_depth = 2 + + jobs = VerticalGradient.generate_jobs(self.diags, ['diagnostic', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box)) + self.assertEqual(jobs[1], VerticalGradient(self.data_manager, '20010101', 0, 1, 'var', box)) + + box = Box() + box.min_depth = 2 + box.max_depth = 2 + + jobs = VerticalGradient.generate_jobs(self.diags, ['diagnostic', 'var', '2']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box)) + self.assertEqual(jobs[1], VerticalGradient(self.data_manager, '20010101', 0, 1, 'var', box)) + + box = Box() + box.min_depth = 1 + box.max_depth = 10 + + jobs = VerticalGradient.generate_jobs(self.diags, ['diagnostic', 'var', '1', 10]) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box)) + self.assertEqual(jobs[1], VerticalGradient(self.data_manager, '20010101', 0, 1, 'var', box)) + + with self.assertRaises(DiagnosticOptionError): + VerticalGradient.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + VerticalGradient.generate_jobs(self.diags, ['diagnostic', 'var', '1', '10', 'extra']) + + def test_str(self): + box = Box() + box.min_depth = 1 + box.max_depth = 10 + + diag = VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box) + self.assertEquals(str(diag), 'Vertical gradient Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Box: 1-10') diff --git a/test/unit/ocean/test_verticalmean.py b/test/unit/ocean/test_verticalmean.py new file mode 100644 index 0000000000000000000000000000000000000000..fc501afda0a4b1c3842c8fd60b16d4fb88b03b9f --- /dev/null +++ b/test/unit/ocean/test_verticalmean.py @@ -0,0 +1,57 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.ocean.verticalmean import VerticalMean +from mock import Mock, patch + + +class TestVerticalMean(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + self.mixed = VerticalMean(self.data_manager, '20000101', 1, 1, 'var', self.box) + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = VerticalMean.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMean(self.data_manager, '20010101', 0, 0, 'var', self.box)) + self.assertEqual(jobs[1], VerticalMean(self.data_manager, '20010101', 0, 1, 'var', self.box)) + + jobs = VerticalMean.generate_jobs(self.diags, ['diagnostic', 'var', '0']) + box = Box() + box.min_depth = 0 + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMean(self.data_manager, '20010101', 0, 0, 'var', box)) + self.assertEqual(jobs[1], VerticalMean(self.data_manager, '20010101', 0, 1, 'var', box)) + + jobs = VerticalMean.generate_jobs(self.diags, ['diagnostic', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMean(self.data_manager, '20010101', 0, 0, 'var', Box())) + self.assertEqual(jobs[1], VerticalMean(self.data_manager, '20010101', 0, 1, 'var', Box())) + + with self.assertRaises(DiagnosticOptionError): + VerticalMean.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + VerticalMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var ' + 'Box: 0-100') diff --git a/test/unit/ocean/test_verticalmeanmeters.py b/test/unit/ocean/test_verticalmeanmeters.py new file mode 100644 index 0000000000000000000000000000000000000000..f696d20d4adc81e59cb62d700cd2fcba7a0f79d0 --- /dev/null +++ b/test/unit/ocean/test_verticalmeanmeters.py @@ -0,0 +1,64 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.ocean.verticalmeanmeters import VerticalMeanMeters +from earthdiagnostics.modelingrealm import ModelingRealms +from mock import Mock, patch + + +class TestVerticalMeanMeters(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box(True) + self.box.min_depth = 0 + self.box.max_depth = 100 + + self.mixed = VerticalMeanMeters(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', self.box, 'T') + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + self.box, 'T')) + self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + self.box, 'T')) + + jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var', '0']) + box = Box(True) + box.min_depth = 0 + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + box, 'T')) + self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + box, 'T')) + + jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + Box(True), 'T')) + self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + Box(True), 'T')) + + with self.assertRaises(DiagnosticOptionError): + VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Box: 0-100m') diff --git a/test/unit/statistics/__init__.py b/test/unit/statistics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb --- /dev/null +++ b/test/unit/statistics/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/test/unit/statistics/test_climatologicalpercentile.py b/test/unit/statistics/test_climatologicalpercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..12752caab2bd8fcc7322b7e80130a939f4652029 --- /dev/null +++ b/test/unit/statistics/test_climatologicalpercentile.py @@ -0,0 +1,42 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestClimatologicalPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.data_manager.variable_list.get_variable.return_value = None + + self.diags = Mock() + self.diags.data_manager = self.data_manager + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent', 'ocean', 'var', '2000', '2001', '11']) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0], ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', + 2000, 2001, 11, + self.diags.config.experiment)) + + with self.assertRaises(Exception): + ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent']) + with self.assertRaises(Exception): + ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent', 'ocean', 'var', '2000', '2001', '11', + 'extra']) + + def test_str(self): + diagnostic = ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', + 2000, 2001, 11, self.diags.config.experiment) + + self.assertEquals(str(diagnostic), 'Climatological percentile Variable: ocean:var Period: 2000-2001 ' + 'Forecast month: 11') diff --git a/test/unit/statistics/test_daysoverpercentile.py b/test/unit/statistics/test_daysoverpercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..cc225cd64a589cb1f453e2582777841cc6b2c2ff --- /dev/null +++ b/test/unit/statistics/test_daysoverpercentile.py @@ -0,0 +1,33 @@ +# coding=utf-8 +from unittest import TestCase + +from mock import Mock + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.statistics.daysoverpercentile import DaysOverPercentile + + +class TestDaysOverPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20011101', 0, 0), ('20011101', 0, 1)) + + def test_generate_jobs(self): + jobs = DaysOverPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '2000', '2001', '11']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, + 2000, 11)) + self.assertEqual(jobs[1], DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, + 2001, 11)) + + with self.assertRaises(Exception): + DaysOverPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '2000', '2001']) + with self.assertRaises(Exception): + DaysOverPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '2000', '2001', '11', 'extra']) + + def test_str(self): + diagnostic = DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, '20001101', 11) + self.assertEquals(str(diagnostic), 'Days over percentile Startdate: 20001101 Variable: ocean:var ' + 'Climatology: 2000-2001') diff --git a/test/unit/statistics/test_discretize.py b/test/unit/statistics/test_discretize.py new file mode 100644 index 0000000000000000000000000000000000000000..402a3772aff95aa7a6cf001cb049d2034be8847a --- /dev/null +++ b/test/unit/statistics/test_discretize.py @@ -0,0 +1,60 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.statistics.discretize import Discretize +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestClimatologicalPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.data_manager.variable_list.get_variable.return_value = None + + self.diags = Mock() + self.diags.data_manager = self.data_manager + self.diags.config.experiment.startdates = ('20000101', '20010101') + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = Discretize.generate_jobs(self.diags, ['discretize', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', + 2000, float('nan'), float('nan'))) + self.assertEqual(jobs[1], Discretize(self.data_manager, '20010101', ModelingRealms.ocean, 'var', + 2000, float('nan'), float('nan'))) + + jobs = Discretize.generate_jobs(self.diags, ['discretize', 'ocean', 'var', '500']) + self.assertEqual(jobs[0], Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', + 500, float('nan'), float('nan'))) + self.assertEqual(jobs[1], Discretize(self.data_manager, '20010101', ModelingRealms.ocean, 'var', + 500, float('nan'), float('nan'))) + + jobs = Discretize.generate_jobs(self.diags, ['discretize', 'ocean', 'var', '500', '0', '40']) + self.assertEqual(jobs[0], Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', + 500, 0, 40)) + self.assertEqual(jobs[1], Discretize(self.data_manager, '20010101', ModelingRealms.ocean, 'var', + 500, 0, 40)) + + with self.assertRaises(Exception): + Discretize.generate_jobs(self.diags, ['discretize']) + with self.assertRaises(Exception): + Discretize.generate_jobs(self.diags, ['discretize', 'ocean', 'var', '500', '0', '40', 'extra']) + + def test_str(self): + diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, 10, 40) + + self.assertEquals(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' + 'Range: [10, 40]') + + diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, + float('nan'), float('nan')) + + self.assertEquals(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' + 'Range: [None, None]') diff --git a/test/unit/statistics/test_monthlypercentile.py b/test/unit/statistics/test_monthlypercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..a902ec89098794bdca2baabce94be5d1aa3d4107 --- /dev/null +++ b/test/unit/statistics/test_monthlypercentile.py @@ -0,0 +1,41 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.box import Box +from earthdiagnostics.statistics.monthlypercentile import MonthlyPercentile +from mock import Mock + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestMonthlyPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + + self.box = Box() + self.box.min_lat = 0 + self.box.max_lat = 0 + self.box.min_lon = 0 + self.box.max_lon = 0 + + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diagnostic = MonthlyPercentile(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', [10, 90]) + + def test_generate_jobs(self): + jobs = MonthlyPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '10-90']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MonthlyPercentile(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + [10, 90])) + self.assertEqual(jobs[1], MonthlyPercentile(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + [10, 90])) + + with self.assertRaises(Exception): + MonthlyPercentile.generate_jobs(self.diags, ['monpercent']) + with self.assertRaises(Exception): + MonthlyPercentile.generate_jobs(self.diags, ['monpercent', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.diagnostic), 'Monthly percentile Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Percentiles: 10, 90') diff --git a/test/unit/test_box.py b/test/unit/test_box.py new file mode 100644 index 0000000000000000000000000000000000000000..37b5ce4911333419281366b9d850e219c8aacb8d --- /dev/null +++ b/test/unit/test_box.py @@ -0,0 +1,93 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.box import Box + + +class TestBox(TestCase): + + def setUp(self): + self.box1 = Box() + self.box1.max_lat = 0 + self.box1.min_lat = -20 + self.box1.max_lon = 0 + self.box1.min_lon = -20 + self.box1.min_depth = 0 + self.box1.max_depth = 20 + + self.box2 = Box(True) + self.box2.max_lat = 20 + self.box2.min_lat = 20 + self.box2.max_lon = 20 + self.box2.min_lon = 20 + self.box2.min_depth = 20 + self.box2.max_depth = 20 + + self.box3 = Box() + + self.box4 = Box() + self.box4.max_lat = -10 + self.box4.min_lat = -20 + self.box4.max_lon = -10 + self.box4.min_lon = -20 + self.box4.min_depth = 0 + self.box4.max_depth = 20 + + def test_max_lat(self): + with self.assertRaises(ValueError): + Box().max_lat = 100 + with self.assertRaises(ValueError): + Box().max_lat = -100 + Box().max_lat = 0 + Box().max_lat = -20 + Box().max_lat = 20 + + def test_min_lat(self): + with self.assertRaises(ValueError): + Box().min_lat = 100 + with self.assertRaises(ValueError): + Box().min_lat = -100 + Box().min_lat = 0 + Box().min_lat = -90 + Box().min_lat = 90 + + def test_max_lon(self): + with self.assertRaises(ValueError): + Box().max_lon = 360 + with self.assertRaises(ValueError): + Box().max_lon = -360 + Box().max_lon = 0 + Box().max_lon = -20 + Box().max_lon = 20 + + def test_min_lon(self): + with self.assertRaises(ValueError): + Box().min_lon = 360 + with self.assertRaises(ValueError): + Box().min_lon = -360 + Box().min_lon = 0 + Box().min_lon = -80 + Box().min_lon = 80 + + def test_get_lat_str(self): + self.assertEquals('20S0N', self.box1.get_lat_str()) + self.assertEquals('20N', self.box2.get_lat_str()) + self.assertEquals('', self.box3.get_lat_str()) + self.assertEquals('20S10S', self.box4.get_lat_str()) + + def test_get_lon_str(self): + self.assertEquals('20W0E', self.box1.get_lon_str()) + self.assertEquals('20E', self.box2.get_lon_str()) + self.assertEquals('', self.box3.get_lon_str()) + self.assertEquals('20W10W', self.box4.get_lon_str()) + + def test_get_depth_str(self): + self.assertEquals('0-20', self.box1.get_depth_str()) + self.assertEquals('20m', self.box2.get_depth_str()) + self.assertEquals('', self.box3.get_depth_str()) + self.assertEquals('0-20', self.box4.get_depth_str()) + + def test__str__(self): + self.assertEquals('20S0N20W0E0-20', str(self.box1)) + self.assertEquals('20N20E20m', str(self.box2)) + self.assertEquals('', str(self.box3)) + self.assertEquals('20S10S20W10W0-20', str(self.box4)) diff --git a/test/unit/test_cdftools.py b/test/unit/test_cdftools.py new file mode 100644 index 0000000000000000000000000000000000000000..28de4ac34e9b60726ddbcac9e1d7d3e7f38a8d1f --- /dev/null +++ b/test/unit/test_cdftools.py @@ -0,0 +1,65 @@ +# coding=utf-8 +from unittest import TestCase + +import os + +from earthdiagnostics.cdftools import CDFTools +import mock + + +# noinspection PyUnusedLocal +def bad_file(path, access=None): + return not os.path.basename(path).startswith('bad') + + +class TestCDFTools(TestCase): + + # noinspection PyUnusedLocal + @mock.patch('os.path.isfile', side_effect=bad_file) + @mock.patch('os.access', side_effect=bad_file) + @mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') + def test_run(self, mock_path, mock_exists, execute_mock): + self.cdftools = CDFTools('/test/path') + execute_mock.return_value = ['Command output'] + with self.assertRaises(ValueError): + self.cdftools.run('badcommand', input='input_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='badinput_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='input_file', output='input_file') + with self.assertRaises(Exception): + self.cdftools.run('command', input='input_file', output='badoutput_file') + + self.cdftools.run('command', input='input_file', output='output_file') + self.cdftools.run('command', input='input_file') + self.cdftools.run('command', input=None) + self.cdftools.run('command', input=['input_file', 'input_file2']) + self.cdftools.run('command', input='input_file', options='-o -p') + self.cdftools.run('command', input='input_file', options=('-o', '-p')) + + # noinspection PyUnusedLocal + @mock.patch('os.path.isfile', side_effect=bad_file) + @mock.patch('os.access', side_effect=bad_file) + @mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') + def test_run(self, mock_path, mock_exists, execute_mock): + self.cdftools = CDFTools('') + execute_mock.return_value = ['Command output'] + with self.assertRaises(ValueError): + self.cdftools.run('badcommand', input='input_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='badinput_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='input_file', output='input_file') + with self.assertRaises(Exception): + self.cdftools.run('command', input='input_file', output='badoutput_file') + + self.cdftools.run('command', input='input_file', output='output_file') + self.cdftools.run('command', input='input_file') + self.cdftools.run('command', input=None) + self.cdftools.run('command', input=['input_file', 'input_file2']) + self.cdftools.run('command', input='input_file', options='-o -p') + self.cdftools.run('command', input='input_file', options=('-o', '-p')) diff --git a/test/unit/test_config.py b/test/unit/test_config.py new file mode 100644 index 0000000000000000000000000000000000000000..419be475362582c7df34169f4a640748ad298e9e --- /dev/null +++ b/test/unit/test_config.py @@ -0,0 +1,319 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.modelingrealm import ModelingRealms + + +class VariableMock(object): + def __init__(self): + self.domain = ModelingRealms.ocean + self.short_name = 'tos' + + def __eq__(self, other): + return self.domain == other.domain and self.short_name == other.short_name + + +class VariableManagerMock(object): + def get_variable(self, alias, silent=False): + if alias == 'bad': + return None + var = VariableMock() + var.short_name = alias + return var + + +class ParserMock(object): + + def __init__(self): + self._values = {} + + def add_value(self, section, var, value): + self._values[self.get_var_string(section, var)] = value + + def get_var_string(self, section, var): + return '{0}:{1}'.format(section, var) + + def get_value(self, section, var, default): + try: + return self._values[self.get_var_string(section, var)] + except KeyError: + return default + + def get_bool_option(self, section, var, default): + return self.get_value(section, var, default) + + def get_path_option(self, section, var, default): + return self.get_value(section, var, default) + + def get_int_option(self, section, var, default=0): + return self.get_value(section, var, default) + + + def get_int_list_option(self, section, var, default=list(), separator=' '): + try: + return [int(val) for val in self._values[self.get_var_string(section, var)].split(separator)] + except KeyError: + return default + + def get_list_option(self, section, var, default=list(), separator=' '): + try: + return [val for val in self._values[self.get_var_string(section, var)].split(separator)] + except KeyError: + return default + + def get_option(self, section, var, default=None): + return self.get_value(section, var, default) + + +class TestCMORConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + self.var_manager = VariableManagerMock() + + def test_basic_config(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.ocean, True) + self.assertEquals(config.atmosphere, True) + self.assertEquals(config.force, False) + self.assertEquals(config.force_untar, False) + self.assertEquals(config.use_grib, True) + self.assertEquals(config.activity, 'CMIP') + self.assertEquals(config.associated_experiment, 'to be filled') + self.assertEquals(config.associated_model, 'to be filled') + self.assertEquals(config.initialization_description, 'to be filled') + self.assertEquals(config.initialization_method, '1') + self.assertEquals(config.initialization_number, 1) + self.assertEquals(config.source, 'to be filled') + self.assertEquals(config.version, '') + self.assertEquals(config.physics_version, '1') + self.assertEquals(config.physics_description, 'to be filled') + self.assertEquals(config.filter_files, '') + self.assertEquals(config.default_atmos_grid, 'gr') + self.assertEquals(config.default_ocean_grid, 'gn') + self.assertEquals(config.min_cmorized_vars, 10) + + def test_cmorize(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.cmorize(VariableMock())) + self.assertTrue(config.cmorize(None)) + + def test_cmorize_list(self): + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:thetao ocean:tos') + + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.cmorize(VariableMock())) + + thetao_mock = VariableMock() + thetao_mock.domain = ModelingRealms.ocean + thetao_mock.short_name = 'thetao' + self.assertTrue(config.cmorize(thetao_mock)) + + def test_bad_list(self): + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', '#ocean:tos') + with self.assertRaises(ConfigException): + CMORConfig(self.mock_parser, self.var_manager) + + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'atmos:tos') + with self.assertRaises(ConfigException): + CMORConfig(self.mock_parser, self.var_manager) + + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:bad') + with self.assertRaises(ConfigException): + CMORConfig(self.mock_parser, self.var_manager) + + def test_not_cmorize(self): + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos') + + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.cmorize(VariableMock())) + + self.assertFalse(config.cmorize(None)) + + tas_mock = VariableMock() + tas_mock.domain = ModelingRealms.atmos + tas_mock.short_name = 'tas' + self.assertFalse(config.cmorize(tas_mock)) + + thetao_mock = VariableMock() + thetao_mock.domain = ModelingRealms.ocean + thetao_mock.short_name = 'thetao' + self.assertFalse(config.cmorize(thetao_mock)) + + def test_comment(self): + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos #ocean:thetao ') + + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.cmorize(VariableMock())) + + thetao_mock = VariableMock() + thetao_mock.domain = ModelingRealms.ocean + thetao_mock.short_name = 'thetao' + self.assertFalse(config.cmorize(thetao_mock)) + + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', '#ocean:tos ocean:thetao ') + with self.assertRaises(ConfigException): + CMORConfig(self.mock_parser, self.var_manager) + + def test_cmorization_chunk(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.chunk_cmorization_requested(1)) + + def test_cmorize_only_some_chunks(self): + self.mock_parser.add_value('CMOR', 'CHUNKS', '3 5') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.chunk_cmorization_requested(3)) + self.assertTrue(config.chunk_cmorization_requested(5)) + self.assertFalse(config.chunk_cmorization_requested(1)) + self.assertFalse(config.chunk_cmorization_requested(4)) + self.assertFalse(config.chunk_cmorization_requested(6)) + + def test_any_required(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.any_required(['tos'])) + + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos ocean:thetao') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.any_required(['tos', 'thetao', 'tas'])) + self.assertTrue(config.any_required(['tos', 'tas'])) + self.assertTrue(config.any_required(['thetao'])) + + self.assertFalse(config.any_required(['tas'])) + + def test_hourly_vars(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.six_hourly), {}) + + self.mock_parser.add_value('CMOR', 'ATMOS_HOURLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.six_hourly), {128: None, + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) + + self.assertEquals(config.get_levels(Frequencies.six_hourly, 128), None) + self.assertEquals(config.get_levels(Frequencies.six_hourly, 129), '1') + self.assertEquals(config.get_levels(Frequencies.six_hourly, 130), '1,2') + self.assertEquals(config.get_levels(Frequencies.six_hourly, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEquals(config.get_levels(Frequencies.six_hourly, 132), '0,5') + + def test_daily_vars(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.daily), {}) + + self.mock_parser.add_value('CMOR', 'ATMOS_DAILY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.daily), {128: None, + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) + + self.assertEquals(config.get_levels(Frequencies.daily, 128), None) + self.assertEquals(config.get_levels(Frequencies.daily, 129), '1') + self.assertEquals(config.get_levels(Frequencies.daily, 130), '1,2') + self.assertEquals(config.get_levels(Frequencies.daily, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEquals(config.get_levels(Frequencies.daily, 132), '0,5') + + def test_monthly_vars(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.monthly), {}) + + self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.monthly), {128: None, + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) + + self.assertEquals(config.get_levels(Frequencies.monthly, 128), None) + self.assertEquals(config.get_levels(Frequencies.monthly, 129), '1') + self.assertEquals(config.get_levels(Frequencies.monthly, 130), '1,2') + self.assertEquals(config.get_levels(Frequencies.monthly, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEquals(config.get_levels(Frequencies.monthly, 132), '0,5') + + def test_bad_frequency_vars(self): + config = CMORConfig(self.mock_parser, self.var_manager) + with self.assertRaises(ValueError): + self.assertEquals(config.get_variables(Frequencies.climatology), {}) + + +class TestTHREDDSConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + + def test_basic_config(self): + config = THREDDSConfig(self.mock_parser) + self.assertEquals(config.server_url, '') + + def test_url(self): + self.mock_parser.add_value('THREDDS', 'SERVER_URL', 'test_url') + config = THREDDSConfig(self.mock_parser) + self.assertEquals(config.server_url, 'test_url') + + +class TestReportConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + + def test_basic_config(self): + config = ReportConfig(self.mock_parser) + self.assertEquals(config.path, '') + self.assertEquals(config.maximum_priority, 10) + + def test_path(self): + self.mock_parser.add_value('REPORT', 'PATH', 'new_path') + config = ReportConfig(self.mock_parser) + self.assertEquals(config.path, 'new_path') + + def test_priority(self): + config = ReportConfig(self.mock_parser) + self.assertEquals(config.maximum_priority, 3) + + +class TestExperimentConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + + def test_basic_config(self): + config = ExperimentConfig(self.mock_parser) + + self.assertEquals(config.startdates, []) + self.assertEquals(config.members, []) + self.assertEquals(config.chunk_size, 0) + self.assertEquals(config.num_chunks, 0) + + self.assertEquals(config.atmos_grid, '') + self.assertEquals(config.atmos_timestep, 6) + self.assertEquals(config.ocean_timestep, 6) + + def test_cmor_version_required(self): + self.mock_parser.add_value('CMOR', 'VERSION', '20001101') + self.mock_parser.add_value('EXPERIMENT', 'DATA_CONVENTION', 'Primavera') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.path, 'new_path') + + def test_startdates(self): + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '20001101 20011101') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200(0|1)1101') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200[0-2](02|05|08|11)01') + config = ExperimentConfig(self.mock_parser) + print(config.startdates) + self.assertEquals(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', + u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', + u'20020801', u'20021101']) + + diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py new file mode 100644 index 0000000000000000000000000000000000000000..c538ee09d04f1308204d62eecf9361be2e475c33 --- /dev/null +++ b/test/unit/test_constants.py @@ -0,0 +1,20 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.constants import Basin + + +class TestBasin(TestCase): + + def setUp(self): + self.basin = Basin('Basin') + + def test_name(self): + self.assertEquals('Basin', self.basin.name) + + def test__eq__(self): + self.assertTrue(Basin('Basin') == self.basin) + self.assertFalse(Basin('bas') == self.basin) + + def test__str__(self): + self.assertEquals(str(self.basin), 'Basin') diff --git a/test/unit/test_data_manager.py b/test/unit/test_data_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..6bdc84b39140e42b8613a147af1acee8e886297a --- /dev/null +++ b/test/unit/test_data_manager.py @@ -0,0 +1,38 @@ +# coding=utf-8 + + +from unittest import TestCase + +from earthdiagnostics.datamanager import UnitConversion + + +class TestConversion(TestCase): + + def test__init(self): + conversion = UnitConversion('km', 'm', 1000, 0) + self.assertEqual(conversion.source, 'km') + self.assertEqual(conversion.destiny, 'm') + self.assertEqual(conversion.factor, 1000) + self.assertEqual(conversion.offset, 0) + + def test_add_conversion(self): + UnitConversion._dict_conversions = dict() + conversion = UnitConversion('km', 'm', 1000, 0) + UnitConversion.add_conversion(conversion) + self.assertIs(UnitConversion._dict_conversions[('km', 'm')], conversion) + UnitConversion._dict_conversions = dict() + + def test_get_factor_offset(self): + UnitConversion._dict_conversions = dict() + conversion = UnitConversion('km', 'm', 1000, 0) + UnitConversion.add_conversion(conversion) + self.assertEqual(UnitConversion.get_conversion_factor_offset('km', 'm'), (1000, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('m', 'km'), (1 / 1000.0, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('1e3 m', 'km'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('10^3 m', 'km'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('km', '1e3 m'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('km', '10^3 m'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('m', 'm'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('m²', 'km'), (None, None)) + + UnitConversion._dict_conversions = dict() diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py new file mode 100644 index 0000000000000000000000000000000000000000..ae9921fa920affc0833b87dffbdf5d5899c6b806 --- /dev/null +++ b/test/unit/test_diagnostic.py @@ -0,0 +1,368 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import * +from unittest import TestCase + +from earthdiagnostics.modelingrealm import ModelingRealms +from mock import patch, Mock + + +class TestDiagnosticOption(TestCase): + + def test_good_default_value(self): + diag = DiagnosticOption('option', 'default') + self.assertEqual('default', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticOption('option') + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('default', diag.parse('')) + + def test_parse_value(self): + diag = DiagnosticOption('option') + self.assertEqual('value', diag.parse('value')) + + +class TestDiagnosticFloatOption(TestCase): + def test_float_default_value(self): + diag = DiagnosticFloatOption('option', 3.0) + self.assertEqual(3.0, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticFloatOption('option', '3') + self.assertEqual(3.0, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticFloatOption('option', 'default') + with self.assertRaises(ValueError): + self.assertEqual('default', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticFloatOption('option') + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('default', diag.parse('')) + + def test_parse_value(self): + diag = DiagnosticFloatOption('option') + self.assertEqual(3.25, diag.parse('3.25')) + + +class TestDiagnosticDomainOption(TestCase): + def test_domain_default_value(self): + diag = DiagnosticDomainOption('option', ModelingRealms.ocean) + self.assertEqual(ModelingRealms.ocean, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticDomainOption('option', 'atmos') + self.assertEqual(ModelingRealms.atmos, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticDomainOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticDomainOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticDomainOption('option') + self.assertEqual(ModelingRealms.seaIce, diag.parse('seaice')) + + +class TestDiagnosticIntOption(TestCase): + def test_int_default_value(self): + diag = DiagnosticIntOption('option', 3) + self.assertEqual(3, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticIntOption('option', '3') + self.assertEqual(3, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticIntOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticIntOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticIntOption('option') + self.assertEqual(3, diag.parse('3')) + + def test_parse_bad_value(self): + diag = DiagnosticIntOption('option') + with self.assertRaises(ValueError): + diag.parse('3.5') + + def test_good_low_limit(self): + diag = DiagnosticIntOption('option', None, 0) + self.assertEqual(1, diag.parse('1')) + + def test_bad_low_limit(self): + diag = DiagnosticIntOption('option', None, 0) + with self.assertRaises(DiagnosticOptionError): + diag.parse('-1') + + def test_good_high_limit(self): + diag = DiagnosticIntOption('option', None, None, 0) + self.assertEqual(-1, diag.parse('-1')) + + def test_bad_high_limit(self): + diag = DiagnosticIntOption('option', None, None, 0) + with self.assertRaises(DiagnosticOptionError): + diag.parse('1') + + +class TestDiagnosticBoolOption(TestCase): + def test_bool_default_value(self): + diag = DiagnosticBoolOption('option', True) + self.assertEqual(True, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticBoolOption('option', 'False') + self.assertEqual(False, diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticBoolOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_True(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('true')) + + def test_parse_true(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('true')) + + def test_parse_t(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('t')) + + def test_parse_yes(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('YES')) + + def test_parse_bad_value(self): + diag = DiagnosticBoolOption('option') + self.assertFalse(diag.parse('3.5')) + + +class TestDiagnosticComplexStrOption(TestCase): + def test_complex_default_value(self): + diag = DiagnosticComplexStrOption('option', 'default&.str&;&.working') + self.assertEqual('default str, working', diag.parse('')) + + def test_simple_default_value(self): + diag = DiagnosticComplexStrOption('default str, working', 'default str, working') + self.assertEqual('default str, working', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticComplexStrOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticComplexStrOption('option') + self.assertEqual('complex string, for testing', diag.parse('complex&.string&;&.for&.testing')) + + +class TestDiagnosticListIntOption(TestCase): + def test_tuple_default_value(self): + diag = DiagnosticListIntOption('option', (3,)) + self.assertEqual((3,), diag.parse('')) + + def test_list_default_value(self): + diag = DiagnosticListIntOption('option', [3]) + self.assertEqual([3], diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticListIntOption('option', '3-4') + self.assertEqual([3, 4], diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticListIntOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticListIntOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticListIntOption('option') + self.assertEqual([3, 2], diag.parse('3-2')) + + def test_parse_single_value(self): + diag = DiagnosticListIntOption('option') + self.assertEqual([3], diag.parse('3')) + + def test_too_low(self): + diag = DiagnosticListIntOption('option', min_limit=5) + with self.assertRaises(DiagnosticOptionError): + diag.parse('3') + + def test_too_high(self): + diag = DiagnosticListIntOption('option', max_limit=5) + with self.assertRaises(DiagnosticOptionError): + diag.parse('8') + + def test_parse_bad_value(self): + diag = DiagnosticListIntOption('option') + with self.assertRaises(ValueError): + diag.parse('3.5') + + +class TestDiagnosticChoiceOption(TestCase): + + def test_choice_value(self): + diag = DiagnosticChoiceOption('option', ('a', 'b')) + self.assertEqual('a', diag.parse('a')) + + def test_choice_default_value(self): + diag = DiagnosticChoiceOption('option', ('a', 'b'), default_value='a') + self.assertEqual('a', diag.parse('')) + + def test_bad_default_value(self): + with self.assertRaises(DiagnosticOptionError): + DiagnosticChoiceOption('option', ('a', 'b'), default_value='c') + + def test_ignore_case_value(self): + diag = DiagnosticChoiceOption('option', ('a', 'b')) + self.assertEqual('b', diag.parse('b')) + self.assertEqual('b', diag.parse('B')) + + diag = DiagnosticChoiceOption('option', ('a', 'b'), ignore_case=False) + self.assertEqual('b', diag.parse('b')) + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('b', diag.parse('B')) + + +class TestDiagnosticVariableOption(TestCase): + + def get_var_mock(self, name): + mock = Mock() + mock.short_name = name + return mock + + def test_parse(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.return_value = self.get_var_mock('var1') + + diag = DiagnosticVariableOption(var_manager_mock) + self.assertEqual('var1', diag.parse('var1')) + + def test_not_recognized(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.return_value = None + + diag = DiagnosticVariableOption(var_manager_mock) + self.assertEqual('var1', diag.parse('var1')) + + +class TestDiagnosticVariableListOption(TestCase): + + def test_parse_multiple(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.side_effect = (self.get_var_mock('var1'), self.get_var_mock('var2')) + diag = DiagnosticVariableListOption(var_manager_mock, 'variables') + self.assertEqual(['var1', 'var2'], diag.parse('var1-var2')) + + def test_parse_one(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.return_value = self.get_var_mock('var1') + diag = DiagnosticVariableListOption(var_manager_mock, 'variables') + self.assertEqual(['var1'], diag.parse('var1')) + + def test_not_recognized(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.return_value = None + diag = DiagnosticVariableListOption(var_manager_mock, 'variables') + self.assertEqual(['var1'], diag.parse('var1')) + + def get_var_mock(self, name): + mock = Mock() + mock.short_name = name + return mock + + +class TestDiagnostic(TestCase): + + def setUp(self): + class MockDiag(Diagnostic): + @classmethod + def generate_jobs(cls, diags, options): + pass + + def declare_data_generated(self): + pass + + def request_data(self): + pass + + def compute(self): + pass + + self.MockDiag = MockDiag + + def test_str(self): + self.assertEqual(str(Diagnostic(None)), 'Developer must override base class __str__ method') + + def test_compute_is_virtual(self): + with self.assertRaises(NotImplementedError): + Diagnostic(None).compute() + + def test_declare_data_generated_is_virtual(self): + with self.assertRaises(NotImplementedError): + Diagnostic(None).declare_data_generated() + + def test_request_data_is_virtual(self): + with self.assertRaises(NotImplementedError): + Diagnostic(None).request_data() + + @patch.object(Diagnostic, 'dispatch') + def test_set_status_call_dispatch(self, dispatch_mock): + diag = Diagnostic(None) + diag.status = DiagnosticStatus.FAILED + dispatch_mock.assert_called_once_with(diag) + + @patch.object(Diagnostic, 'dispatch') + def test_set_status_call_dispatch(self, dispatch_mock): + diag = Diagnostic(None) + diag.status = diag.status + assert not dispatch_mock.called, 'Dispatch should not have been called' + + def test_register(self): + with self.assertRaises(ValueError): + Diagnostic.register(TestDiagnostic) + + with self.assertRaises(ValueError): + Diagnostic.register(self.MockDiag) + + self.MockDiag.alias = 'mock' + Diagnostic.register(self.MockDiag) + + def test_get_diagnostic(self): + self.assertIsNone(Diagnostic.get_diagnostic('none')) + self.MockDiag.alias = 'mock' + Diagnostic.register(self.MockDiag) + self.assertIs(self.MockDiag, Diagnostic.get_diagnostic('mock')) + + def test_generate_jobs(self): + with self.assertRaises(NotImplementedError): + Diagnostic.generate_jobs(None, ['']) + + def test_compute(self): + with self.assertRaises(NotImplementedError): + Diagnostic(None).compute() + + def test_repr(self): + self.assertEquals(Diagnostic(None).__repr__(), str(Diagnostic(None))) + + def test_empty_process_options(self): + self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py new file mode 100644 index 0000000000000000000000000000000000000000..122793ec6e24f2c02de20bfbb1c7f2d8132d9d4f --- /dev/null +++ b/test/unit/test_earthdiags.py @@ -0,0 +1,8 @@ +# # coding=utf-8 +# from unittest import TestCase +# +# from earthdiagnostics.earthdiags import EarthDiags +# +# +# class TestEarthDiags(TestCase): +# pass diff --git a/test/unit/test_frequency.py b/test/unit/test_frequency.py new file mode 100644 index 0000000000000000000000000000000000000000..34e32a3695d7f2f364100573dfbbbad5fe7cda7e --- /dev/null +++ b/test/unit/test_frequency.py @@ -0,0 +1,35 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.frequency import Frequency +from earthdiagnostics.variable_type import VariableType + + +class TestFrequency(TestCase): + + def test_not_supported(self): + with self.assertRaises(ValueError): + Frequency('badfreq') + + def test_get_monthly_mean(self): + self.assertEqual(Frequency('m').folder_name(VariableType.MEAN), 'monthly_mean') + + def test_get_monthly_stats(self): + self.assertEqual(Frequency('m').folder_name(VariableType.STATISTIC), 'monthly_statistics') + + def test_get_daily_mean(self): + self.assertEqual(Frequency('d').folder_name(VariableType.MEAN), 'daily_mean') + + def test_get_daily_stats(self): + self.assertEqual(Frequency('d').folder_name(VariableType.STATISTIC), 'daily_statistics') + + def test_get_6hourlymean(self): + self.assertEqual(Frequency('6hr').folder_name(VariableType.MEAN), '6hourly') + + def test_get_6hourlystatistics(self): + self.assertEqual(Frequency('6hr').folder_name(VariableType.STATISTIC), '6hourly_statistics') + + def test_get_climatology(self): + self.assertEqual(Frequency('clim').folder_name(VariableType.STATISTIC), 'clim') + self.assertEqual(Frequency('clim').folder_name(VariableType.MEAN), 'clim') + diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py new file mode 100644 index 0000000000000000000000000000000000000000..eec9d0cec9a9055ef6df139f05bf97bd307aedf8 --- /dev/null +++ b/test/unit/test_modelling_realm.py @@ -0,0 +1,52 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms + + +class TestModellingRealms(TestCase): + + def test_parse(self): + self.assertEquals(ModelingRealms.parse('atmos'), ModelingRealms.atmos) + self.assertEquals(ModelingRealms.parse('atmoschem'), ModelingRealms.atmosChem) + self.assertEquals(ModelingRealms.parse('atmoSChem'), ModelingRealms.atmosChem) + with self.assertRaises(ValueError): + ModelingRealms.parse('badrealm') + + +class TestModellingRealm(TestCase): + + def setUp(self): + self.realm = ModelingRealm('ocean') + + def test_constructor_fail_on_bad_realm(self): + with self.assertRaises(ValueError): + ModelingRealm('badrealm') + + def test_comparison(self): + self.assertEqual(ModelingRealm('ocean'), self.realm) + self.assertNotEqual(ModelingRealm('OCEAN'), self.realm) + self.assertNotEqual(ModelingRealm('atmos'), self.realm) + + def test_get_omon(self): + self.assertEqual(self.realm.get_table_name(Frequencies.monthly, 'specs'), 'Omon') + + def test_get_oimon(self): + self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'specs'), 'OImon') + + def test_get_simon(self): + self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'cmip6'), 'SImon') + + def test_get_limon(self): + self.assertEqual(ModelingRealm('landIce').get_table_name(Frequencies.monthly, 'specs'), 'LImon') + + def test_get_day(self): + self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.daily, 'specs'), 'day') + + def test_get_6hrplev(self): + self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, 'specs'), '6hrPlev') + + + + diff --git a/test/unit/test_publisher.py b/test/unit/test_publisher.py new file mode 100644 index 0000000000000000000000000000000000000000..5fe325ff819c7840370abbab9e7cc41d8e09d326 --- /dev/null +++ b/test/unit/test_publisher.py @@ -0,0 +1,37 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.publisher import Publisher +from mock import Mock + + +class TestPublisher(TestCase): + + def test_suscribe(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber, callback=suscriber.callback) + self.assertIn(suscriber, pub.suscribers) + + def test_suscribe_default(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber) + self.assertTrue(hasattr(suscriber, 'update')) + self.assertIn(suscriber, pub.suscribers) + + def test_unsuscribe(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber, callback=suscriber.callback) + pub.unsubscribe(suscriber) + + self.assertNotIn(suscriber, pub.suscribers) + + def test_dispatch(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber, callback=suscriber.callback) + + pub.dispatch(1, 2, 3) + suscriber.callback.assert_called_with(1, 2, 3) + diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..70664ab41837731c0925b21b35b0f64376ad8f9f --- /dev/null +++ b/test/unit/test_utils.py @@ -0,0 +1,60 @@ +# coding=utf-8 +from unittest import TestCase +import mock + +from earthdiagnostics.utils import TempFile, Utils + + +class TestTempFile(TestCase): + def setUp(self): + TempFile.scratch_folder = '/tmp' + TempFile.prefix = 'prefix' + + def test_get(self): + self.assertEquals(TempFile.get('tempfile', clean=False), '/tmp/tempfile') + self.assertEquals(TempFile.get('tempfile2', clean=True), '/tmp/tempfile2') + self.assertNotIn('/tmp/tempfile', TempFile.files) + self.assertIn('/tmp/tempfile2', TempFile.files) + + TempFile.autoclean = True + self.assertEquals(TempFile.get('tempfile3'), '/tmp/tempfile3') + self.assertIn('/tmp/tempfile3', TempFile.files) + + TempFile.autoclean = False + self.assertEquals(TempFile.get('tempfile4'), '/tmp/tempfile4') + self.assertNotIn('/tmp/tempfile4', TempFile.files) + + with mock.patch('tempfile.mkstemp') as mkstemp_mock: + with mock.patch('os.close') as close_mock: + mkstemp_mock.return_value = (34, 'path_to_tempfile') + TempFile.get() + TempFile.get(suffix='suffix') + + mkstemp_mock.assert_has_calls((mock.call(dir='/tmp', prefix='prefix', suffix='.nc'), + mock.call(dir='/tmp', prefix='prefix', suffix='suffix'))) + close_mock.assert_has_calls((mock.call(34), mock.call(34))) + + def test_clean(self): + with mock.patch('os.path.exists') as exists_mock: + with mock.patch('tempfile.mkstemp'): + with mock.patch('os.close'): + with mock.patch('os.remove'): + TempFile.clean() + TempFile.clean() + exists_mock.side_effect = [True, False] + TempFile.autoclean = True + TempFile.get('tempfile') + TempFile.get('tempfile2') + TempFile.clean() + self.assertEquals(len(TempFile.files), 0) + + +class TestUtils(TestCase): + + def test_rename_variable(self): + with mock.patch('earthdiagnostics.utils.Utils.rename_variables') as rename_mock: + Utils.rename_variable('file', 'old', 'new') + Utils.rename_variable('file', 'old', 'new', False, True) + rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True, False), + mock.call('file', {'old': 'new'}, False, True))) + diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py new file mode 100644 index 0000000000000000000000000000000000000000..c53baa633654b4b31525c08e25dad727b52f863e --- /dev/null +++ b/test/unit/test_variable.py @@ -0,0 +1,184 @@ +# coding=utf-8 + +from mock import Mock +from unittest import TestCase +from earthdiagnostics.variable import CMORTable, VariableAlias, Variable, VariableJsonException +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.constants import Basins +from earthdiagnostics.frequency import Frequencies + + +class TestCMORTable(TestCase): + + def setUp(self): + self.frequency = Mock() + + def test_str(self): + self.assertEquals(str(CMORTable('name', 'm', 'Month YEAR')), 'name') + + +class TestVariableAlias(TestCase): + + def test_str(self): + alias = VariableAlias('alias') + self.assertEquals(str(alias), 'alias') + alias.basin = 'basin' + self.assertEquals(str(alias), 'alias Basin: basin') + alias.grid = 'grid' + self.assertEquals(str(alias), 'alias Basin: basin Grid: grid') + + +class TestVariable(TestCase): + + def test_parse_json(self): + var = Variable() + json = {'out_name': 'out_name', + 'standard_name': 'standard_name', + 'long_name': 'long_name', + 'modeling_realm': 'ocean', + 'valid_min': 'valid_min', + 'valid_max': 'valid_max', + 'units': 'units', + } + var.parse_json(json, 'out_name') + + self.assertEqual(var.short_name, 'out_name') + self.assertEqual(var.standard_name, 'standard_name') + self.assertEqual(var.long_name, 'long_name') + + self.assertEqual(var.valid_min, 'valid_min') + self.assertEqual(var.valid_max, 'valid_max') + self.assertEqual(var.units, 'units') + self.assertEqual(var.priority, 1) + + self.assertEqual(var.domain, ModelingRealms.ocean) + + def test_parse_json_no_out_name(self): + var = Variable() + json = {'standard_name': 'standard_name', + 'long_name': 'long_name', + 'modeling_realm': 'ocean', + 'valid_min': 'valid_min', + 'valid_max': 'valid_max', + 'units': 'units', + } + with self.assertRaises(VariableJsonException): + var.parse_json(json, 'out_name') + + def test_parse_json_with_priority(self): + var = Variable() + json = {'out_name': 'out_name', + 'standard_name': 'standard_name', + 'long_name': 'long_name', + 'modeling_realm': 'ocean', + 'valid_min': 'valid_min', + 'valid_max': 'valid_max', + 'units': 'units', + 'priority': '2', + } + var.parse_json(json, 'out_name') + + self.assertEqual(var.short_name, 'out_name') + self.assertEqual(var.standard_name, 'standard_name') + self.assertEqual(var.long_name, 'long_name') + + self.assertEqual(var.valid_min, 'valid_min') + self.assertEqual(var.valid_max, 'valid_max') + self.assertEqual(var.units, 'units') + self.assertEqual(var.priority, 2) + + self.assertEqual(var.domain, ModelingRealms.ocean) + + def test_parse_json_with_primavera_priority(self): + var = Variable() + json = {'out_name': 'out_name', + 'standard_name': 'standard_name', + 'long_name': 'long_name', + 'modeling_realm': 'ocean', + 'valid_min': 'valid_min', + 'valid_max': 'valid_max', + 'units': 'units', + 'primavera_priority': '2', + } + var.parse_json(json, 'out_name') + + self.assertEqual(var.short_name, 'out_name') + self.assertEqual(var.standard_name, 'standard_name') + self.assertEqual(var.long_name, 'long_name') + + self.assertEqual(var.valid_min, 'valid_min') + self.assertEqual(var.valid_max, 'valid_max') + self.assertEqual(var.units, 'units') + self.assertEqual(var.priority, 2) + + self.assertEqual(var.domain, ModelingRealms.ocean) + + def test_get_modelling_realm(self): + var = Variable() + domain = var.get_modelling_realm(('ocean',)) + self.assertEqual(ModelingRealms.ocean, domain) + + domain = var.get_modelling_realm(('ocean', 'atmos')) + self.assertEqual(ModelingRealms.ocean, domain) + + domain = var.get_modelling_realm(('ocean', 'ocnBgchem')) + self.assertEqual(ModelingRealms.ocnBgchem, domain) + + domain = var.get_modelling_realm(('ocean', 'seaIce')) + self.assertEqual(ModelingRealms.seaIce, domain) + + domain = var.get_modelling_realm(('atmos', 'atmosChem')) + self.assertEqual(ModelingRealms.atmosChem, domain) + + domain = var.get_modelling_realm(('land', 'landIce')) + self.assertEqual(ModelingRealms.landIce, domain) + + domain = var.get_modelling_realm(tuple()) + self.assertIsNone(domain) + + def test_parse_csv(self): + var = Variable() + var.parse_csv(['not_used', 'out_name', 'standard_name', 'long_name', 'ocean', 'global', 'units', + 'valid_min', 'valid_max', 'grid', 'Amon: ']) + self.assertEqual(var.short_name, 'out_name') + self.assertEqual(var.standard_name, 'standard_name') + self.assertEqual(var.long_name, 'long_name') + + self.assertEqual(var.valid_min, 'valid_min') + self.assertEqual(var.valid_max, 'valid_max') + self.assertEqual(var.units, 'units') + self.assertEqual(var.grid, 'grid') + + self.assertEqual(var.domain, ModelingRealms.ocean) + self.assertEqual(var.basin, Basins().Global) + + def test_get_table(self): + var = Variable() + var.domain = ModelingRealms.atmos + table = var.get_table(Frequencies.monthly, 'specs') + self.assertEqual(table.frequency, Frequencies.monthly) + self.assertEqual(table.name, 'Amon') + self.assertEqual(table.date, 'December 2013') + + def test_get_table_added(self): + var = Variable() + var.domain = ModelingRealms.atmos + var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) + table = var.get_table(Frequencies.monthly, 'specs') + self.assertEqual(table.frequency, Frequencies.monthly) + self.assertEqual(table.name, 'Amon') + self.assertEqual(table.date, 'December 2013') + + def test_get_table_not_added(self): + var = Variable() + var.domain = ModelingRealms.atmos + var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) + table = var.get_table(Frequencies.daily, 'specs') + self.assertEqual(table.frequency, Frequencies.daily) + self.assertEqual(table.name, 'day') + self.assertEqual(table.date, 'December 2013') + + def test_get_table_not_matching(self): + var = Variable() + with self.assertRaises(ValueError): + var.get_table(Frequencies.daily, 'specs') diff --git a/test/unit/test_variable_type.py b/test/unit/test_variable_type.py new file mode 100644 index 0000000000000000000000000000000000000000..28dd44f56c34599ce1c9d293d6a1a430e71709d7 --- /dev/null +++ b/test/unit/test_variable_type.py @@ -0,0 +1,20 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.variable_type import VariableType + + +class TestVariableType(TestCase): + + def test_mean(self): + self.assertEqual(VariableType.to_str(VariableType.MEAN), 'mean') + + def test_statistics(self): + self.assertEqual(VariableType.to_str(VariableType.STATISTIC), 'statistics') + + def test_bad_one(self): + with self.assertRaises(ValueError): + VariableType.to_str('bad type') + + + diff --git a/testing_ocean_pp_moore.job b/testing_ocean_pp_moore.job deleted file mode 100755 index 9e2cab3eb0fbcc58d192e0e6b61056aa2d7c79bb..0000000000000000000000000000000000000000 --- a/testing_ocean_pp_moore.job +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#$ -l h_vmem=4G -#$ -l s_rt=24:00:00 -#$ -l h_rt=24:00:00 - -set -evx -oceanpp_repository=/home/Earth/pbretonn/es_git/ocean_diagnostics -workdir=/scratch/tmp/post_ocean/$$ - -mkdir -p $workdir -cd $workdir -if [ -e ${oceanpp_repository}/ocean_pp.bash ] ; then - cp ${oceanpp_repository}/ocean_pp.bash . -else - echo "Please fill up the location of your ocean_pp repository" - exit 1 -fi - -#lstexp=('b02s_bis') -#lstexp=('b02s_ter' 'l00w_bis') #orig -lstexp=('b02s' 'i00k' 'i01t' 'l00v' 'l00w' 'glorys' 'nemovar' 'b02s_bis' 'b02s_ter' 'l00w_bis') #orig -for exp in ${lstexp[@]} ; do - tmp=${oceanpp_repository//\//\\\/} - sed -e "s/PATHCOMMONOCEANDIAG=.*/PATHCOMMONOCEANDIAG=${tmp}/g" /shared/earth/software/scripts/testing_ocean_pp/config_file-ocean_pp_${exp}.bash &> config_file-ocean_pp_${exp}.bash - ./ocean_pp.bash config_file-ocean_pp_${exp}.bash -done