diff --git a/.gitignore b/.gitignore index c583596b75d541f47c68b9ca0c3513c10efc4e10..cafa800b40cfb7157c94ee50969e873bfa644429 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ doc/build/* *.err *.out +*.nc .coverage htmlcov \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..ee454f263f45cb841deeeca4641091235517f4ca --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "earthdiagnostics/cmor_tables/cmip6"] + path = earthdiagnostics/cmor_tables/cmip6 + url = https://github.com/jvegasbsc/cmip6-cmor-tables.git diff --git a/MANIFEST.in b/MANIFEST.in index d3fa2e05d9aed9cd9ca4d233a33903d126df4437..fdd60f8f4758922148447187aae13cca9236e00f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,4 @@ -include earthdiagnostics/*.csv -include earthdiagnostics/*.so +graft earthdiagnostics include diags.conf include README -include VERSION -include earthdiagnostics/EarthDiagnostics.pdf \ No newline at end of file +include VERSION \ No newline at end of file diff --git a/VERSION b/VERSION index 989de3b2fcadf8b5f6a672023913c7a0834e5e22..b68884dd019aa160f491c3a61fcaa49d92f8d244 100644 --- a/VERSION +++ b/VERSION @@ -1 +1,2 @@ -3.0.0b22 +3.0.0b28 + diff --git a/diags.conf b/diags.conf index 0c17c47d198c03bedc86f45cca3386b6cc4c4e22..c096fcbddc64d2c7b29786814d10fe1001e1d101 100644 --- a/diags.conf +++ b/diags.conf @@ -1,19 +1,22 @@ [DIAGNOSTICS] # Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) -DATA_ADAPTOR = THREDDS +DATA_ADAPTOR = CMOR # Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER # Root path for the cmorized data to use DATA_DIR = /esnas:/esarchive # Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) DATA_TYPE = exp +# CMORization type to use. Important also for THREDDS as it affects variable name conventions. +# Options: SPECS (default), PRIMAVERA, CMIP6 +DATA_CONVENTION = SPECS # Path to NEMO's mask and grid files needed for CDFTools CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or -# an alias defined in the ALIAS section (see more below). If you are using the diagnpostics just to CMORize, leave it +# an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = climpercent,atmos,sfcWind,1 +DIAGS = # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. @@ -27,7 +30,7 @@ MAX_CORES = 1 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False -FORCE = True +FORCE = False # If true, CMORizes ocean files. Default = True OCEAN_FILES = True # If true, CMORizes atmosphere files. Default = True @@ -65,8 +68,9 @@ SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = ecmwf -MODEL = system4_m1 +INSTITUTE = IC3 +MODEL = EC-EARTH3 +NAME = windstress # Model version: Available versions MODEL_VERSION =Ec2.3_O1L46 # Atmospheric output timestep in hours @@ -82,9 +86,9 @@ OCEAN_TIMESTEP = 6 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = resilience -STARTDATES = 19810101 -MEMBERS = 0 +EXPID = a07o +STARTDATES = 20000201 20000501 20010201 20010501 20020201 20020501 20030201 20030501 20040201 20040501 20050201 20050501 20060201 20060501 20070201 20070501 20080201 20080501 20090201 20090501 +MEMBERS = 0 1 2 3 4 5 6 7 MEMBER_DIGITS = 1 CHUNK_SIZE = 7 CHUNKS = 1 diff --git a/doc/source/conf.py b/doc/source/conf.py index ec240e9bd22f5a115b142a51e38bba79fc6b2fb4..fdfc92c2d14744fd11f505435717d0d2914991b4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -59,12 +59,12 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the -# built documents. +# built documents.source ~/vi # # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b22' +release = '3.0.0b28' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf index b674e5c9495795b63c67b9eed0843513c33303ea..c81cd0eb00aa2b2eb28d562c0bf5843075455420 100644 Binary files a/earthdiagnostics/EarthDiagnostics.pdf and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index 06331b1d4d9e390f6cf7143fd5c356fb506ebb37..96368d7a88fdc17628f948d3ce2a878c1a5f53c5 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -58,6 +58,7 @@ class CDFTools(object): if not os.path.exists(output): raise Exception('Error executing {0}\n Output file not created', ' '.join(line)) + # noinspection PyShadowingBuiltins @staticmethod def _check_input(command, input, line): if input: @@ -71,18 +72,18 @@ class CDFTools(object): if not os.path.exists(element): raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) - @staticmethod - def is_exe(fpath): + # noinspection PyMethodMayBeStatic + def is_exe(self, fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) def _check_command_existence(self, command): if self.path: - if CDFTools.is_exe(os.path.join(self.path, command)): + if self.is_exe(os.path.join(self.path, command)): return else: for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, command) - if CDFTools.is_exe(exe_file): + if self.is_exe(exe_file): return raise ValueError('Error executing {0}\n Command does not exist in {1}', command, self.path) diff --git a/earthdiagnostics/cmor_tables/cmip6 b/earthdiagnostics/cmor_tables/cmip6 new file mode 160000 index 0000000000000000000000000000000000000000..8bae68e85e2dfa6ecd71bccb94479344d3acf75c --- /dev/null +++ b/earthdiagnostics/cmor_tables/cmip6 @@ -0,0 +1 @@ +Subproject commit 8bae68e85e2dfa6ecd71bccb94479344d3acf75c diff --git a/earthdiagnostics/cmor_table.csv b/earthdiagnostics/cmor_tables/default.csv similarity index 68% rename from earthdiagnostics/cmor_table.csv rename to earthdiagnostics/cmor_tables/default.csv index 5c403f0d47f23dc83ae18a26a5e300f8b17797e3..fc2742efab4bc0e096eccfd8683b659f2f838646 100644 --- a/earthdiagnostics/cmor_table.csv +++ b/earthdiagnostics/cmor_tables/default.csv @@ -1,302 +1,299 @@ -Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid -iiceages:siage:iice_otd,ageice,age_of_sea_ice,Age of sea ice,seaIce,,,,, -al,al,surface_albedo,Albedo,atmos,,,,, -bgfrcsal,bgfrcsal,change_over_time_in_heat_content_from_forcing,Change over time in salt content from forcing,ocean,,,,, -bgfrctem,bgfrctem,change_over_time_in_heat_content_from_forcing,Change over time in heat content from forcing,ocean,,,,, -bgfrcvol,bgfrcvol,change_over_time_in_volume_from_forcing,Change over time in volume from forcing,ocean,,,,, -bgheatco,bgheatco,change_over_time_in_heat_content,Change over time in sea water heat content,ocean,,,,, -bgsaline,bgsaline,change_over_time_in_sea_water_practical_salinity,Change over time in sea water salinity,ocean,,,,, -bgsaltco,bgsaltco,change_over_time_in_salt_content,Change over time in sea water salt content,ocean,,,,, -bgtemper,bgtemper,change_over_time_in_sea_water_potential_temperature,Change over time in sea water potential temperature,ocean,,,,, -bgvole3t,bgvole3t,change_over_time_in_volume_variation,Change over time in volume variation (e3t),ocean,,,,, -bgvolssh,bgvolssh,change_over_time_in_sea_surface_height,Change over time in sea surface height,ocean,,,,, -bld,bld,boundary_layer_dissipation,Boundary layer dissipation,atmos,,,,, -iicebome:iocewflx,bmelt,tendency_of_sea_ice_amount_due_to_basal_melting,Rate of melt at sea ice base,seaIce,,,,, -sobowlin,bowlin,bowl_index,Bowl index,ocean,,,,, -cc,cl,cloud_area_fraction_in_atmosphere_layer,Cloud area fraction,atmos,,,,, -hcc,clh,high_cloud_area_fraction,High cloud fraction,atmos,,,,, -lcc,cll,low_cloud_area_fraction,Low cloud fraction,atmos,,,,, -mcc,clm,medium_cloud_area_fraction,Medium cloud fraction,atmos,,,,, -ciwc,cli,mass_fraction_of_cloud_ice_in_air,Mass fraction of cloud ice,atmos,,,,, -tcc,clt,cloud_area_fraction,Total cloud fraction,atmos,,,,, -clwc,clw,mass_fraction_of_cloud_liquid_water_in_air,Mass fraction of cloud liquid water,atmos,,,,, -tcw,clwvi,atmosphere_cloud_condensed_water_content,Condensed water path,atmos,,,,, -iicedive:sidive,divice,Strain Rate Divergence of Sea Ice,Divergence_of_sea_ice_velocity,seaIce,,,,, -e,evspsbl,water_evaporation_flux,Evaporation,atmos,,,,, -fal,fal,forecast_albedo,Forecast albedo,atmos,,,,, -sowaflep,fatmosocean,atmosphere_ocean_water_flux,Atmos=>ocean net freshwater,ocean,,,,, -sowaflcd,fdilution,dilution_water_flux,Concentration/dilution water flux,ocean,,,,, -sophtldf,fhbasindif,northward_ocean_heat_transport_due_to_diffusion,Northward ocean heat transport due to diffusion,ocean,,,,, -iowaflup,ficeocean,ice_ocean_water_flux,Ice=>ocean net freshwater,ocean,,,,, -sorunoff,friver,water_flux_into_sea_water_from_rivers,Water flux into sea water from rivers ,ocean,,,,, -sowaflup,fupward,upward_water_flux,Net upward water flux,ocean,,,,, -gwd,gwd,gravity_wave_dissipation,Gravity wave dissipation,atmos,,,,, -ibgheatco,hcicega,global mean ice heat content,Global mean ice heat content,seaIce,,,,, -sbgheatco,hcsnga,global mean snow heat content,Global mean snow heat content,seaIce,,,,, -heatc,heatc,integral_of_sea_water_potential_temperature_wrt_depth_expressed_as_heat_content,Heat content vertically integrated,ocean,,,,, -sohtatl,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Atl,,,, -sohtind,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Ind,,,, -sohtipc,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,IndPac,,,, -sohtpac,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Pac,,,, -sophtadv,hfbasinadv,northward_ocean_heat_transport_due_to_advection,Northward ocean heat transport due to advection ,ocean,,,,, -sophteiv,hfbasinba,northward_ocean_heat_transport_due_to_bolus_advection,Northward ocean heat transport due to bolus advection ,ocean,,,,, -qt_oce:sohefldo:qt,hfds,surface_downward_heat_flux_in_sea_water,Downward heat flux at sea water surface,ocean,,,,, -slhf,hfls,surface_upward_latent_heat_flux,Surface upward latent heat flux,atmos,,,,, -sshf,hfss,surface_upward_sensible_heat_flux,Surface upward sensible heat flux,atmos,,,,, -sophtove,htovovrt,northward_ocean_heat_transport_due_to_overturning,Northward ocean heat transport due to overturning ,ocean,,,,, -q,hus,specific_humidity,Specific humidity,atmos,,,,, -soicealb,ialb,sea_ice_albedo,Sea ice albedo,seaIce,,,,, -ibgfrcsfx,ibgfrcsfx,global_mean_forcing_salt,Global mean forcing salt (sfx),seaIce,,,,, -ibgfrcvol,ibgfrcvol,globa_mean_forcing_volume,Global mean forcing volume (emp),seaIce,,,,, -ibghfxbog,ibghfxbog,heat_fluxes_causing_bottom_ice_growth,Heat fluxes causing bottom ice growth,seaIce,,,,, -ibghfxbom,ibghfxbom,heat_fluxes_causing_bottom_ice_melt,Heat fluxes causing bottom ice melt,seaIce,,,,, -ibghfxdhc,ibghfxdhc,Heat_content_variation_in_snow_and_ice,Heat content variation in snow and ice,seaIce,,,,, -ibghfxdif,ibghfxdif,heat_fluxes_causing_ice temperature_change,Heat fluxes causing ice temperature change,seaIce,,,,, -ibghfxdyn,ibghfxdyn,heat_fluxes_from_ice-ocean_exchange_during_dynamic,Heat fluxes from ice-ocean exchange during dynamic,seaIce,,,,, -ibghfxin,ibghfxin,total_heat_fluxes_at_the_ice_surface,Total heat fluxes at the ice surface,seaIce,,,,, -ibghfxopw,ibghfxopw,heat_fluxes_causing_open_water_ice_formation,Heat fluxes causing open water ice formation,seaIce,,,,, -ibghfxout,ibghfxout,non_solar_heat_fluxes_received_by_the_ocean,Non solar heat fluxes received by the ocean,seaIce,,,,, -ibghfxres,ibghfxres,heat_fluxes_from_ice-ocean_exchange_during_resultant,Heat fluxes from ice-ocean exchange during resultant,seaIce,,,,, -ibghfxsnw,ibghfxsnw,heat_fluxes_from_snow-ocean_exchange,Heat fluxes from snow-ocean exchange,seaIce,,,,, -ibghfxspr,ibghfxspr,Heat_content_of_snow_precip,Heat content of snow precip,seaIce,,,,, -ibghfxsub,ibghfxsub,heat_fluxes_from_sublimation,Heat fluxes from sublimation,seaIce,,,,, -ibghfxsum,ibghfxsum,heat_fluxes_causing_surface_ice_melt,Heat fluxes causing surface ice melt,seaIce,,,,, -ibghfxthd,ibghfxthd,heat_fluxes_from_ice-ocean_exchange_during_thermo,Heat fluxes from ice-ocean exchange during thermo,seaIce,,,,, -ibgsfxbog,ibgsfxbogga,salt_flux_thermo,Global mean salt flux (thermo),seaIce,,,,, -ibgsfxbom,ibgsfxbomga,salt_flux_bottom_melt,Global mean salt flux (bottom melt),seaIce,,,,, -ibgsfxbri,ibgsfxbriga,salt_flux_brines,Global mean salt flux (brines),seaIce,,,,, -ibgsfxdyn,ibgsfxdynga,salt_flux_dynamic,Global mean salt flux (dynamic),seaIce,,,,, -ibgsfx,ibgsfxga,salt_flux,Global mean salt flux (total),seaIce,,,,, -ibgsfxopw,ibgsfxopwga,salt_flux_open_waters,Global mean salt flux (open water),seaIce,,,,, -ibgsfxres,ibgsfxresga,salt_flux_resultant,Global mean salt flux (resultant),seaIce,,,,, -ibgsfxsni,ibgsfxsniga,salt_flux_snow_ice_growth,Global mean salt flux (snow-ice growth),seaIce,,,,, -ibgsfxsum,ibgsfxsumga,salt_flux_surface_melt,Global mean salt flux (surface melt),seaIce,,,,, -ibgvfxbog,ibgvfxbogga,volume_flux_bottom_growth,Global mean volume flux (bottom growth),seaIce,,,,, -ibgvfxbom,ibgvfxbomga,volume_flux_bottom_melt,Global mean volume flux (bottom melt),seaIce,,,,, -ibgvfxdyn,ibgvfxdynga,volume_flux_dynamic_growth,Global mean volume flux (dynamic growth),seaIce,,,,, -ibgvfx,ibgvfxga,volume_flux_emp,Global mean volume flux (emp),seaIce,,,,, -ibgvfxopw,ibgvfxopwga,volume_flux_open_water_growth,Global mean volume flux (open water growth),seaIce,,,,, -ibgvfxres,ibgvfxresga,volume_flux_resultant,Global mean volume flux (resultant),seaIce,,,,, -ibgvfxsni,ibgvfxsniga,volume_flux_snow_ice_growth,Global mean volume flux (snow-ice growth),seaIce,,,,, -ibgvfxsnw,ibgvfxsnwga,volume_flux_snow_melt,Global mean volume flux (snow melt),seaIce,,,,, -ibgvfxspr,ibgvfxsprga,snheco,Global mean volume flux (snow precip),seaIce,,,,, -ibgvfxsub,ibgvfxsubga,volume_flux_snow_sublimation,Global mean volume flux (snow sublimation),seaIce,,,,, -ibgvfxsum,ibgvfxsumga,volume_flux_surface_melt,Global mean volume flux (surface melt),seaIce,,,,, -ibgvolgrm,ibgvolgrm,global_mean_ice_growth+melt_volume,Global mean ice growth+melt volume,seaIce,,,,, -ibrinvol,ibrinvol,brine_volume,Brine volume,seaIce,,,,, -sibricat,ibrinvolcat,brine_volume_in_categories,Brine volume for categories,seaIce,,,,, -iicebopr,iicebopr,daily_bottom_thermo_ice_production,Daily bottom thermo ice production,seaIce,,,,, -iicecolf,iicecolf,frazil_ice_collection_thickness,Frazil ice collection thickness,seaIce,,,,, -iicedypr,iicedypr,daily_dynamic_ice_production,Daily dynamic ice production,seaIce,,,,, -iice_etd,iiceetd,brine_volume_distribution,Brine volume distribution,seaIce,,,,, -iicelapr,iicelapr,daily_lateral_thermo_ice_production,Daily lateral thermo ice prod.,seaIce,,,,, -iicenflx,iicenflx,nonsolar_flux_ice_ocean_surface,Non-solar flux at ice/ocean surface,seaIce,,,,, -iicesflx,iicesflx,solar_flux_ice_ocean_surface,Solar flux at ice/ocean surface,seaIce,,,,, -iiceshea,iiceshea,shear,Shear,seaIce,,,,, -iicesipr,iicesipr,daily_snowice_ice_production,Daily snowice ice production,seaIce,,,,, -iicfsbri,iicfsbri,brine_salt_flux,Fsbri - brine salt flux,seaIce,,,,, -iicfseqv,iicfseqv,equivalent_FW_salt_flux,Fseqv - equivalent fw salt flux,seaIce,,,,, -ioceflxb,ioceflxb,oceanic_flux_ar_ice_base,Oceanic flux at the ice base,seaIce,,,,, -iocehebr,iocehebr,heat_flux_due_to_brine_release,Heat flux due to brine release,seaIce,,,,, -iocesafl,iocesafl,salt_flux_ocean_surface,Salt flux at ocean surface,seaIce,,,,, -iocesflx,iocesflx,solar_fux_ocean_surface,Solar flux at ocean surface,seaIce,,,,, -iocetflx,iocetflx,total_flux_ocean_surface,Total flux at ocean surface,seaIce,,,,, -iocwnsfl,iocwnsfl,nonsolar_flux_ocean_surface,Non-solar flux at ocean surface,seaIce,,,,, -isstempe,isstempe,sea_surface_temperature,Sea surface temperature,seaIce,,K,,, -scmastot,masso,sea_water_mass,Sea water mass ,ocean,,,,, -mldkz5,mldkz5,ocean_mixed_layer_thickness_defined_by_vertical_tracer_diffusivity,Turbocline depth (kz = 5e-4),ocean,,,,, -somxl010:mldr10_1,mlotst,ocean_mixed_layer_thickness_defined_by_sigma_t,Ocean mixed layer thickness defined by sigma T ,ocean,,,,, -swvl1,mrlsl1,moisture_content_of_soil_layer_1, Water content of soil layer 1,land,,,,, -swvl2,mrlsl2,moisture_content_of_soil_layer_2, Water content of soil layer 2,land,,,,, -swvl3,mrlsl3,moisture_content_of_soil_layer_3, Water content of soil layer 3,land,,,,, -swvl4,mrlsl4,moisture_content_of_soil_layer_4, Water content of soil layer 4,land,,,,, -ro,mrro,runoff_flux,Total runoff,atmos,,,,, -tp:precip,pr,precipitation_flux,Precipitation,atmos,,,,, -cp,prc,convective_precipitation_flux,Convective precipitation,atmos,,,,, -lsp,prs,stratiform_precipitation_flux,Stratiform precipitation,atmos,,,,, -isnowpre,prsn,snowfall_flux,Surface snowfall rate into the sea ice portion of the grid cell,seaIce,,,,, -sf:snowpre,prsn,snowfall_flux,Snowfall flux,atmos,,,,, -tcwv,prw,atmosphere_water_vapor_content,Water vapor path,atmos,,,,, -msl,psl,air_pressure_at_sea_level,Sea level pressure,atmos,,,,, -qns_ice,qnsice,non_solar_heat_flux_at_ice_surface,Non-solar heat flux at ice surface: sum over categories,seaIce,,,,, -qt_ice,qtice,surface_downward_heat_flux_in_air,Surface downward heat flux in air,seaIce,,,,, -strd,rlds,surface_downwelling_longwave_flux_in_air,Surface downwelling longwave radiation,atmos,,,,, -strc:str,rls,surface_longwave_flux_in_air,Surface longwave radiation,atmos,,,,, -ttr,rlut,toa_outgoing_longwave_flux,Toa outgoing longwave radiation,atmos,,,,, -ttrc,rlutcs,toa_outgoing_longwave_flux_assuming_clear_sky,"Top net thermal radiation, clear sky",atmos,,,,, -ssrd,rsds,surface_downwelling_shortwave_flux_in_air,Surface downwelling shortwave radiation,atmos,,,,, -tsr,rsdt,toa_incoming_shortwave_flux,Toa incident shortwave radiation,atmos,,,,, -soshfldo,rsntds,net_downward_shortwave_flux_at_sea_water_surface,Net downward shortwave radiation at sea water surface ,ocean,,,,, -ssr,rss,surface_shortwave_flux_in_air,Surface shortwave radiation,atmos,,,,, -ssrc,rsscs,surface_shortwave_flux_in_air_assuming_clear_sky,Surface clear-sky shortwave radiation,atmos,,,,, -tsrc,rsut,toa_outgoing_shortwave_flux,Toa outgoing shortwave radiation,atmos,,,,, -saltc,saltc,salt_content_vertically_integrated,Salt content vertically integrated,ocean,,,,, -es,sbl,surface_snow_and_ice_sublimation_flux,Surface snow and ice sublimation flux,landIce,,,,, -sosalflx,sfs,salt_flux_surface,Surface salt flux,ocean,,,,, -si,si,solar_insolation,Solar insolation,atmos,,,,, -NArea,siarean,sea_ice_area,Total area of sea ice in the northern hemisphere,seaIce,,10^6 km2,,, -SArea,siareas,sea_ice_area,Total area of sea ice in the southern hemisphere,seaIce,,10^6 km2,,, -iiceconc:siconc:soicecov:ileadfra:ci,sic,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,, -ci,sic,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,,ifs -iice_itd:siconc_cat:siconcat,siccat,ice_area_in_categories,Ice area in categories,seaIce,,,,, -ibgarea,sicga,sea_ice_content,Global mean sea ice content,seaIce,,,,, -NExnsidc,siextentn,sea_ice_extent,Total area of all northern-hemisphere grid cells that are covered by at least 15 % areal fraction of sea ice,seaIce,,10^6 km2,,, -SExnsidc,siextents,sea_ice_extent,Total area of all southern-hemisphere grid cells that are covered by at least 15 % areal fraction of sea ice,seaIce,,10^6 km2,,, -iiceprod,sigr,ice_production,Ice production,seaIce,,,,, -iiceheco,siheco,integral_of_sea_ice_temperature_wrt_depth_expressed_as_heat_content,Sea ice heat content,seaIce,,,,, -ibgsaltco,sisaltcga,global mean ice salt content,Global mean ice salt content,seaIce,,,,, -iicethic:sithic,sit,sea_ice_thickness,Sea Ice Thickness,seaIce,,m,,, -iice_hid:sithic_cat:sithicat,sitcat,ice_thicknesss_in_categories,Ice thickness in categories,seaIce,,,,, -iicetemp,sitemp,ice_temperature,Mean ice temperature,seaIce,,K,,, -ibgtemper,sitempga,sea_ice_temperature,Global mean sea ice temperature,seaIce,,K,,, -iicevelo:sivelo,sivelo,ice_velocity,Ice velocity,seaIce,,,,, -iicevelu:sivelu,sivelu,ice_velocity_u,Ice velocity u,seaIce,,,,, -iicevelv:sivelv,sivelv,ice_velocity_v,Ice velocity v,seaIce,,,,, -ibgvoltot,sivolga,sea_ice_volume,Global mean sea ice volume,seaIce,,,,, -sivoln:NVolume,sivoln,sea_ice_volume,Total volume of sea ice in the northern hemisphere,seaIce,,10^3 km3,,, -sivols:SVolume,sivols,sea_ice_volume,Total volume of sea ice in the southern hemisphere,seaIce,,10^3 km3,,, -sivolu,sivolu,sea_ice_volume_per_unit_gridcell_area,Sea ice volume per gridcell area unit,seaIce,,,,, -sostatl,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,, -sostind,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,, -sostipc,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,, -sostpac,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,, -sopstadv,sltbasinadv,northward_ocean_salt_transport_due_to_advection,Northward ocean salt transport due to advection ,ocean,,,,, -sopsteiv,sltbasinba,northward_ocean_salt_transport_due_to_bolus_advection,Northward ocean salt transport due to bolus advection ,ocean,,,,, -sopstldf,sltbasindif,northward_ocean_salt_transport_due_to_diffusion,Northward ocean salt transport due to diffusion,ocean,,,,, -sltnortha,sltnortha,northward_ocean_salt_transport,Atlantic northward ocean salt transport,ocean,,,,, -sopstove,sltovovrt,northward_ocean_salt_transport_due_to_overturning,Northward ocean salt transport due to overturning ,ocean,,,,, -zosalatl,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Atl,psu,,, -zosalglo,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Glob,psu,,, -zosalind,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Ind,psu,,, -zosalipc,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,IndPac,psu,,, -zosalpac,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Pac,psu,,, -asn,snal,snow_albedo,Snow albedo,landIce,,,,, -iice_hsd:snthicat,sndcat,snow_thickness_in_categories,Snow thickness in in categories,seaIce,,,,, -isnoheco,snheco,snow_heat_content,Snow total heat content,seaIce,,,,, -sd,snld,lwe_thickness_of_surface_snow_amount,Snow depth,atmos,,,,, -smlt,snm,surface_snow_melt_flux,Surface snow melt,landIce,,,,, -isnowthi,snthic,surface_snow_thickness,Surface snow thickness,seaIce,,,,, -sbgvoltot,snvolga,snow_volume,Global mean snow volume,seaIce,,,,, -snvolu,snvolu,snow_volume_per_unit_gridcell_area,Snow volume per gridcell area unit,seaIce,,,,, -vosaline:mean_3Dsosaline,so,sea_water_salinity,Sea water salinity,ocean,,psu,,, -scsaltot,soga,sea_water_salinity,Global mean sea water salinity ,ocean,,psu,,, -hfnortha,sohtatl,northward_ocean_heat_transport,Atlantic northward ocean heat transport,ocean,,,,, -soleaeiw,soleaeiw,eddy_induced_velocity_coefficient,Eddy induced vel. coeff. at w-point,ocean,,,,, -soleahtw,soleahtw,lateral_eddy_diffusivity,Lateral eddy diffusivity,ocean,,,,, -somixhgt,somixhgt,mixing_layer_depth_turbocline,Mixing layer depth (turbocline),ocean,,,,, -sosaline:isssalin:mean_sosaline,sos,sea_surface_salinity,Sea surface salinity ,ocean,,psu,,, -sothedep,sothedep,thermocline_depth,Thermocline depth (max dt/dz),ocean,,,,, -src,src,skin_reservoir_content,Skin reservoir content,land,,,,, -zosrfatl,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Atl,,,, -zosrfglo,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Glob,,,, -zosrfind,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Ind,,,, -zosrfipc,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,IndPac,,,, -zosrfpac,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Pac,,,, -rsn,srho,snow_density,Snow density,landIce,,,,, -iicesali:iice_std,ssi,sea_ice_salinity,Sea ice salinity,seaIce,,psu,,, -salincat,ssicat,sea_ice_salinity_in_categories,Sea-ice bulk salinity for categories,seaIce,,psu,,, -ibgsaline,ssiga,sea_ice_salinity,Global mean sea ice salinity ,seaIce,,psu,,, -iicestre,streng,compressive_strength_of_sea_ice,Compressive sea ice strength,seaIce,,,,, -so20chgt,t20d,depth_of_isosurface_of_sea_water_potential_temperature,,ocean,,,,, -t,ta,air_temperature,Air temperature,atmos,,K,,, -t2m,tas,air_temperature,Near-surface air temperature,atmos,,K,170,370, -mx2t,tasmax,air_temperature,Daily maximum near-surface air temperature,atmos,,K,,, -mn2t,tasmin,air_temperature,Daily minimum near-surface air temperature,atmos,,K,,, -ewss,tauu,surface_downward_eastward_stress,Surface downward eastward wind stress,atmos,,,,, -utau_ice:iocestru:iicestru,strairx,surface_downward_x_stress,X-Component of Atmospheric Stress On Sea Ice,seaIce,,N m-2,,, -sozotaux,tauuo,surface_downward_x_stress,Surface downward x stress ,ocean,,,,, -nsss,tauv,surface_downward_northward_stress,Surface downward northward wind stress,atmos,,,,, -vtau_ice:iocestrv:iicestrv,strairy,surface_downward_y_stress,Y-Component of Atmospheric Stress On Sea Ice,seaIce,,N m-2,,, -sozotauy:sometauy,tauvo,surface_downward_y_stress,Surface downward y stress ,ocean,,,,, -d2m,tdps,dew_point_temperature,2m dewpoint temperature,atmos,,K,,, -votemper:mean_3Dsosstsst,thetao,sea_water_potential_temperature,Sea water potential temperature,ocean,,K,,, -sctemtot,thetaoga,sea_water_potential_temperature,Global average sea water potential temperature ,ocean,,K,,, -iicesume,tmelt,tendency_of_sea_ice_amount_due_to_surface_melting,Rate of melt at upper surface of sea ice,seaIce,,,,, -sosstsst:mean_sosstsst,tos,sea_surface_temperature,Sea surface temperature ,ocean,,K,170,370, -sstk,tos,sea_surface_temperature,Sea surface temperature ,ocean,,K,170,370,ifs -tossq,tossq,square_of_sea_surface_temperature,Square of sea surface temperature ,ocean,,K2,,, -zotematl,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Atl,K,,, -zotemglo,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Glob,K,,, -zotemind,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Ind,K,,, -zotemipc,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,IndPac,K,,, -zotempac,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Pac,K,,, -skt,ts,surface_temperature,Surface temperature,atmos,,K,,, -iicesurt:soicetem:sistem,tsice,surface_temperature,Surface temperature of sea ice,seaIce,,K,,, -istl1,tsice,surface_temperature,Surface temperature of ice,landIce,,K,,, -stl1,tsl1,soil_temperature_level_1,Temperature of soil level 1,land,,,,, -stl2,tsl2,soil_temperature_level_2,Temperature of soil level 2,land,,,,, -stl3,tsl3,soil_temperature_level_3,Temperature of soil level 3,land,,,,, -stl4,tsl4,soil_temperature_level_4,Temperature of soil level 4,land,,,,, -tsn,tsn,temperature_in_surface_snow,Snow internal temperature,landIce,,,,, -u,ua,eastward_wind,U velocity,atmos,,,,, -u10m,uas,eastward_wind,Eastward near-surface wind,atmos,,,,, -vozocrtx,uo,sea_water_x_velocity,Sea water x velocity,ocean,,,,, -uos,uos,sea_surface_x_velocity,Sea surface x velocity,ocean,,,,, -v,va,northward_wind,V velocity,atmos,,,,, -v10m,vas,northward_wind,Northward near-surface wind,atmos,,,,, -vomecrty,vo,sea_water_y_velocity,Sea water y velocity,ocean,,,,, -vos,vos,sea_surface_y_velocity,Sea surface y velocity,ocean,,,,, -voddmavs,voddmavs,salt_vertical_eddy_diffusivity,Salt vertical eddy diffusivity,ocean,,,,, -vozoeivu,voeivu,sea_water_x_EIV_current,Zonal eiv current,ocean,,,,, -vomeeivv,voeivv,sea_water_y_EIV_current,Meridional eiv current,ocean,,,,, -voveeivw,voeivz,sea_water_z_EIV_current,Vertical eiv current,ocean,,,,, -scvoltot,volo,sea_water_volume,Sea water volume ,ocean,,,,, -votkeavm,votkeavm,vertical_eddy_viscosity,Vertical eddy viscosity,ocean,,,,, -votkeavt,votkeavt,vertical_eddy_diffusivity,Vertical eddy diffusivity,ocean,,,,, -votkeevd,votkeevd,enhanced_vertical_diffusivity,Enhanced vertical diffusivity,ocean,,,,, -votkeevm,votkeevm,enhanced_vertical_viscosity,Enhanced vertical viscosity,ocean,,,,, -sobarstf,vsftbarot,ocean_barotropic_volume_streamfunction,Ocean barotropic volume streamfunction ,ocean,,,,, -zomsfatl,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Atl,,,, -zomsfglo,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Glob,,,, -zomsfind,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Ind,,,, -zomsfipc:zomsfinp,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,IndPac,,,, -zomsfpac,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Pac,,,, -zomsfeiv,vsftmyzba,ocean_meridional_overturning_mass_streamfunction_due_to_bolus_advection,Ocean meridional overturning volume streamfunction due to bolus advection ,ocean,,,,, -w,wa,vertical_velocity,Vertical velocity,atmos,,,,, -z,zg,geopotential_height,Geopotential height,atmos,,,,, -vovecrtz,zo,sea_water_z_velocity,Sea water z velocity,ocean,,,,, -sossheigh:sossheig:mean_sossheig,zos,sea_surface_height_above_geoid,Sea surface height above geoid ,ocean,,,,, -scsshtot,zosga,global_average_sea_level_change,Global average sea level change ,ocean,,,,, -scsshste,zossga,global_average_steric_sea_level_change,Global average steric sea level change ,ocean,,,,, -zossq,zossq,square_of_sea_surface_height_above_geoid,Square of sea surface height above geoid ,ocean,,,,, -scsshtst,zostoga,snthic,Global average thermosteric sea level change ,ocean,,,,, -heatc,ohc,ocean_heat_content,Ocean heat content,ocean,,J,,, -ohcsum,ohcsum,total_ocean_heat_content,Total Ocean heat content,ocean,,J,,, -ohcvmean,ohcvmean,average_ocean_heat_content,Average Ocean heat content,ocean,,J m-3,,, -ohc,ohc,ocean_heat_content,Ocean heat content,ocean,,J m-2,,, -transix,transix,sea_ice_x_transport,X-Component of Sea Ice Mass Transport,seaIce,,kg s-1,,, -transiy,transiy,sea_ice_y_transport,Y-Component of Sea Ice Mass Transport,seaIce,,kg s-1,,, -windsp,sfcWind,wind_speed,Near-Surface Wind Speed,atmos,,,,, -vsfsit,vsfsit,virtual_salt_flux_into_sea_water_due_to_sea_ice_thermodynamics,Virtual Salt Flux into Sea Water due to Sea Ice Thermodynamics ,ocean,,,,, -sfdsi,sfdsi,downward_sea_ice_basal_salt_flux,Downward Sea Ice Basal Salt Flux,ocean,,,,, -hfsithermds,hfsithermds,heat_flux_into_sea_water_due_to_sea_ice_thermodynamics,Heat Flux into Sea Water due to Sea Ice Thermodynamics ,ocean,,,,, -u2o,uosq,square_of_sea_water_x_velocity,Square of Sea Water X Velocity ,ocean,,,,, -v2o,vosq,square_of_sea_water_y_velocity,Square of Sea Water Y Velocity ,ocean,,,,, -vozomatr,umo,ocean_mass_x_transport,Ocean Mass X Transport ,ocean,,,,, -vomematr,vmo,ocean_mass_y_transport,Ocean Mass Y Transport ,ocean,,,,, -sozohetr,hfx,ocean_heat_x_transport,Ocean Heat X Transport ,ocean,,,,, -somehetr,hfy,ocean_heat_y_transport,Ocean Heat Y Transport ,ocean,,,,, -uto,uothetao,product_of_xward_sea_water_velocity_and_temperature,Product of X-ward Sea Water Velocity and Temperature,ocean,,,,, -vto,vothetao,product_of_yward_sea_water_velocity_and_temperature,Product of Y-ward Sea Water Velocity and Temperature,ocean,,,,, -uso,uoso,product_of_xward_sea_water_velocity_and_salinity,Product of X-ward Sea Water Velocity and Salinity,ocean,,,,, -vso,voso,product_of_yward_sea_water_velocity_and_salinity,Product of Y-ward Sea Water Velocity and Salinity,ocean,,,,, -wfo,wfo,water_flux_into_sea_water,Water Flux into Sea Water ,ocean,,,,, -emp_oce,evsmpr,evap_minus_precip_over_sea_water,Evap minus Precip over ocean,ocean,,,,, -emp_ice,evsmpr,evap_minus_precip_over_sea_ice,Evap minus Precip over ice,seaIce,,,,, -qsr_oce,rsntds,net_downward_shortwave_flux_at_sea_water_surface,Net Downward Shortwave Radiation at Sea Water Surface ,ocean,,,,, -qns_oce,rlds,surface_net_downward_longwave_flux,Surface Net Downward Longwave Radiation,ocean,,,,, -qsr_ice,rsdssi,surface_downwelling_shortwave_flux_in_air,Downwelling Shortwave over Sea Ice,seaIce,,,,, -qns_ice,rldssi,surface_downwelling_longwave_flux_in_air,Downwelling Long Wave over Sea Ice,seaIce,,,,, -sfx,sfx,downward_salt_flux,Downward Salt Flux,ocean,,,,, -taum,taum,surface_downward_stress_module,Surface Downward Stress Module,ocean,,,,, -zfull,zfull,depth_below_geoid,Depth Below Geoid of Ocean Layer,ocean,,,,, -zhalf,zhalf,depth_below_geoid,Depth Below Geoid of Ocean Layer,ocean,,,,, -pbo,pbo,sea_water_pressure_at_sea_floor,Sea Water Pressure at Sea Floor,ocean,,,,, -thkcello,thkcello,cell_thickness,Cell Thickness,ocean,,,,, -ficeberg,ficeberg,water_flux_into_sea_water_from_icebergs,Water Flux into Sea Water From Icebergs ,ocean,,,,, -rsdo,rsds,downwelling_shortwave_flux_in_sea_water,Downwelling Shortwave Radiation in Sea Water ,ocean,,,,, -wo,wo,sea_water_upward_velocity,Sea Water Upward Velocity ,ocean,,,,, -w2o,wosq,square_of_sea_water_upward_velocity,Square of Sea Water Upward Velocity ,ocean,,,,, -difvho,difvho,ocean_vertical_heat_diffusivity,Ocean Vertical Heat Diffusivity,ocean,,,,, -vovematr,wmo,upward_ocean_mass_transport,Upward Ocean Mass Transport ,ocean,,,,, -qtr_ice,qtr,shortwave_flux_transmitted_through_ice,Shortwave Flux Transmitted Through The Ice,seaIce,,,,, +Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables +iiceages:siage:iice_otd,ageice,age_of_sea_ice,Age of sea ice,seaIce,,,,,, +al,al,surface_albedo,Albedo,atmos,,,,,, +bgfrcsal,bgfrcsal,change_over_time_in_heat_content_from_forcing,Change over time in salt content from forcing,ocean,,,,,, +bgfrctem,bgfrctem,change_over_time_in_heat_content_from_forcing,Change over time in heat content from forcing,ocean,,,,,, +bgfrcvol,bgfrcvol,change_over_time_in_volume_from_forcing,Change over time in volume from forcing,ocean,,,,,, +bgheatco,bgheatco,change_over_time_in_heat_content,Change over time in sea water heat content,ocean,,,,,, +bgsaline,bgsaline,change_over_time_in_sea_water_practical_salinity,Change over time in sea water salinity,ocean,,,,,, +bgsaltco,bgsaltco,change_over_time_in_salt_content,Change over time in sea water salt content,ocean,,,,,, +bgtemper,bgtemper,change_over_time_in_sea_water_potential_temperature,Change over time in sea water potential temperature,ocean,,,,,, +bgvole3t,bgvole3t,change_over_time_in_volume_variation,Change over time in volume variation (e3t),ocean,,,,,, +bgvolssh,bgvolssh,change_over_time_in_sea_surface_height,Change over time in sea surface height,ocean,,,,,, +bld,bld,boundary_layer_dissipation,Boundary layer dissipation,atmos,,,,,, +iicebome:iocewflx,bmelt,tendency_of_sea_ice_amount_due_to_basal_melting,Rate of melt at sea ice base,seaIce,,,,,, +sobowlin,bowlin,bowl_index,Bowl index,ocean,,,,,, +cc,cl,cloud_area_fraction_in_atmosphere_layer,Cloud area fraction,atmos,,,,,, +hcc,clh,high_cloud_area_fraction,High cloud fraction,atmos,,,,,, +lcc,cll,low_cloud_area_fraction,Low cloud fraction,atmos,,,,,, +mcc,clm,medium_cloud_area_fraction,Medium cloud fraction,atmos,,,,,, +ciwc,cli,mass_fraction_of_cloud_ice_in_air,Mass fraction of cloud ice,atmos,,,,,, +tcc,clt,cloud_area_fraction,Total cloud fraction,atmos,,,,,, +clwc,clw,mass_fraction_of_cloud_liquid_water_in_air,Mass fraction of cloud liquid water,atmos,,,,,, +tcw,clwvi,atmosphere_cloud_condensed_water_content,Condensed water path,atmos,,,,,, +iicedive:sidive,divice,Strain Rate Divergence of Sea Ice,Divergence_of_sea_ice_velocity,seaIce,,,,,, +e,evspsbl,water_evaporation_flux,Evaporation,atmos,,,,,, +fal,fal,forecast_albedo,Forecast albedo,atmos,,,,,, +sowaflep,fatmosocean,atmosphere_ocean_water_flux,Atmos=>ocean net freshwater,ocean,,,,,, +sowaflcd,fdilution,dilution_water_flux,Concentration/dilution water flux,ocean,,,,,, +sophtldf,fhbasindif,northward_ocean_heat_transport_due_to_diffusion,Northward ocean heat transport due to diffusion,ocean,,,,,, +iowaflup,ficeocean,ice_ocean_water_flux,Ice=>ocean net freshwater,ocean,,,,,, +sorunoff,friver,water_flux_into_sea_water_from_rivers,Water flux into sea water from rivers ,ocean,,,,,, +sowaflup,fupward,upward_water_flux,Net upward water flux,ocean,,,,,, +gwd,gwd,gravity_wave_dissipation,Gravity wave dissipation,atmos,,,,,, +ibgheatco,hcicega,global mean ice heat content,Global mean ice heat content,seaIce,,,,,, +sbgheatco,hcsnga,global mean snow heat content,Global mean snow heat content,seaIce,,,,,, +heatc,heatc,integral_of_sea_water_potential_temperature_wrt_depth_expressed_as_heat_content,Heat content vertically integrated,ocean,,J m-2,,,, +sohtatl,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Atl,,,,, +sohtind,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Ind,,,,, +sohtipc,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,IndPac,,,,, +sohtpac,hfbasin,northward_ocean_heat_transport,Northward ocean heat transport,ocean,Pac,,,,, +sophtadv,hfbasinadv,northward_ocean_heat_transport_due_to_advection,Northward ocean heat transport due to advection ,ocean,,,,,, +sophteiv,hfbasinba,northward_ocean_heat_transport_due_to_bolus_advection,Northward ocean heat transport due to bolus advection ,ocean,,,,,, +qt_oce:sohefldo:qt,hfds,surface_downward_heat_flux_in_sea_water,Downward heat flux at sea water surface,ocean,,,,,, +slhf,hfls,surface_upward_latent_heat_flux,Surface upward latent heat flux,atmos,,,,,, +sshf,hfss,surface_upward_sensible_heat_flux,Surface upward sensible heat flux,atmos,,,,,, +sophtove,htovovrt,northward_ocean_heat_transport_due_to_overturning,Northward ocean heat transport due to overturning ,ocean,,,,,, +q,hus,specific_humidity,Specific humidity,atmos,,,,,, +soicealb,ialb,sea_ice_albedo,Sea ice albedo,seaIce,,,,,, +ibgfrcsfx,ibgfrcsfx,global_mean_forcing_salt,Global mean forcing salt (sfx),seaIce,,,,,, +ibgfrcvol,ibgfrcvol,globa_mean_forcing_volume,Global mean forcing volume (emp),seaIce,,,,,, +ibghfxbog,ibghfxbog,heat_fluxes_causing_bottom_ice_growth,Heat fluxes causing bottom ice growth,seaIce,,,,,, +ibghfxbom,ibghfxbom,heat_fluxes_causing_bottom_ice_melt,Heat fluxes causing bottom ice melt,seaIce,,,,,, +ibghfxdhc,ibghfxdhc,Heat_content_variation_in_snow_and_ice,Heat content variation in snow and ice,seaIce,,,,,, +ibghfxdif,ibghfxdif,heat_fluxes_causing_ice temperature_change,Heat fluxes causing ice temperature change,seaIce,,,,,, +ibghfxdyn,ibghfxdyn,heat_fluxes_from_ice-ocean_exchange_during_dynamic,Heat fluxes from ice-ocean exchange during dynamic,seaIce,,,,,, +ibghfxin,ibghfxin,total_heat_fluxes_at_the_ice_surface,Total heat fluxes at the ice surface,seaIce,,,,,, +ibghfxopw,ibghfxopw,heat_fluxes_causing_open_water_ice_formation,Heat fluxes causing open water ice formation,seaIce,,,,,, +ibghfxout,ibghfxout,non_solar_heat_fluxes_received_by_the_ocean,Non solar heat fluxes received by the ocean,seaIce,,,,,, +ibghfxres,ibghfxres,heat_fluxes_from_ice-ocean_exchange_during_resultant,Heat fluxes from ice-ocean exchange during resultant,seaIce,,,,,, +ibghfxsnw,ibghfxsnw,heat_fluxes_from_snow-ocean_exchange,Heat fluxes from snow-ocean exchange,seaIce,,,,,, +ibghfxspr,ibghfxspr,Heat_content_of_snow_precip,Heat content of snow precip,seaIce,,,,,, +ibghfxsub,ibghfxsub,heat_fluxes_from_sublimation,Heat fluxes from sublimation,seaIce,,,,,, +ibghfxsum,ibghfxsum,heat_fluxes_causing_surface_ice_melt,Heat fluxes causing surface ice melt,seaIce,,,,,, +ibghfxthd,ibghfxthd,heat_fluxes_from_ice-ocean_exchange_during_thermo,Heat fluxes from ice-ocean exchange during thermo,seaIce,,,,,, +ibgsfxbog,ibgsfxbogga,salt_flux_thermo,Global mean salt flux (thermo),seaIce,,,,,, +ibgsfxbom,ibgsfxbomga,salt_flux_bottom_melt,Global mean salt flux (bottom melt),seaIce,,,,,, +ibgsfxbri,ibgsfxbriga,salt_flux_brines,Global mean salt flux (brines),seaIce,,,,,, +ibgsfxdyn,ibgsfxdynga,salt_flux_dynamic,Global mean salt flux (dynamic),seaIce,,,,,, +ibgsfx,ibgsfxga,salt_flux,Global mean salt flux (total),seaIce,,,,,, +ibgsfxopw,ibgsfxopwga,salt_flux_open_waters,Global mean salt flux (open water),seaIce,,,,,, +ibgsfxres,ibgsfxresga,salt_flux_resultant,Global mean salt flux (resultant),seaIce,,,,,, +ibgsfxsni,ibgsfxsniga,salt_flux_snow_ice_growth,Global mean salt flux (snow-ice growth),seaIce,,,,,, +ibgsfxsum,ibgsfxsumga,salt_flux_surface_melt,Global mean salt flux (surface melt),seaIce,,,,,, +ibgvfxbog,ibgvfxbogga,volume_flux_bottom_growth,Global mean volume flux (bottom growth),seaIce,,,,,, +ibgvfxbom,ibgvfxbomga,volume_flux_bottom_melt,Global mean volume flux (bottom melt),seaIce,,,,,, +ibgvfxdyn,ibgvfxdynga,volume_flux_dynamic_growth,Global mean volume flux (dynamic growth),seaIce,,,,,, +ibgvfx,ibgvfxga,volume_flux_emp,Global mean volume flux (emp),seaIce,,,,,, +ibgvfxopw,ibgvfxopwga,volume_flux_open_water_growth,Global mean volume flux (open water growth),seaIce,,,,,, +ibgvfxres,ibgvfxresga,volume_flux_resultant,Global mean volume flux (resultant),seaIce,,,,,, +ibgvfxsni,ibgvfxsniga,volume_flux_snow_ice_growth,Global mean volume flux (snow-ice growth),seaIce,,,,,, +ibgvfxsnw,ibgvfxsnwga,volume_flux_snow_melt,Global mean volume flux (snow melt),seaIce,,,,,, +ibgvfxspr,ibgvfxsprga,snheco,Global mean volume flux (snow precip),seaIce,,,,,, +ibgvfxsub,ibgvfxsubga,volume_flux_snow_sublimation,Global mean volume flux (snow sublimation),seaIce,,,,,, +ibgvfxsum,ibgvfxsumga,volume_flux_surface_melt,Global mean volume flux (surface melt),seaIce,,,,,, +ibgvolgrm,ibgvolgrm,global_mean_ice_growth+melt_volume,Global mean ice growth+melt volume,seaIce,,,,,, +ibrinvol,ibrinvol,brine_volume,Brine volume,seaIce,,,,,, +sibricat,ibrinvolcat,brine_volume_in_categories,Brine volume for categories,seaIce,,,,,, +iicebopr,iicebopr,daily_bottom_thermo_ice_production,Daily bottom thermo ice production,seaIce,,,,,, +iicecolf,iicecolf,frazil_ice_collection_thickness,Frazil ice collection thickness,seaIce,,,,,, +iicedypr,iicedypr,daily_dynamic_ice_production,Daily dynamic ice production,seaIce,,,,,, +iice_etd,iiceetd,brine_volume_distribution,Brine volume distribution,seaIce,,,,,, +iicelapr,iicelapr,daily_lateral_thermo_ice_production,Daily lateral thermo ice prod.,seaIce,,,,,, +iicenflx,iicenflx,nonsolar_flux_ice_ocean_surface,Non-solar flux at ice/ocean surface,seaIce,,,,,, +iicesflx,iicesflx,solar_flux_ice_ocean_surface,Solar flux at ice/ocean surface,seaIce,,,,,, +iiceshea,iiceshea,shear,Shear,seaIce,,,,,, +iicesipr,iicesipr,daily_snowice_ice_production,Daily snowice ice production,seaIce,,,,,, +iicfsbri,iicfsbri,brine_salt_flux,Fsbri - brine salt flux,seaIce,,,,,, +iicfseqv,iicfseqv,equivalent_FW_salt_flux,Fseqv - equivalent fw salt flux,seaIce,,,,,, +ioceflxb,ioceflxb,oceanic_flux_ar_ice_base,Oceanic flux at the ice base,seaIce,,,,,, +iocehebr,iocehebr,heat_flux_due_to_brine_release,Heat flux due to brine release,seaIce,,,,,, +iocesafl,iocesafl,salt_flux_ocean_surface,Salt flux at ocean surface,seaIce,,,,,, +iocesflx,iocesflx,solar_fux_ocean_surface,Solar flux at ocean surface,seaIce,,,,,, +iocetflx,iocetflx,total_flux_ocean_surface,Total flux at ocean surface,seaIce,,,,,, +iocwnsfl,iocwnsfl,nonsolar_flux_ocean_surface,Non-solar flux at ocean surface,seaIce,,,,,, +isstempe,isstempe,sea_surface_temperature,Sea surface temperature,seaIce,,K,,,, +scmastot,masso,sea_water_mass,Sea water mass ,ocean,,,,,, +mldkz5,mldkz5,ocean_mixed_layer_thickness_defined_by_vertical_tracer_diffusivity,Turbocline depth (kz = 5e-4),ocean,,,,,, +somxl010:mldr10_1,mlotst,ocean_mixed_layer_thickness_defined_by_sigma_t,Ocean mixed layer thickness defined by sigma T ,ocean,,,,,, +swvl1,mrlsl1,moisture_content_of_soil_layer_1, Water content of soil layer 1,land,,,,,, +swvl2,mrlsl2,moisture_content_of_soil_layer_2, Water content of soil layer 2,land,,,,,, +swvl3,mrlsl3,moisture_content_of_soil_layer_3, Water content of soil layer 3,land,,,,,, +swvl4,mrlsl4,moisture_content_of_soil_layer_4, Water content of soil layer 4,land,,,,,, +ro,mrro,runoff_flux,Total runoff,atmos,,,,,, +tp:precip,pr,precipitation_flux,Precipitation,atmos,,,,,, +cp,prc,convective_precipitation_flux,Convective precipitation,atmos,,,,,, +lsp,prs,stratiform_precipitation_flux,Stratiform precipitation,atmos,,,,,, +isnowpre,prsn,snowfall_flux,Surface snowfall rate into the sea ice portion of the grid cell,seaIce,,,,,, +sf:snowpre,prsn,snowfall_flux,Snowfall flux,atmos,,,,,, +tcwv,prw,atmosphere_water_vapor_content,Water vapor path,atmos,,,,,, +msl,psl,air_pressure_at_sea_level,Sea level pressure,atmos,,,,,, +qns_ice,qnsice,non_solar_heat_flux_at_ice_surface,Non-solar heat flux at ice surface: sum over categories,seaIce,,,,,, +qt_ice,qtice,surface_downward_heat_flux_in_air,Surface downward heat flux in air,seaIce,,,,,, +strd,rlds,surface_downwelling_longwave_flux_in_air,Surface downwelling longwave radiation,atmos,,,,,, +strc:str,rls,surface_longwave_flux_in_air,Surface longwave radiation,atmos,,,,,, +ttr,rlut,toa_outgoing_longwave_flux,Toa outgoing longwave radiation,atmos,,,,,, +ttrc,rlutcs,toa_outgoing_longwave_flux_assuming_clear_sky,"Top net thermal radiation, clear sky",atmos,,,,,, +ssrd,rsds,surface_downwelling_shortwave_flux_in_air,Surface downwelling shortwave radiation,atmos,,,,,, +tsr,rsdt,toa_incoming_shortwave_flux,Toa incident shortwave radiation,atmos,,,,,, +soshfldo,rsntds,net_downward_shortwave_flux_at_sea_water_surface,Net downward shortwave radiation at sea water surface ,ocean,,,,,, +ssr,rss,surface_shortwave_flux_in_air,Surface shortwave radiation,atmos,,,,,, +ssrc,rsscs,surface_shortwave_flux_in_air_assuming_clear_sky,Surface clear-sky shortwave radiation,atmos,,,,,, +tsrc,rsut,toa_outgoing_shortwave_flux,Toa outgoing shortwave radiation,atmos,,,,,, +saltc,saltc,salt_content_vertically_integrated,Salt content vertically integrated,ocean,,,,,, +es,sbl,surface_snow_and_ice_sublimation_flux,Surface snow and ice sublimation flux,landIce,,,,,, +sosalflx,sfs,salt_flux_surface,Surface salt flux,ocean,,,,,, +si,si,solar_insolation,Solar insolation,atmos,,,,,, +NArea,siarean,sea_ice_area,Total area of sea ice in the northern hemisphere,seaIce,,10^6 km2,,,, +SArea,siareas,sea_ice_area,Total area of sea ice in the southern hemisphere,seaIce,,10^6 km2,,,, +iiceconc:siconc:soicecov:ileadfra:ci,sic,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,,, +ci,sic,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,,ifs, +iice_itd:siconc_cat:siconcat,siccat,ice_area_in_categories,Ice area in categories,seaIce,,,,,, +ibgarea,sicga,sea_ice_content,Global mean sea ice content,seaIce,,,,,, +NExnsidc,siextentn,sea_ice_extent,Total area of all northern-hemisphere grid cells that are covered by at least 15 % areal fraction of sea ice,seaIce,,10^6 km2,,,, +SExnsidc,siextents,sea_ice_extent,Total area of all southern-hemisphere grid cells that are covered by at least 15 % areal fraction of sea ice,seaIce,,10^6 km2,,,, +iiceprod,sigr,ice_production,Ice production,seaIce,,,,,, +iiceheco,siheco,integral_of_sea_ice_temperature_wrt_depth_expressed_as_heat_content,Sea ice heat content,seaIce,,,,,, +ibgsaltco,sisaltcga,global mean ice salt content,Global mean ice salt content,seaIce,,,,,, +iicethic:sithic,sit,sea_ice_thickness,Sea Ice Thickness,seaIce,,m,,,, +iice_hid:sithic_cat:sithicat,sitcat,ice_thicknesss_in_categories,Ice thickness in categories,seaIce,,,,,, +iicetemp,sitemp,ice_temperature,Mean ice temperature,seaIce,,K,,,, +ibgtemper,sitempga,sea_ice_temperature,Global mean sea ice temperature,seaIce,,K,,,, +iicevelo:sivelo,sivelo,ice_velocity,Ice velocity,seaIce,,,,,, +iicevelu:sivelu,sivelu,ice_velocity_u,Ice velocity u,seaIce,,,,,, +iicevelv:sivelv,sivelv,ice_velocity_v,Ice velocity v,seaIce,,,,,, +ibgvoltot,sivolga,sea_ice_volume,Global mean sea ice volume,seaIce,,,,,, +sivoln:NVolume,sivoln,sea_ice_volume,Total volume of sea ice in the northern hemisphere,seaIce,,10^3 km3,,,, +sivols:SVolume,sivols,sea_ice_volume,Total volume of sea ice in the southern hemisphere,seaIce,,10^3 km3,,,, +sivolu,sivolu,sea_ice_volume_per_unit_gridcell_area,Sea ice volume per gridcell area unit,seaIce,,,,,, +sostatl,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,,, +sostind,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,,, +sostipc,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,,, +sostpac,sltbasin,northward_ocean_salt_transport,Northward ocean salt transport,ocean,,,,,, +sopstadv,sltbasinadv,northward_ocean_salt_transport_due_to_advection,Northward ocean salt transport due to advection ,ocean,,,,,, +sopsteiv,sltbasinba,northward_ocean_salt_transport_due_to_bolus_advection,Northward ocean salt transport due to bolus advection ,ocean,,,,,, +sopstldf,sltbasindif,northward_ocean_salt_transport_due_to_diffusion,Northward ocean salt transport due to diffusion,ocean,,,,,, +sltnortha,sltnortha,northward_ocean_salt_transport,Atlantic northward ocean salt transport,ocean,,,,,, +sopstove,sltovovrt,northward_ocean_salt_transport_due_to_overturning,Northward ocean salt transport due to overturning ,ocean,,,,,, +zosalatl,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Atl,psu,,,, +zosalglo,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Glob,psu,,,, +zosalind,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Ind,psu,,,, +zosalipc,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,IndPac,psu,,,, +zosalpac,sltzmean,zonal_mean_salinity,Zonal mean salinity,ocean,Pac,psu,,,, +asn,snal,snow_albedo,Snow albedo,landIce,,,,,, +iice_hsd:snthicat,sndcat,snow_thickness_in_categories,Snow thickness in in categories,seaIce,,,,,, +isnoheco,snheco,snow_heat_content,Snow total heat content,seaIce,,,,,, +sd,snld,lwe_thickness_of_surface_snow_amount,Snow depth,atmos,,,,,, +smlt,snm,surface_snow_melt_flux,Surface snow melt,landIce,,,,,, +isnowthi,snthic,surface_snow_thickness,Surface snow thickness,seaIce,,,,,, +sbgvoltot,snvolga,snow_volume,Global mean snow volume,seaIce,,,,,, +snvolu,snvolu,snow_volume_per_unit_gridcell_area,Snow volume per gridcell area unit,seaIce,,,,,, +vosaline:mean_3Dsosaline,so,sea_water_salinity,Sea water salinity,ocean,,psu,,,, +scsaltot,soga,sea_water_salinity,Global mean sea water salinity ,ocean,,psu,,,, +soleaeiw,soleaeiw,eddy_induced_velocity_coefficient,Eddy induced vel. coeff. at w-point,ocean,,,,,, +soleahtw,soleahtw,lateral_eddy_diffusivity,Lateral eddy diffusivity,ocean,,,,,, +somixhgt,somixhgt,mixing_layer_depth_turbocline,Mixing layer depth (turbocline),ocean,,,,,, +sosaline:isssalin:mean_sosaline,sos,sea_surface_salinity,Sea surface salinity ,ocean,,psu,,,, +sothedep,sothedep,thermocline_depth,Thermocline depth (max dt/dz),ocean,,,,,, +src,src,skin_reservoir_content,Skin reservoir content,land,,,,,, +zosrfatl,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Atl,,,,, +zosrfglo,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Glob,,,,, +zosrfind,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Ind,,,,, +zosrfipc,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,IndPac,,,,, +zosrfpac,srfzmean,zonal_mean_surface,Zonal mean surface,ocean,Pac,,,,, +rsn,srho,snow_density,Snow density,landIce,,,,,, +iicesali:iice_std,ssi,sea_ice_salinity,Sea ice salinity,seaIce,,psu,,,, +salincat,ssicat,sea_ice_salinity_in_categories,Sea-ice bulk salinity for categories,seaIce,,psu,,,, +ibgsaline,ssiga,sea_ice_salinity,Global mean sea ice salinity ,seaIce,,psu,,,, +iicestre,streng,compressive_strength_of_sea_ice,Compressive sea ice strength,seaIce,,,,,, +so20chgt,t20d,depth_of_isosurface_of_sea_water_potential_temperature,,ocean,,,,,, +t,ta,air_temperature,Air temperature,atmos,,K,,,, +t2m,tas,air_temperature,Near-surface air temperature,atmos,,K,170,370,, +mx2t,tasmax,air_temperature,Daily maximum near-surface air temperature,atmos,,K,,,, +mn2t,tasmin,air_temperature,Daily minimum near-surface air temperature,atmos,,K,,,, +ewss,tauu,surface_downward_eastward_stress,Surface downward eastward wind stress,atmos,,,,,, +utau_ice:iocestru:iicestru,strairx,surface_downward_x_stress,X-Component of Atmospheric Stress On Sea Ice,seaIce,,N m-2,,,, +sozotaux,tauuo,surface_downward_x_stress,Surface downward x stress ,ocean,,,,,, +nsss,tauv,surface_downward_northward_stress,Surface downward northward wind stress,atmos,,,,,, +vtau_ice:iocestrv:iicestrv,strairy,surface_downward_y_stress,Y-Component of Atmospheric Stress On Sea Ice,seaIce,,N m-2,,,, +sozotauy:sometauy,tauvo,surface_downward_y_stress,Surface downward y stress ,ocean,,,,,, +d2m,tdps,dew_point_temperature,2m dewpoint temperature,atmos,,K,,,, +votemper:mean_3Dsosstsst,thetao,sea_water_potential_temperature,Sea water potential temperature,ocean,,K,,,, +sctemtot,thetaoga,sea_water_potential_temperature,Global average sea water potential temperature ,ocean,,K,,,, +iicesume,tmelt,tendency_of_sea_ice_amount_due_to_surface_melting,Rate of melt at upper surface of sea ice,seaIce,,,,,, +sosstsst:mean_sosstsst,tos,sea_surface_temperature,Sea surface temperature ,ocean,,K,170,370,, +sstk,tos,sea_surface_temperature,Sea surface temperature ,ocean,,K,170,370,ifs, +tossq,tossq,square_of_sea_surface_temperature,Square of sea surface temperature ,ocean,,K2,,,, +zotematl,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Atl,K,,,, +zotemglo,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Glob,K,,,, +zotemind,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Ind,K,,,, +zotemipc,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,IndPac,K,,,, +zotempac,toszmean,zonal_mean_temperature,Zonal mean temperature,ocean,Pac,K,,,, +skt,ts,surface_temperature,Surface temperature,atmos,,K,,,, +iicesurt:soicetem:sistem,tsice,surface_temperature,Surface temperature of sea ice,seaIce,,K,,,, +istl1,tsice,surface_temperature,Surface temperature of ice,landIce,,K,,,, +stl1,tsl1,soil_temperature_level_1,Temperature of soil level 1,land,,,,,, +stl2,tsl2,soil_temperature_level_2,Temperature of soil level 2,land,,,,,, +stl3,tsl3,soil_temperature_level_3,Temperature of soil level 3,land,,,,,, +stl4,tsl4,soil_temperature_level_4,Temperature of soil level 4,land,,,,,, +tsn,tsn,temperature_in_surface_snow,Snow internal temperature,landIce,,,,,, +u,ua,eastward_wind,U velocity,atmos,,,,,, +u10m,uas,eastward_wind,Eastward near-surface wind,atmos,,,,,, +vozocrtx,uo,sea_water_x_velocity,Sea water x velocity,ocean,,,,,, +uos,uos,sea_surface_x_velocity,Sea surface x velocity,ocean,,,,,, +v,va,northward_wind,V velocity,atmos,,,,,, +v10m,vas,northward_wind,Northward near-surface wind,atmos,,,,,, +vomecrty,vo,sea_water_y_velocity,Sea water y velocity,ocean,,,,,, +vos,vos,sea_surface_y_velocity,Sea surface y velocity,ocean,,,,,, +voddmavs,voddmavs,salt_vertical_eddy_diffusivity,Salt vertical eddy diffusivity,ocean,,,,,, +vozoeivu,voeivu,sea_water_x_EIV_current,Zonal eiv current,ocean,,,,,, +vomeeivv,voeivv,sea_water_y_EIV_current,Meridional eiv current,ocean,,,,,, +voveeivw,voeivz,sea_water_z_EIV_current,Vertical eiv current,ocean,,,,,, +scvoltot,volo,sea_water_volume,Sea water volume ,ocean,,,,,, +votkeavm,votkeavm,vertical_eddy_viscosity,Vertical eddy viscosity,ocean,,,,,, +votkeavt,votkeavt,vertical_eddy_diffusivity,Vertical eddy diffusivity,ocean,,,,,, +votkeevd,votkeevd,enhanced_vertical_diffusivity,Enhanced vertical diffusivity,ocean,,,,,, +votkeevm,votkeevm,enhanced_vertical_viscosity,Enhanced vertical viscosity,ocean,,,,,, +sobarstf,vsftbarot,ocean_barotropic_volume_streamfunction,Ocean barotropic volume streamfunction ,ocean,,,,,, +zomsfatl,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Atl,,,,, +zomsfglo,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Glob,,,,, +zomsfind,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Ind,,,,, +zomsfipc:zomsfinp,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,IndPac,,,,, +zomsfpac,vsftmyz,ocean_meridional_overturning_volume_streamfunction,Ocean meridional overturning volume streamfunction ,ocean,Pac,,,,, +zomsfeiv,vsftmyzba,ocean_meridional_overturning_mass_streamfunction_due_to_bolus_advection,Ocean meridional overturning volume streamfunction due to bolus advection ,ocean,,,,,, +w,wa,vertical_velocity,Vertical velocity,atmos,,,,,, +z,zg,geopotential_height,Geopotential height,atmos,,,,,, +vovecrtz,zo,sea_water_z_velocity,Sea water z velocity,ocean,,,,,, +sossheigh:sossheig:mean_sossheig,zos,sea_surface_height_above_geoid,Sea surface height above geoid ,ocean,,,,,, +scsshtot,zosga,global_average_sea_level_change,Global average sea level change ,ocean,,,,,, +scsshste,zossga,global_average_steric_sea_level_change,Global average steric sea level change ,ocean,,,,,, +zossq,zossq,square_of_sea_surface_height_above_geoid,Square of sea surface height above geoid ,ocean,,,,,, +scsshtst,zostoga,snthic,Global average thermosteric sea level change ,ocean,,,,,, +heatcsum,heatcsum,total_ocean_heat_content,Total Ocean heat content,ocean,,J,,,, +heatcvmean,heatcvmean,average_ocean_heat_content,Average Ocean heat content,ocean,,J m-3,,,, +transix,transix,sea_ice_x_transport,X-Component of Sea Ice Mass Transport,seaIce,,kg s-1,,,, +transiy,transiy,sea_ice_y_transport,Y-Component of Sea Ice Mass Transport,seaIce,,kg s-1,,,, +windsp,sfcWind,wind_speed,Near-Surface Wind Speed,atmos,,,,,, +vsfsit,vsfsit,virtual_salt_flux_into_sea_water_due_to_sea_ice_thermodynamics,Virtual Salt Flux into Sea Water due to Sea Ice Thermodynamics ,ocean,,,,,, +sfdsi,sfdsi,downward_sea_ice_basal_salt_flux,Downward Sea Ice Basal Salt Flux,ocean,,,,,, +hfsithermds,hfsithermds,heat_flux_into_sea_water_due_to_sea_ice_thermodynamics,Heat Flux into Sea Water due to Sea Ice Thermodynamics ,ocean,,,,,, +u2o,uosq,square_of_sea_water_x_velocity,Square of Sea Water X Velocity ,ocean,,,,,, +v2o,vosq,square_of_sea_water_y_velocity,Square of Sea Water Y Velocity ,ocean,,,,,, +vozomatr,umo,ocean_mass_x_transport,Ocean Mass X Transport ,ocean,,,,,, +vomematr,vmo,ocean_mass_y_transport,Ocean Mass Y Transport ,ocean,,,,,, +sozohetr,hfx,ocean_heat_x_transport,Ocean Heat X Transport ,ocean,,,,,, +somehetr,hfy,ocean_heat_y_transport,Ocean Heat Y Transport ,ocean,,,,,, +uto,uothetao,product_of_xward_sea_water_velocity_and_temperature,Product of X-ward Sea Water Velocity and Temperature,ocean,,,,,, +vto,vothetao,product_of_yward_sea_water_velocity_and_temperature,Product of Y-ward Sea Water Velocity and Temperature,ocean,,,,,, +uso,uoso,product_of_xward_sea_water_velocity_and_salinity,Product of X-ward Sea Water Velocity and Salinity,ocean,,,,,, +vso,voso,product_of_yward_sea_water_velocity_and_salinity,Product of Y-ward Sea Water Velocity and Salinity,ocean,,,,,, +wfo,wfo,water_flux_into_sea_water,Water Flux into Sea Water ,ocean,,,,,, +emp_oce,evsmpr,evap_minus_precip_over_sea_water,Evap minus Precip over ocean,ocean,,,,,, +emp_ice,evsmpr,evap_minus_precip_over_sea_ice,Evap minus Precip over ice,seaIce,,,,,, +qsr_oce,rsntds,net_downward_shortwave_flux_at_sea_water_surface,Net Downward Shortwave Radiation at Sea Water Surface ,ocean,,,,,, +qns_oce,rlds,surface_net_downward_longwave_flux,Surface Net Downward Longwave Radiation,ocean,,,,,, +qsr_ice,rsdssi,surface_downwelling_shortwave_flux_in_air,Downwelling Shortwave over Sea Ice,seaIce,,,,,, +qns_ice,rldssi,surface_downwelling_longwave_flux_in_air,Downwelling Long Wave over Sea Ice,seaIce,,,,,, +sfx,sfx,downward_salt_flux,Downward Salt Flux,ocean,,,,,, +taum,taum,surface_downward_stress_module,Surface Downward Stress Module,ocean,,,,,, +zfull,zfull,depth_below_geoid,Depth Below Geoid of Ocean Layer,ocean,,,,,, +zhalf,zhalf,depth_below_geoid,Depth Below Geoid of Ocean Layer,ocean,,,,,, +pbo,pbo,sea_water_pressure_at_sea_floor,Sea Water Pressure at Sea Floor,ocean,,,,,, +thkcello,thkcello,cell_thickness,Cell Thickness,ocean,,,,,, +ficeberg,ficeberg,water_flux_into_sea_water_from_icebergs,Water Flux into Sea Water From Icebergs ,ocean,,,,,, +rsdo,rsds,downwelling_shortwave_flux_in_sea_water,Downwelling Shortwave Radiation in Sea Water ,ocean,,,,,, +wo,wo,sea_water_upward_velocity,Sea Water Upward Velocity ,ocean,,,,,, +w2o,wosq,square_of_sea_water_upward_velocity,Square of Sea Water Upward Velocity ,ocean,,,,,, +difvho,difvho,ocean_vertical_heat_diffusivity,Ocean Vertical Heat Diffusivity,ocean,,,,,, +vovematr,wmo,upward_ocean_mass_transport,Upward Ocean Mass Transport ,ocean,,,,,, +qtr_ice,qtr,shortwave_flux_transmitted_through_ice,Shortwave Flux Transmitted Through The Ice,seaIce,,,,,, diff --git a/earthdiagnostics/cmor_tables/primavera.xlsx b/earthdiagnostics/cmor_tables/primavera.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..cd537007ea823e4d846766b6fe2ccf42f2d5afb3 Binary files /dev/null and b/earthdiagnostics/cmor_tables/primavera.xlsx differ diff --git a/earthdiagnostics/cmor_tables/specs.csv b/earthdiagnostics/cmor_tables/specs.csv new file mode 100644 index 0000000000000000000000000000000000000000..90f01d9150608f127fd1de1aa6b2b5e0f14c6f84 --- /dev/null +++ b/earthdiagnostics/cmor_tables/specs.csv @@ -0,0 +1 @@ +Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index fa90b12dd3a2b548e24ba3497e248e5a8b056fa4..9ed85f87ed304c0bd5e89004a6a6d222b1f85de9 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -10,7 +10,8 @@ import pygrib from autosubmit.config.log import Log from autosubmit.date.chunk_date_lib import parse_date, chunk_end_date, previous_day, date2str, add_months -from earthdiagnostics.variable import Variable, Domains +from earthdiagnostics.frequency import Frequency, Frequencies +from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import TempFile, Utils @@ -19,7 +20,7 @@ class Cmorizer(object): Class to manage CMORization :param data_manager: experiment's data manager - :type data_manager: DataManager + :type data_manager: CMORManager :param startdate: startdate to cmorize :type startdate: str :param member: member to cmorize @@ -50,7 +51,7 @@ class Cmorizer(object): self.original_files_path = os.path.join(self.config.data_dir, self.experiment.expid, 'original_files', self.startdate, self.member_str, 'outputs') self.atmos_timestep = None - self.cmor_scratch = os.path.join(self.config.scratch_dir, 'CMOR') + self.cmor_scratch = os.path.join(self.config.scratch_dir, 'CMOR', self.startdate, self.member_str) def cmorize_ocean(self): """ @@ -58,7 +59,9 @@ class Cmorizer(object): :return: """ if not self.cmor.ocean: + Log.info('Skipping ocean cmorization due to configuration') return + Log.info('\nCMORizing ocean\n') self._cmorize_ocean_files('MMO') self._cmorize_ocean_files('PPO') self._cmorize_ocean_files('diags') @@ -69,10 +72,18 @@ class Cmorizer(object): tar_files.sort() count = 1 for tarfile in tar_files: + if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), ModelingRealms.ocean): + Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) + count += 1 + continue + Log.info('Unpacking oceanic file {0}/{1}'.format(count, len(tar_files))) - self._unpack_tar_file(tarfile) - self._cmorize_nc_files() - Log.result('Oceanic file {0}/{1} finished'.format(count, len(tar_files))) + try: + self._unpack_tar_file(tarfile) + self._cmorize_nc_files() + Log.result('Oceanic file {0}/{1} finished'.format(count, len(tar_files))) + except Exception as ex: + Log.error('Could not CMORize oceanic file {0}: {1}', count, ex) count += 1 def _cmorize_nc_files(self): @@ -80,12 +91,12 @@ class Cmorizer(object): self._cmorize_nc_file(filename) def _correct_fluxes(self): - fluxes_vars = ("prsn", "rss", "rls", "rsscs", "rsds", "rlds") + fluxes_vars = ("prsn", "rss", "rls", "rsscs", "rsds", "rlds", "hfss", 'hfls') for filename in glob.glob(os.path.join(self.cmor_scratch, '*.nc')): handler = Utils.openCdf(filename) for varname in handler.variables.keys(): - cmor_var = Variable.get_variable(varname, True) - if cmor_var.short_name not in fluxes_vars: + cmor_var = self.data_manager.variable_list.get_variable(varname, True) + if cmor_var is None or cmor_var.short_name not in fluxes_vars: continue handler.variables[varname][:] = handler.variables[varname][:] / self.experiment.atmos_timestep * 3600 handler.close() @@ -99,21 +110,24 @@ class Cmorizer(object): def _merge_mma_files(self, tarfile): temp = TempFile.get() - for filename in glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_SH_*.nc')): - Utils.cdo.sp2gpl(options='-O', input=filename, output=temp) - shutil.move(temp, filename) sh_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_SH_*.nc')) - Utils.cdo.mergetime(input=sh_files, output=os.path.join(self.cmor_scratch, 'sh.nc')) gg_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_GG_*.nc')) - Utils.cdo.mergetime(input=gg_files, output=os.path.join(self.cmor_scratch, 'gg.nc')) + + merged_sh = TempFile.get() + merged_gg = TempFile.get() + + for filename in sh_files: + Utils.cdo.sp2gpl(options='-O', input=filename, output=temp) + shutil.move(temp, filename) + Utils.cdo.mergetime(input=sh_files, output=merged_sh) + Utils.cdo.mergetime(input=gg_files, output=merged_gg) for filename in sh_files + gg_files: os.remove(filename) - Utils.nco.ncks(input=os.path.join(self.cmor_scratch, 'sh.nc'), - output=os.path.join(self.cmor_scratch, 'gg.nc'), options='-A') - os.remove(os.path.join(self.cmor_scratch, 'sh.nc')) + Utils.nco.ncks(input=merged_sh, output=merged_gg, options='-A') + os.remove(merged_sh) tar_startdate = tarfile[0:-4].split('_')[5].split('-') new_name = 'MMA_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate) - shutil.move(os.path.join(self.cmor_scratch, 'gg.nc'), os.path.join(self.cmor_scratch, new_name)) + shutil.move(merged_gg, os.path.join(self.cmor_scratch, new_name)) def cmorize_atmos(self): """ @@ -121,8 +135,10 @@ class Cmorizer(object): :return: """ if not self.cmor.atmosphere: + Log.info('Skipping atmosphere cmorization due to configuration') return + Log.info('\nCMORizing atmosphere\n') if self.cmor.use_grib and self.gribfiles_available(): self._cmorize_grib_files() else: @@ -133,33 +149,47 @@ class Cmorizer(object): tar_files.sort() count = 1 for tarfile in tar_files: + if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), ModelingRealms.atmos): + Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) + count += 1 + continue Log.info('Unpacking atmospheric file {0}/{1}'.format(count, len(tar_files))) - self._unpack_tar_file(tarfile) - self._merge_mma_files(tarfile) - self._correct_fluxes() - self._cmorize_nc_files() - Log.result('Atmospheric file {0}/{1} finished'.format(count, len(tar_files))) + try: + self._unpack_tar_file(tarfile) + self._merge_mma_files(tarfile) + self._correct_fluxes() + self._cmorize_nc_files() + Log.result('Atmospheric file {0}/{1} finished'.format(count, len(tar_files))) + except Exception as ex: + Log.error('Could not cmorize atmospheric file {0}: {1}', count, ex) + count += 1 def _cmorize_grib_files(self): - count = 1 + chunk = 1 chunk_start = parse_date(self.startdate) while os.path.exists(self.get_original_grib_path(chunk_start, 'GG')) or \ os.path.exists(self.get_original_grib_path(chunk_start, 'SH')): - chunk_end = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', 'standard') - chunk_end = previous_day(chunk_end, 'standard') - Log.info('CMORizing chunk {0}-{1}', date2str(chunk_start), date2str(chunk_end)) - for grid in ('SH', 'GG'): - Log.info('Processing {0} variables', grid) - - if not os.path.exists(self.get_original_grib_path(chunk_start, grid)): - continue - self.cmorize_grib_file(chunk_end, chunk_start, count, grid) + if self.cmorization_required(chunk, ModelingRealms.atmos): + chunk_end = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', 'standard') + chunk_end = previous_day(chunk_end, 'standard') + Log.info('CMORizing chunk {0}-{1}', date2str(chunk_start), date2str(chunk_end)) + try: + for grid in ('SH', 'GG'): + Log.info('Processing {0} variables', grid) + + if not os.path.exists(self.get_original_grib_path(chunk_start, grid)): + continue + self.cmorize_grib_file(chunk_end, chunk_start, grid) + except Exception as ex: + Log.error('Can not cmorize GRIB file for chunk {0}-{1}: {2}', + date2str(chunk_start), date2str(chunk_end), ex) chunk_start = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', 'standard') + chunk += 1 - def cmorize_grib_file(self, chunk_end, chunk_start, count, grid): + def cmorize_grib_file(self, chunk_end, chunk_start, grid): for month in range(0, self.experiment.chunk_size): current_date = add_months(chunk_start, month, 'standard') original_gribfile = self.get_original_grib_path(current_date, grid) @@ -197,17 +227,17 @@ class Cmorizer(object): cdo_reftime = parse_date(self.startdate).strftime('%Y-%m-%d,00:00') - self._ungrib_vars(cdo_reftime, gribfile, current_date.month, '{0}hr'.format(self.atmos_timestep)) - self._ungrib_vars(cdo_reftime, gribfile, current_date.month, '1d') - self._ungrib_vars(cdo_reftime, gribfile, current_date.month, '1m') + self._ungrib_vars(cdo_reftime, gribfile, current_date.month, Frequency('{0}hr'.format(self.atmos_timestep))) + self._ungrib_vars(cdo_reftime, gribfile, current_date.month, Frequencies.daily) + self._ungrib_vars(cdo_reftime, gribfile, current_date.month, Frequencies.monthly) for splited_file in glob.glob('{0}_*.128.nc'.format(gribfile)): os.remove(splited_file) Log.result('Month {0}, {1} variables finished', date2str(current_date), grid) - count += 1 - self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, '1m') - self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, '1d') + + self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, Frequencies.monthly) + self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, Frequencies.daily) self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, '{0}hr'.format(self.atmos_timestep)) @@ -262,6 +292,7 @@ class Cmorizer(object): handler.close() os.remove(filename) + # noinspection PyMethodMayBeStatic def _remove_valid_limits(self, filename): handler = Utils.openCdf(filename) for variable in handler.variables.keys(): @@ -275,15 +306,15 @@ class Cmorizer(object): def _get_nc_file_frequency(self, filename): file_parts = os.path.basename(filename).split('_') if self.experiment.expid in [file_parts[1], file_parts[2]]: - frequency = 'm' + frequency = Frequency('m') elif self.experiment.expid == file_parts[0]: try: parse_date(file_parts[1]) - frequency = 'm' + frequency = Frequency('m') except ValueError: - frequency = file_parts[1][1].lower() + frequency = Frequency(file_parts[1][1]) else: - frequency = file_parts[1][1].lower() + frequency = Frequency(file_parts[1][1]) return frequency def _contains_requested_variables(self, filename): @@ -297,28 +328,28 @@ class Cmorizer(object): :param file_path: path to the file :type file_path: str :param handler: netCDF4 handler for the file - :type handler: netCDF$.Dataset + :type handler: netCDF4.Dataset :param frequency: variable's frequency - :type frequency: str + :type frequency: Frequency :param variable: variable's name :type variable: str """ temp = TempFile.get() - var_cmor = Variable.get_variable(variable) + alias, var_cmor = self.data_manager.variable_list.get_variable_and_alias(variable) if var_cmor is None: return if not self.cmor.cmorize(var_cmor): return - frequency = self.translate_frequency(frequency) + frequency = Frequency.parse(frequency) Utils.nco.ncks(input=file_path, output=temp, options='-v {0}'.format(variable)) self._rename_level_variables(temp, var_cmor) self._add_coordinate_variables(handler, temp) - if var_cmor.basin is None: + if alias.basin is None: region = None else: - region = var_cmor.basin.fullname + region = alias.basin.fullname date_str = self.get_date_str(file_path) if date_str is None: @@ -329,19 +360,35 @@ class Cmorizer(object): self.data_manager.send_file(temp, var_cmor.domain, var_cmor.short_name, self.startdate, self.member, frequency=frequency, rename_var=variable, date_str=date_str, region=region, - move_old=True, grid=var_cmor.grid, cmorized=True) + move_old=True, grid=alias.grid, cmorized=True) def get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') if file_parts[0] in (self.experiment.expid, 'MMA', 'MMO') or file_parts[0].startswith('ORCA'): # Model output - return '{0}-{1}'.format(file_parts[2][0:6], file_parts[3][0:6]) + if file_parts[-1].endswith('.tar'): + file_parts = file_parts[-1][0:-4].split('-') + return '{0}-{1}'.format(file_parts[0][0:6], file_parts[1][0:6]) + else: + return '{0}-{1}'.format(file_parts[2][0:6], file_parts[3][0:6]) elif file_parts[1] == self.experiment.expid: # Files generated by the old version of the diagnostics return '{0}-{1}'.format(file_parts[4][0:6], file_parts[5][0:6]) else: return None + def get_chunk(self, file_path): + chunk_start = parse_date(self.get_date_str(file_path).split('-')[0]) + current_date = parse_date(self.startdate) + chunk = 1 + while current_date < chunk_start: + current_date = chunk_end_date(current_date, self.experiment.chunk_size, 'month', 'standard') + chunk += 1 + + if current_date != chunk_start: + raise Exception('File {0} start date is not a valid chunk start date'.format(file_path)) + return chunk + @staticmethod def _add_coordinate_variables(handler, temp): handler_cmor = Utils.openCdf(temp) @@ -353,26 +400,20 @@ class Cmorizer(object): @staticmethod def _rename_level_variables(temp, var_cmor): - if var_cmor.domain == Domains.ocean: + if var_cmor.domain == ModelingRealms.ocean: Utils.rename_variables(temp, {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', 'depth': 'lev'}, False, True) - if var_cmor.domain in [Domains.landIce, Domains.land]: + if var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: Utils.rename_variables(temp, {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', 'depth_4': 'sdepth'}, False, True) - if var_cmor.domain == Domains.atmos: + if var_cmor.domain == ModelingRealms.atmos: Utils.rename_variables(temp, {'depth': 'plev'}, False, True) @staticmethod def translate_frequency(frequency): - if frequency == 'd': - frequency = 'day' - elif frequency == 'm': - frequency = 'mon' - elif frequency == 'h': + if frequency == 'h': frequency = '6hr' - else: - raise Exception('Frequency {0} not supported'.format(frequency)) - return frequency + return Frequency(frequency) @staticmethod def _merge_grib_files(current_month, prev_gribfile, gribfile): @@ -398,14 +439,14 @@ class Cmorizer(object): new_units = None cdo_operator = '-selmon,{0}'.format(month) - if frequency in ('month', 'monthly', 'mon', '1m'): + if frequency == Frequencies.monthly: if var_code == 201: cdo_operator = "-monmean -daymax {0}".format(cdo_operator) elif var_code == 202: cdo_operator = "-monmean -daymax {0}".format(cdo_operator) else: cdo_operator = "-monmean {0} ".format(cdo_operator) - if frequency in ('day', 'daily', '1d'): + if frequency == Frequencies.daily: if var_code == 201: cdo_operator = "-daymax {0} ".format(cdo_operator) elif var_code == 202: @@ -500,7 +541,8 @@ class Cmorizer(object): var.standard_name = "forecast_period" leadtime = Utils.get_datetime_from_netcdf(handler) startdate = parse_date(self.startdate) - leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate for time in leadtime] + leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate + for time in leadtime] for lt in range(0, len(leadtime)): var[lt] = leadtime[lt].days handler.close() @@ -517,10 +559,7 @@ class Cmorizer(object): handler.creation_date = datetime.now().strftime('%Y-%m-%d(T%H:%M:%SZ)') handler.experiment_id = experiment.experiment_name handler.forecast_reference_time = parse_date(self.startdate).strftime('%Y-%m-%d(T%H:%M:%SZ)') - if frequency == 'd': - handler.frequency = 'day' - elif frequency == 'm': - handler.frequency = 'mon' + handler.frequency = frequency.frequency handler.institute_id = experiment.institute handler.institution = experiment.institute handler.initialization_method = cmor.initialization_method @@ -529,12 +568,13 @@ class Cmorizer(object): handler.physics_description = cmor.physics_description handler.model_id = experiment.model handler.associated_model = cmor.associated_model - handler.project_id = 'SPECS' + handler.project_id = self.config.data_convention.upper() handler.realization = str(self.member + 1) handler.source = cmor.source handler.startdate = 'S{0}'.format(self.startdate) handler.tracking_id = str(uuid.uuid1()) - handler.title = "{0} model output prepared for SPECS {1}".format(experiment.model, experiment.experiment_name) + handler.title = "{0} model output prepared for {2} {1}".format(experiment.model, experiment.experiment_name, + self.config.data_convention.upper()) handler.close() def gribfiles_available(self): @@ -542,6 +582,9 @@ class Cmorizer(object): gribfiles = glob.glob(grb_path) return len(gribfiles) > 0 + def cmorization_required(self, chunk, domain): + return self.config.cmor.force or not self.data_manager.is_cmorized(self.startdate, self.member, chunk, domain) + class CMORException(Exception): pass diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 785b515ecc886bb226a07b5e4247674b6102ae0e..86259cc40e068aecd45608bd16894bde7cb533d8 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -1,5 +1,6 @@ # coding=utf-8 import glob +import shutil from datetime import datetime import os @@ -8,8 +9,10 @@ from autosubmit.date.chunk_date_lib import parse_date, chunk_start_date, chunk_e from earthdiagnostics.cmorizer import Cmorizer from earthdiagnostics.datamanager import DataManager, NetCDFFile +from earthdiagnostics.frequency import Frequencies, Frequency +from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import TempFile, Utils -from earthdiagnostics.variable import Variable, VarType +from earthdiagnostics.variable_type import VariableType class CMORManager(DataManager): @@ -18,19 +21,39 @@ class CMORManager(DataManager): """ def __init__(self, config): super(CMORManager, self).__init__(config) + self._dic_cmorized = dict() data_folders = self.config.data_dir.split(':') + experiment_folder = self.experiment.model.lower() + if experiment_folder.startswith('ec-earth'): + experiment_folder = 'ecearth' + self.config.data_dir = None for data_folder in data_folders: if os.path.isdir(os.path.join(data_folder, self.experiment.expid)): self.config.data_dir = data_folder break + data_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) + if os.path.isdir(os.path.join(data_folder, self.experiment.expid)): + self.config.data_dir = data_folder + break + if not self.config.data_dir: raise Exception('Can not find model data') self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') + def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + filepath = self.get_file_path(startdate, member, domain, var, chunk, frequency, box, grid, None, None) + + # noinspection PyBroadException + try: + return os.path.isfile(filepath) + except Exception: + return False + def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VarType.MEAN): + vartype=VariableType.MEAN): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy @@ -49,7 +72,9 @@ class CMORManager(DataManager): :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str + :type frequency: Frequency|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType :return: path to the copy created on the scratch folder :rtype: str """ @@ -70,52 +95,79 @@ class CMORManager(DataManager): :param domain: file's domain :type domain: Domain :param var: file's var - :type var: str + :type var: var :param chunk: file's chunk :type chunk: int :param frequency: file's frequency - :type frequency: str + :type frequency: Frequency :param box: file's box :type box: Box :param grid: file's grid :type grid: str|NoneType :param year: file's year - :type year: int|str + :type year: int|str|NoneType :param date_str: date string to add directly. Overrides year or chunk configurations - :type date_str: str + :type date_str: str|NoneType :return: path to the file - :rtype: str + :rtype: str|NoneType """ if not frequency: frequency = self.config.frequency var = self._get_final_var_name(box, var) folder_path = self._get_full_cmor_folder_path(startdate, member, domain, var, frequency, grid) - file_name = self._get_cmor_file_name(startdate, member, domain, var, frequency, chunk, year, date_str) + file_name = self._get_cmor_file_name(startdate, member, domain, var, frequency, chunk, year, date_str, grid) filepath = os.path.join(folder_path, file_name) return filepath - def _get_cmor_file_name(self, startdate, member, domain, var, frequency, chunk, year, date_str): - domain_abreviattion = domain.get_table_name(frequency) + def _get_cmor_file_name(self, startdate, member, domain, var, frequency, chunk, year, date_str, grid): + cmor_var = self.variable_list.get_variable(var) + if cmor_var is None: + cmor_table = domain.get_table(frequency, self.config.data_convention) + else: + cmor_table = cmor_var.get_table(frequency, self.config.data_convention) + if chunk is not None: time_bound = self._get_chunk_time_bounds(startdate, chunk) elif year: - if frequency is not 'yr': + if frequency != Frequencies.yearly: raise ValueError('Year may be provided instead of chunk only if frequency is "yr"') time_bound = str(year) elif date_str: time_bound = date_str else: raise ValueError('Chunk, year and date_str can not be None at the same time') - file_name = '{0}_{1}_{2}_{3}_S{4}_r{5}i1p1_{6}.nc'.format(var, domain_abreviattion, self.experiment.model, - self.experiment.experiment_name, startdate, + + if time_bound: + time_bound = '_{0}.nc'.format(time_bound) + else: + time_bound = '.nc' + + if self.config.data_convention == 'specs': + + file_name = '{0}_{1}_{2}_{3}_S{4}_r{5}i1p1{6}'.format(var, + cmor_table.name, + self.experiment.model, + self.experiment.experiment_name, + startdate, member + 1, time_bound) + elif self.config.data_convention in ('primavera', 'cmip6'): + file_name = '{0}_{1}_{2}_{3}_S{4}-r{5}i1p1_{6}{7}'.format(var, + cmor_table.name, + self.experiment.experiment_name, + self.experiment.model, + startdate, + member + 1, + grid, + time_bound) + else: + raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name def _get_full_cmor_folder_path(self, startdate, member, domain, var, frequency, grid): - folder_path = os.path.join(self._get_startdate_path(startdate), frequency, domain.name, var) + folder_path = os.path.join(self._get_startdate_path(startdate), str(frequency), domain.name, var) if grid: folder_path = os.path.join(folder_path, grid) folder_path = os.path.join(folder_path, 'r{0}i1p1'.format(member + 1)) @@ -131,7 +183,7 @@ class CMORManager(DataManager): return time_bound def link_file(self, domain, var, startdate, member, chunk=None, grid=None, box=None, - frequency=None, year=None, date_str=None, move_old=False, vartype=VarType.MEAN): + frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ Creates the link of a given file from the CMOR repository. @@ -154,7 +206,9 @@ class CMORManager(DataManager): :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType :return: path to the copy created on the scratch folder :rtype: str """ @@ -168,7 +222,7 @@ class CMORManager(DataManager): def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False, vartype=VarType.MEAN): + diagnostic=None, cmorized=False, vartype=VariableType.MEAN): """ Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge with already existing ones as needed @@ -201,14 +255,16 @@ class CMORManager(DataManager): :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str + :type frequency: Frequency :param diagnostic: diagnostic used to generate the file :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType """ original_var = var - cmor_var = Variable.get_variable(var) + cmor_var = self.variable_list.get_variable(original_var) var = self._get_final_var_name(box, var) if rename_var and rename_var != var: @@ -221,7 +277,8 @@ class CMORManager(DataManager): filepath = self.get_file_path(startdate, member, domain, var, chunk, frequency, None, grid, year, date_str) - netcdf_file = NetCDFFile(filepath, filetosend, domain, var, cmor_var) + netcdf_file = NetCDFFile(filepath, filetosend, domain, var, cmor_var, self.config.data_convention, region) + netcdf_file.frequency = frequency if diagnostic: netcdf_file.add_diagnostic_history(diagnostic) elif cmorized: @@ -294,22 +351,26 @@ class CMORManager(DataManager): # Check if cmorized and convert if not for startdate, member in self.experiment.get_member_list(): - if not self.config.cmor.force and not self.config.cmor.force_untar and self._is_cmorized(startdate, member): - continue if not self._unpack_cmor_files(startdate, member): self._cmorize_member(startdate, member) - def _is_cmorized(self, startdate, member): + def is_cmorized(self, startdate, member, chunk, domain): + identifier = (startdate, member, chunk, domain.name) + if identifier not in self._dic_cmorized: + self._dic_cmorized[identifier] = self._is_cmorized(startdate, member, chunk, domain) + return self._dic_cmorized[identifier] + + def _is_cmorized(self, startdate, member, chunk, domain): startdate_path = self._get_startdate_path(startdate) - if not os.path.exists(startdate_path): + if not os.path.isdir(startdate_path): return False for freq in os.listdir(startdate_path): - freq_path = os.path.join(startdate_path, freq) - for domain in os.listdir(freq_path): - domain_path = os.path.join(freq_path, domain) + domain_path = os.path.join(startdate_path, freq, + domain.name) + if os.path.isdir(domain_path): for var in os.listdir(domain_path): - member_path = os.path.join(domain_path, var, 'r{0}i1p1'.format(member + 1)) - if os.path.exists(member_path): + var_path = self.get_file_path(startdate, member, domain, var, chunk, Frequency(freq)) + if os.path.isfile(var_path): return True return False @@ -320,34 +381,53 @@ class CMORManager(DataManager): cmorizer = Cmorizer(self, startdate, member) cmorizer.cmorize_ocean() cmorizer.cmorize_atmos() - Log.result('CMORized startdate {0} member {1}!\n\n', startdate, member_str, datetime.now() - start_time) + Log.result('CMORized startdate {0} member {1}! Ellpased time: {2}\n\n', startdate, member_str, + datetime.now() - start_time) def _unpack_cmor_files(self, startdate, member): if self.config.cmor.force: return False - filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, 'tar.gz') + chunk = 1 + cmorized = False + + if not self.config.cmor.force_untar: + while self.is_cmorized(startdate, member, chunk, ModelingRealms.ocean) or\ + self.is_cmorized(startdate, member, chunk, ModelingRealms.atmos): + chunk += 1 + + while self._unpack_chunk(startdate, member, chunk): + chunk += 1 + cmorized = True + + return cmorized + + def _unpack_chunk(self, startdate, member, chunk): + + filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar.gz') if len(filepaths) > 0: - Log.info('Unzipping cmorized data...') + Log.info('Unzipping cmorized data for {0} {1} {2}...', startdate, member, chunk) Utils.unzip(filepaths, True) if not os.path.exists(self.cmor_path): os.mkdir(self.cmor_path) - filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, 'tar') + filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar') if len(filepaths) > 0: - Log.info('Unpacking cmorized data...') + Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) Utils.untar(filepaths, self.cmor_path) self._correct_paths(startdate) self._create_links(startdate) return True return False - def _get_transferred_cmor_data_filepaths(self, startdate, member, extension): + def _get_transferred_cmor_data_filepaths(self, startdate, member, chunk, extension): tar_path = os.path.join(self.config.data_dir, self.experiment.expid, 'original_files', 'cmorfiles') tar_original_files = os.path.join(self.config.data_dir, 'original_files', self.experiment.expid, 'cmorfiles') - file_name = 'CMOR?_{0}_{1}_{2}_*.{3}'.format(self.experiment.expid, startdate, - self.experiment.get_member_str(member), extension) + file_name = 'CMOR?_{0}_{1}_{2}_{3}-*.{4}'.format(self.experiment.expid, startdate, + self.experiment.get_member_str(member), + self.experiment.get_chunk_start_str(startdate, chunk), + extension) filepaths = glob.glob(os.path.join(tar_path, file_name)) filepaths += glob.glob(os.path.join(tar_path, 'outputs', file_name)) filepaths += glob.glob(os.path.join(tar_original_files, file_name)) @@ -381,37 +461,55 @@ class CMORManager(DataManager): Log.debug('Done') def _remove_extra_output_folder(self): - bad_path = os.path.join(self.cmor_path, 'output', self.experiment.institute) + bad_path = os.path.join(self.cmor_path, 'output') if os.path.exists(bad_path): Log.debug('Moving CMOR files out of the output folder') - Utils.execute_shell_command(['mv', bad_path, os.path.join(bad_path, '..', '..')]) - os.rmdir(os.path.join(self.cmor_path, 'output')) + CMORManager.copytree(bad_path, self.cmor_path) + shutil.rmtree(bad_path) Log.debug('Done') + @staticmethod + def copytree(source, destiny): + if not os.path.exists(destiny): + os.makedirs(destiny) + shutil.copystat(source, destiny) + lst = os.listdir(source) + for item in lst: + item_source = os.path.join(source, item) + item_destiny = os.path.join(destiny, item) + if os.path.isdir(item_source): + CMORManager.copytree(item_source, item_destiny) + else: + shutil.copy2(item_source, item_destiny) + def _create_links(self, startdate): - Log.info('Creating links for CMOR files ()') + Log.info('Creating links for CMOR files ({0})', startdate) path = self._get_startdate_path(startdate) for freq in os.listdir(path): + frequency = Frequency.parse(freq) for domain in os.listdir(os.path.join(path, freq)): for var in os.listdir(os.path.join(path, freq, domain)): for member in os.listdir(os.path.join(path, freq, domain, var)): for name in os.listdir(os.path.join(path, freq, domain, var, member)): filepath = os.path.join(path, freq, domain, var, member, name) if os.path.isfile(filepath): - self._create_link(domain, filepath, freq, var, "", False, vartype=VarType.MEAN) + self._create_link(domain, filepath, frequency, var, "", False, + vartype=VariableType.MEAN) else: for filename in os.listdir(filepath): - self._create_link(domain, os.path.join(filepath, filename), freq, var, "", False, - vartype=VarType.MEAN) - Log.info('Creating lings for CMOR files') + self._create_link(domain, os.path.join(filepath, filename), frequency, var, "", + False, vartype=VariableType.MEAN) + Log.debug('Links ready') def _get_startdate_path(self, startdate): """ Returns the path to the startdate's CMOR folder :param startdate: target startdate :type startdate: str - :return: path to the startdate's CMOR folder + :return: path to the startdate's CMOR º :rtype: str """ return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.experiment.institute, self.experiment.model, self.experiment.experiment_name, 'S' + startdate) + + diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index ab206c12ab8d8953939ff066950c3d3ccf4c0d31..2b9c736f963c9105f7fa2c229db0f016ff06a936 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -2,10 +2,11 @@ import os from autosubmit.config.log import Log -from autosubmit.date.chunk_date_lib import parse_date, chunk_start_date, chunk_end_date +from autosubmit.date.chunk_date_lib import parse_date, chunk_start_date, chunk_end_date, date2str +from earthdiagnostics.frequency import Frequency, Frequencies from earthdiagnostics.parser import Parser -from earthdiagnostics.variable import Variable +from earthdiagnostics.variable import VariableManager from utils import Utils @@ -35,11 +36,11 @@ class Config(object): raise Exception('Data type must be exp, obs or recon') self.con_files = Utils.expand_path(parser.get_option('DIAGNOSTICS', 'CON_FILES')) "Mask and meshes folder path" + self.data_convention = parser.get_option('DIAGNOSTICS', 'DATA_CONVENTION', 'SPECS').lower() self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') - self.frequency = parser.get_option('DIAGNOSTICS', 'FREQUENCY').lower() + self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY')) "Default data frequency to be used by the diagnostics" - if self.frequency == 'month': - self.frequency = 'mon' + self.cdftools_path = Utils.expand_path(parser.get_option('DIAGNOSTICS', 'CDFTOOLS_PATH')) "Path to CDFTOOLS executables" self.max_cores = parser.get_int_option('DIAGNOSTICS', 'MAX_CORES', 100000) @@ -134,7 +135,7 @@ class CMORConfig(object): if self._variable_list is None: return True for var in variables: - if self.cmorize(Variable.get_variable(var, silent=True)): + if self.cmorize(VariableManager().get_variable(var, silent=True)): return True return False @@ -166,11 +167,11 @@ class CMORConfig(object): return range(start, end, step) def get_variables(self, frequency): - if frequency in ('hour', 'hourly') or frequency[1:] == 'hr': + if frequency in (Frequencies.three_hourly, Frequencies.six_hourly): return self._var_hourly - elif frequency in ('day', 'daily', '1d'): + elif frequency == Frequencies.daily: return self._var_daily - elif frequency in ('month', 'monthly', 'mon', '1m'): + elif frequency == Frequencies.monthly: return self._var_monthly raise Exception('Frequency not recognized: {0}'.format(frequency)) @@ -256,7 +257,7 @@ class ExperimentConfig(object): date = parse_date(startdate) chunks = list() for chunk in range(1, self.num_chunks + 1): - chunk_start = chunk_start_date(date, chunk, self.chunk_size, 'month', self.calendar) + chunk_start = self.get_chunk_start(date, chunk) if chunk_start.year > year: break elif chunk_start.year == year or chunk_end_date(chunk_start, self.chunk_size, 'month', @@ -265,6 +266,20 @@ class ExperimentConfig(object): return chunks + def get_chunk_start(self, startdate, chunk): + if isinstance(startdate, basestring): + startdate = parse_date(startdate) + return chunk_start_date(startdate, chunk, self.chunk_size, 'month', self.calendar) + + def get_chunk_start_str(self, startdate, chunk): + return date2str(self.get_chunk_start(startdate, chunk)) + + def get_chunk_end(self, startdate, chunk): + return chunk_end_date(self.get_chunk_start(startdate, chunk), self.chunk_size, 'month', self.calendar) + + def get_chunk_end_str(self, startdate, chunk): + return date2str(self.get_chunk_end(startdate, chunk)) + def get_full_years(self, startdate): """ Returns the list of full years that are in the given startdate diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index b8b63d31e525b90b6a732dced2f9e1daedad7349..971e36a4d26ec6531517c4d94025ec99fe6b8ce7 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -33,6 +33,9 @@ class Basin(object): return False return True + def __str__(self): + return self._fullname + @property def shortname(self): """ @@ -140,11 +143,13 @@ class Basins(object): " Hudson " Icelandic_Sea = Basin('Iceland', 'Icelandic_Sea') " Icelandic_Sea " + Irminger_Sea = Basin('Irminger', 'Irminger_Sea') + " Irminger_Sea " Kara_Gate_Strait = Basin('KaraGate', 'Kara_Gate_Strait') " Kara_Gate_Strait " - Kara_Sea = Basin('Kara', 'Kara') + Kara_Sea = Basin('Kara', 'Kara_Sea') " Kara_Sea " - Labrador_Sea = Basin('Labrador', 'Labrador') + Labrador_Sea = Basin('Labrador', 'Labrador_Sea') " Labrador_Sea " Laptev_East_Siberian_Chukchi_Seas = Basin('LaptevESiberianChukchi', 'Laptev_East_Siberian_Chukchi_Seas') " Laptev_East_Siberian_Chukchi_Seas " diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 312300d2ecf48429981f8d1a1940d99910e0a713..3e6f6f3d628adfeabfd8a5992f575f726dac3aba 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -10,7 +10,9 @@ import re from cfunits import Units from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Variable, Domains, VarType +from earthdiagnostics.variable import Variable, VariableManager +from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.modelingrealm import ModelingRealms class DataManager(object): @@ -24,13 +26,41 @@ class DataManager(object): self.config = config self.experiment = config.experiment self._checked_vars = list() - Variable.load_variables() + self.variable_list = VariableManager() + self.variable_list.load_variables(self.config.data_convention) UnitConversion.load_conversions() self.lock = threading.Lock() + def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Checks if a given file exists + + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + raise NotImplementedError() def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VarType.MEAN): + vartype=VariableType.MEAN): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy @@ -49,7 +79,9 @@ class DataManager(object): :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType :return: path to the copy created on the scratch folder :rtype: str """ @@ -57,7 +89,7 @@ class DataManager(object): def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False, vartype=VarType.MEAN): + diagnostic=None, cmorized=False, vartype=VariableType.MEAN): """ Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge with already existing ones as needed @@ -90,11 +122,13 @@ class DataManager(object): :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str + :type frequency: Frequency :param diagnostic: diagnostic used to generate the file :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType """ raise NotImplementedError() @@ -129,13 +163,13 @@ class DataManager(object): if grid: var = '{0}-{1}'.format(var, grid) - if domain in [Domains.ocean, Domains.seaIce]: + if domain in [ModelingRealms.ocean, ModelingRealms.seaIce]: return '{0}_f{1}h'.format(var, self.experiment.ocean_timestep) else: return '{0}_f{1}h'.format(var, self.experiment.atmos_timestep) def _create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): - freq_str = self.frequency_folder_name(frequency, vartype) + freq_str = frequency.folder_name(vartype) if not grid: grid = 'original' @@ -183,21 +217,9 @@ class DataManager(object): os.symlink(filepath, link_path) self.lock.release() - @staticmethod - def frequency_folder_name(frequency, vartype): - if frequency in ('d', 'daily', 'day'): - freq_str = 'daily_{0}'.format(VarType.to_str(vartype)) - elif frequency == 'clim': - freq_str = 'clim' - elif frequency.endswith('hr'): - freq_str = frequency[:-2] + 'hourly' - else: - freq_str = 'monthly_{0}'.format(VarType.to_str(vartype)) - return freq_str - # Overridable methods (not mandatory) def link_file(self, domain, var, startdate, member, chunk=None, grid=None, box=None, - frequency=None, year=None, date_str=None, move_old=False, vartype=VarType.MEAN): + frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ Creates the link of a given file from the CMOR repository. @@ -221,6 +243,8 @@ class DataManager(object): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType :return: path to the copy created on the scratch folder :rtype: str """ @@ -249,14 +273,15 @@ class NetCDFFile(object): :param cmor_var: :type cmor_var: Variable """ - def __init__(self, remote_file, local_file, domain, var, cmor_var): + def __init__(self, remote_file, local_file, domain, var, cmor_var, data_convention, region): self.remote_file = remote_file self.local_file = local_file self.domain = domain self.var = var self.cmor_var = cmor_var - self.region = None + self.region = region self.frequency = None + self.data_convention = data_convention def send(self): Utils.convert2netcdf4(self.local_file) @@ -316,7 +341,8 @@ class NetCDFFile(object): var_handler = handler.variables[self.var] self._fix_variable_name(var_handler) handler.modeling_realm = self.cmor_var.domain.name - handler.table_id = 'Table {0} (December 2013)'.format(self.cmor_var.domain.get_table_name(self.frequency)) + table = self.cmor_var.get_table(self.frequency, self.data_convention) + handler.table_id = 'Table {0} ({1})'.format(table.name, table.date) if self.cmor_var.units: self._fix_units(var_handler) handler.sync() @@ -347,7 +373,7 @@ class NetCDFFile(object): def _fix_coordinate_variables_metadata(self, handler): if 'lev' in handler.variables: handler.variables['lev'].short_name = 'lev' - if self.domain == Domains.ocean: + if self.domain == ModelingRealms.ocean: handler.variables['lev'].standard_name = 'depth' if 'lon' in handler.variables: handler.variables['lon'].short_name = 'lon' @@ -427,8 +453,6 @@ class NetCDFFile(object): handler.close() - - class UnitConversion(object): """ Class to manage unit conversions diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 5eac342437954275e5147ff98d09f7a253432aa2..86787556c32cc91a34b34b2e39436305c737b7f9 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -1,5 +1,8 @@ # coding=utf-8 -from earthdiagnostics.variable import VarType +from earthdiagnostics.constants import Basins +from earthdiagnostics.frequency import Frequency +from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.modelingrealm import ModelingRealm class Diagnostic(object): @@ -56,12 +59,12 @@ class Diagnostic(object): def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - vartype=VarType.MEAN): + vartype=VariableType.MEAN): """ :param filetosend: :param domain: - :type domain: Domain + :type domain: ModelingRealm :param var: :param startdate: :param member: @@ -71,9 +74,12 @@ class Diagnostic(object): :param box: :param rename_var: :param frequency: + :type frequency: Frequency :param year: :param date_str: :param move_old: + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType :return: """ self.data_manager.send_file(filetosend, domain, var, startdate, member, chunk, grid, region, @@ -103,9 +109,106 @@ class Diagnostic(object): """ raise NotImplementedError("Class must override generate_jobs class method") + @classmethod + def process_options(cls, options, options_available): + processed = dict() + options = options[1:] + if len(options) > len(options_available): + raise DiagnosticOptionError('You have specified more options than available for diagnostic ' + '{0}'.format(cls.alias)) + for x in range(len(options_available)): + option_definition = options_available[x] + if len(options) <= x: + option_value = '' + else: + option_value = options[x] + processed[option_definition.name] = option_definition.parse(option_value) + return processed + def __str__(self): """ Must be implemented by derived classes :return: """ return 'Developer must override base class __str__ method' + + +class DiagnosticOption(object): + + def __init__(self, name, default_value=None): + self.name = name + self.default_value = default_value + + def parse(self, option_value): + option_value = self.check_default(option_value) + return option_value + + def check_default(self, option_value): + if option_value == '': + if self.default_value is None: + raise DiagnosticOptionError('Option {0} is not optional'.format(self.name)) + else: + return self.default_value + return option_value + + +class DiagnosticFloatOption(DiagnosticOption): + def parse(self, option_value): + return float(self.check_default(option_value)) + + +class DiagnosticIntOption(DiagnosticOption): + + def __init__(self, name, default_value=None, min_limit=None, max_limit=None): + super(DiagnosticIntOption, self).__init__(name, default_value) + self.min_limit = min_limit + self.max_limit = max_limit + + def parse(self, option_value): + value = int(self.check_default(option_value)) + if self.min_limit is not None and value < self.min_limit: + raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) + if self.max_limit is not None and value > self.max_limit: + raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) + return value + + +class DiagnosticListIntOption(DiagnosticOption): + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, tuple) or isinstance(option_value, list): + return option_value + return [int(i) for i in option_value.split('-')] + + +class DiagnosticDomainOption(DiagnosticOption): + def parse(self, option_value): + return ModelingRealm.parse(self.check_default(option_value)) + + +class DiagnosticFrequencyOption(DiagnosticOption): + def parse(self, option_value): + return Frequency.parse(self.check_default(option_value)) + + +class DiagnosticBasinOption(DiagnosticOption): + def parse(self, option_value): + return Basins.parse(self.check_default(option_value)) + + +class DiagnosticComplexStrOption(DiagnosticOption): + def parse(self, option_value): + return self.check_default(option_value).replace('&;', ',').replace('&.', ' ') + + +class DiagnosticBoolOption(DiagnosticOption): + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, bool): + return option_value + else: + return option_value.lower() in ('true', 't', 'yes') + + +class DiagnosticOptionError(Exception): + pass diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 74f718a266632a6328156d7c298a07c13b36a6d3..990b9213b9d12db697d297cf5d58f5db4098ef65 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -20,6 +20,7 @@ from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.ocean import * from earthdiagnostics.general import * from earthdiagnostics.statistics import * +from earthdiagnostics.variable import VariableManager class EarthDiags(object): @@ -54,6 +55,7 @@ class EarthDiags(object): self.time = dict() self.data_manager = None self.threads = None + self.had_errors = False Log.debug('Diags ready') Log.info('Running diags for experiment {0}, startdates {1}, members {2}', self.config.experiment.expid, self.config.experiment.startdates, self.config.experiment.members) @@ -71,6 +73,8 @@ class EarthDiags(object): help="opens documentation and exits") parser.add_argument('--clean', action='store_true', help="clean the scratch folder and exits") + parser.add_argument('--report', action='store_true', + help="generates a report about the available files") parser.add_argument('-lf', '--logfile', choices=('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'), default='DEBUG', type=str, @@ -108,11 +112,13 @@ class EarthDiags(object): diags = EarthDiags(config_file_path) if args.clean: - diags.clean() + result = diags.clean() + elif args.report: + result = diags.report() else: - diags.run() + result = diags.run() TempFile.clean() - return True + return result def _create_dic_variables(self): self.dic_variables = dict() @@ -129,6 +135,7 @@ class EarthDiags(object): """ Run the diagnostics """ + self.had_errors = False Log.debug('Using netCDF version {0}', netCDF4.getlibversion()) if not os.path.exists(self.config.scratch_dir): os.makedirs(self.config.scratch_dir) @@ -138,13 +145,7 @@ class EarthDiags(object): self._register_diagnostics() - parse_date('20000101') - - if self.config.data_adaptor == 'CMOR': - self.data_manager = CMORManager(self.config) - elif self.config.data_adaptor == 'THREDDS': - self.data_manager = THREDDSManager(self.config) - self.data_manager.prepare() + self._prepare_data_manager() # Run diagnostics Log.info('Running diagnostics') @@ -169,6 +170,14 @@ class EarthDiags(object): Log.result("Diagnostics finished at {0}", finish_time) Log.result("Time ellapsed: {0}\n", finish_time - time) self.print_stats() + return self.had_errors + + def _prepare_data_manager(self): + if self.config.data_adaptor == 'CMOR': + self.data_manager = CMORManager(self.config) + elif self.config.data_adaptor == 'THREDDS': + self.data_manager = THREDDSManager(self.config) + self.data_manager.prepare() def print_stats(self): Log.info('Time consumed by each diagnostic class') @@ -237,12 +246,60 @@ class EarthDiags(object): Diagnostic.register(HeatContentLayer) Diagnostic.register(HeatContent) - def clean(self): Log.info('Removing scratch folder...') if os.path.exists(self.config.scratch_dir): shutil.rmtree(self.config.scratch_dir) Log.result('Scratch folder removed') + return True + + def report(self): + Log.info('Looking for existing vars...') + self._prepare_data_manager() + for startdate in self.config.experiment.startdates: + for member in self.config.experiment.members: + results = self._get_variable_report(startdate, member) + report_path = os.path.join(self.config.scratch_dir, '{0}_fc{1}.report'.format(startdate, member)) + self.create_report(report_path, results) + + Log.result('Report finished') + return True + + def _get_variable_report(self, startdate, member): + var_manager = VariableManager() + results = list() + for var in var_manager.get_all_variables(): + if var.priority is None or var.domain is None: + continue + for table in var.tables: + if not self.data_manager.file_exists(var.domain, var.short_name, startdate, member, 1, + frequency=table.frequency): + results.append((var, table)) + return results + + def create_report(self, report_path, results): + current_table = None + current_priority = 0 + results = sorted(results, key=lambda result: result[0].short_name) + results = sorted(results, key=lambda result: result[0].priority) + results = sorted(results, key=lambda result: result[1].name) + + file_handler = open(report_path, 'w') + + for var, table in results: + if current_table != table.name: + file_handler.write('\nTable {0}\n'.format(table.name)) + file_handler.write('===================================\n') + current_table = table.name + current_priority = 0 + + if current_priority != var.priority: + file_handler.write('\nMissing variables with priority {0}:\n'.format(var.priority)) + file_handler.write('--------------------------------------\n') + current_priority = var.priority + + file_handler.write('{0:12}: {1}\n'.format(var.short_name, var.standard_name)) + file_handler.close() def _run_jobs(self, queue, numthread): def _run_job(current_job, retrials=1): @@ -281,6 +338,7 @@ class EarthDiags(object): else: Log.result('Thread {0} finished after running successfully {1} of {2} tasks', numthread, count, count + len(failed_jobs)) + self.had_errors = False for job in failed_jobs: Log.error('Job {0} could not be run', job) return @@ -333,7 +391,8 @@ class EarthDiags(object): def main(): - EarthDiags.parse_args() + if not EarthDiags.parse_args(): + exit(1) if __name__ == "__main__": diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py new file mode 100644 index 0000000000000000000000000000000000000000..12e1cbe42c9ce4b570b4339af650ac999094e0ce --- /dev/null +++ b/earthdiagnostics/frequency.py @@ -0,0 +1,57 @@ +# coding=utf-8 +from earthdiagnostics.variable_type import VariableType + + +class Frequency(object): + + _recognized = {'f': 'fx', 'fx': 'fx', 'fixed': 'fx', + 'c': 'clim', 'clim': 'clim', 'climatology': 'clim', 'monclim': 'clim', '1hrclimmon': 'clim', + 'y': 'year', 'yr': 'year', 'year': 'year', 'yearly': 'year', + 'm': 'mon', 'mon': 'mon', 'monthly': 'mon', + 'd': 'day', 'daily': 'day', 'day': 'day', + '6': '6hr', '6hr': '6hr', '6_hourly': '6hr', '6hourly': '6hr', '6 hourly': '6hr', + '3': '3hr', '3hr': '3hr', '3_hourly': '3hr', '3hourly': '3hr', '3 hourly': '3hr', + '1': '1hr', 'hr': '1hr', 'hourly': '1hr', '1hr': '1hr', '1 hourly': '1hr', + 'subhr': 'subhr'} + + def __init__(self, freq): + freq = freq.lower() + try: + self.frequency = Frequency._recognized[freq] + except KeyError: + raise Exception('Frequency {0} not supported'.format(freq)) + + def __eq__(self, other): + return self.frequency == other.frequency + + def __str__(self): + return self.frequency + + def folder_name(self, vartype): + if self == Frequencies.daily: + freq_str = 'daily_{0}'.format(VariableType.to_str(vartype)) + elif self == Frequencies.climatology: + freq_str = 'clim' + elif self in (Frequencies.three_hourly, Frequencies.six_hourly, Frequencies.hourly): + freq_str = self.frequency[:-2] + 'hourly' + else: + freq_str = 'monthly_{0}'.format(VariableType.to_str(vartype)) + return freq_str + + @staticmethod + def parse(freq): + if isinstance(freq, Frequency): + return freq + return Frequency(freq) + + +class Frequencies(object): + fixed = Frequency('fx') + climatology = Frequency('clim') + yearly = Frequency('year') + monthly = Frequency('mon') + daily = Frequency('day') + six_hourly = Frequency('6hr') + three_hourly = Frequency('3hr') + hourly = Frequency('hr') + subhourly = Frequency('subhr') diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 59edf3c65059657bb560882fc2cdd5f063edd2bd..4b0bc129de0f3eb69478b5930b1b7fd5c64ec28b 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -1,7 +1,7 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticComplexStrOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils -from earthdiagnostics.variable import Domain +from earthdiagnostics.modelingrealm import ModelingRealm class Attribute(Diagnostic): @@ -24,7 +24,7 @@ class Attribute(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: Domain + :type domain: ModelingRealm """ alias = 'att' @@ -63,23 +63,18 @@ class Attribute(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the variable, domain, attributte name and value to write') - if num_options > 5: - raise Exception('You must between 4 and 5 parameters for the rewrite diagnostic') - variable = options[1] - domain = Domain(options[2]) - name = options[3] - value = options[4] - value = value.replace('&;', ',').replace('&.', ' ') - if num_options >= 5: - grid = options[5] - else: - grid = None + + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('name'), + DiagnosticComplexStrOption('value'), + DiagnosticOption('grid')) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Attribute(diags.data_manager, startdate, member, chunk, domain, variable, grid, name, value)) + job_list.append(Attribute(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'], options['grid'], + options['value'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 69983f37176c2b6774e302ee2681bc69131c8a08..5c2e5ece56cd8fbf02f90646b189bf8584f4958f 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -1,9 +1,10 @@ # coding=utf-8 import os -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticFrequencyOption +from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain +from earthdiagnostics.modelingrealm import ModelingRealm class MonthlyMean(Diagnostic): @@ -25,7 +26,7 @@ class MonthlyMean(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: Domain + :type domain: ModelingRealm :param frequency: original frequency :type frequency: str :param grid: original data grid @@ -65,26 +66,16 @@ class MonthlyMean(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to average monthly') - if num_options > 4: - raise Exception('You must specify between 2 and 4 parameters for the monthly mean diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - frequency = options[3] - else: - frequency = 'day' - if num_options >= 4: - grid = options[4] - else: - grid = None + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticFrequencyOption('frequency', Frequencies.daily), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, domain, variable, - frequency, grid)) + job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) return job_list def compute(self): @@ -98,5 +89,5 @@ class MonthlyMean(Diagnostic): os.remove(variable_file) self.send_file(temp, self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency='mon', grid=self.grid) + frequency=Frequencies.monthly, grid=self.grid) diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 3009f20e1c9cfd8116668617406537de9e3354ee..f12764b20e2ad2da13ece731156005ca96364516 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -1,6 +1,6 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic -from earthdiagnostics.variable import Domain +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption +from earthdiagnostics.modelingrealm import ModelingRealm class Relink(Diagnostic): @@ -22,7 +22,7 @@ class Relink(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: Domain + :type domain: ModelingRealm :param move_old: if true, looks for files following the old convention and moves to avoid collisions :type move_old: bool """ @@ -59,20 +59,14 @@ class Relink(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to link') - if num_options > 3: - raise Exception('You must between 2 and 3 parameters for the relink diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - move_old = bool(options[3].lower()) - else: - move_old = True + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticBoolOption('move_old', True)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Relink(diags.data_manager, startdate, member, chunk, domain, variable, move_old)) + job_list.append(Relink(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['move_old'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index c7acd74401dd0fd5f9ce625d306849b7f62fbaae..ab6b87c7146f350b4633cdf31b5ad953d33e2275 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -1,6 +1,6 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic -from earthdiagnostics.variable import Domain +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption +from earthdiagnostics.modelingrealm import ModelingRealm class Rewrite(Diagnostic): @@ -23,7 +23,7 @@ class Rewrite(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: Domain + :type domain: ModelingRealm """ alias = 'rewrite' @@ -58,20 +58,14 @@ class Rewrite(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to rewrite') - if num_options > 3: - raise Exception('You must between 2 and 3 parameters for the rewrite diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - grid = options[3] - else: - grid = None + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, domain, variable, grid)) + job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index d41797b5ffb629f93af3af691cdf8756904c1f5d..ceaac656cedcac6ee8c7b17f6881a471eea72b58 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,8 +1,9 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticFloatOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils -from earthdiagnostics.variable import Domain +from earthdiagnostics.modelingrealm import ModelingRealm import numpy as np +import math class Scale(Diagnostic): @@ -26,7 +27,7 @@ class Scale(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: Domain + :type domain: ModelingRealm """ alias = 'scale' @@ -69,37 +70,19 @@ class Scale(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the acale and offset values and the variable and domain to scale') - if num_options > 5: - raise Exception('You must between 4 and 5 parameters for the rewrite diagnostic') - value = float(options[1]) - offset = float(options[2]) - variable = options[3] - domain = Domain(options[4]) - if num_options >= 5: - grid = options[5] - else: - grid = None - if num_options >= 6: - if options[6].lower() == 'none': - min_limit = None - else: - min_limit = float(options[6]) - else: - min_limit = None - if num_options >= 7: - if options[7].lower() == 'none': - max_limit = None - else: - max_limit = float(options[7]) - else: - max_limit = None + options_available = (DiagnosticFloatOption('value'), + DiagnosticFloatOption('offset'), + DiagnosticDomainOption('domain'), + DiagnosticOption('variable'), + DiagnosticOption('grid', ''), + DiagnosticFloatOption('min_limit', float('nan')), + DiagnosticFloatOption('max_limit', float('nan'))) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Scale(diags.data_manager, startdate, member, chunk, value, offset, domain, variable, grid, - min_limit, max_limit)) + job_list.append(Scale(diags.data_manager, startdate, member, chunk, + options['value'], options['offset'], options['domain'], options['variable'], + options['grid'], options['min_limit'], options['max_limit'])) return job_list def compute(self): @@ -119,9 +102,9 @@ class Scale(Diagnostic): grid=self.grid) def _check_limits(self): - if self.min_limit is not None and np.amin(self.original_values) < self.min_limit: + if not math.isnan(self.min_limit) and np.amin(self.original_values) < self.min_limit: return False - if self.max_limit is not None and np.amax(self.original_values) > self.max_limit: + if not math.isnan(self.max_limit) is not None and np.amax(self.original_values) > self.max_limit: return False return True diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py new file mode 100644 index 0000000000000000000000000000000000000000..3a970eee675dcc735145737b6cb1c903e4c80b5f --- /dev/null +++ b/earthdiagnostics/modelingrealm.py @@ -0,0 +1,76 @@ +# coding=utf-8 +from earthdiagnostics.frequency import Frequencies + + +class ModelingRealm(object): + + @staticmethod + def parse(domain_name): + if isinstance(domain_name, ModelingRealm): + return domain_name + return ModelingRealm(domain_name) + + def __init__(self, domain_name): + domain_name = domain_name.lower() + if domain_name == 'seaice': + self.name = 'seaIce' + elif domain_name == 'landice': + self.name = 'landIce' + elif domain_name == 'atmoschem': + self.name = 'atmosChem' + elif domain_name == 'ocnbgchem': + self.name = 'ocnBgchem' + elif domain_name in ['ocean', 'atmos', 'land', 'aerosol']: + self.name = domain_name + else: + raise ValueError('Domain {0} not recognized!'.format(domain_name)) + + def __eq__(self, other): + return other.__class__ == ModelingRealm and self.name == other.name + + def __str__(self): + return self.name + + def get_table_name(self, frequency, data_convention): + """ + Returns the table name for a domain-frequency pair + :param data_convention: Data convention in use + :type data_convention: str + :param frequency: variable's frequency + :type frequency: str + :return: variable's table name + :rtype: str + """ + if frequency in (Frequencies.monthly, Frequencies.climatology): + if self.name == 'seaIce': + if data_convention == 'specs': + prefix = 'OI' + else: + prefix = 'SI' + elif self.name == 'landIce': + prefix = 'LI' + else: + prefix = self.name[0].upper() + table_name = prefix + str(frequency) + elif frequency == Frequencies.six_hourly: + table_name = '6hrPlev' + else: + table_name = 'day' + return table_name + + def get_table(self, frequency, data_convention): + table_name = self.get_table_name(frequency, data_convention) + from earthdiagnostics.variable import CMORTable + return CMORTable(table_name, frequency, 'December 2013') + + +class ModelingRealms(object): + seaIce = ModelingRealm('seaice') + ocean = ModelingRealm('ocean') + landIce = ModelingRealm('landIce') + atmos = ModelingRealm('atmos') + land = ModelingRealm('land') + aerosol = ModelingRealm('aerosol') + atmosChem = ModelingRealm('atmosChem') + ocnBgchem = ModelingRealm('ocnBgchem') + diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 3942b7f3b3b37a27372b3143d466fb9f7f2d74ab..d2ea66ef1699f48bc4912eef7fdedadec62b3be2 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -1,12 +1,12 @@ # coding=utf-8 import numpy as np from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption from earthdiagnostics.box import Box from earthdiagnostics.utils import Utils, TempFile import os -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class AreaMoc(Diagnostic): @@ -68,24 +68,21 @@ class AreaMoc(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the box to use') - if num_options > 5: - raise Exception('You must specify between 4 and 5 parameters for area moc diagnostic') - box = Box() - box.min_lat = int(options[1]) - box.max_lat = int(options[2]) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) - if num_options > 4: - basin = Basins.parse(options[5]) - else: - basin = Basins.Global + options_available = (DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) + box = Box() + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(AreaMoc(diags.data_manager, startdate, member, chunk, basin, box)) + job_list.append(AreaMoc(diags.data_manager, startdate, member, chunk, options['basin'], box)) return job_list def compute(self): @@ -96,7 +93,7 @@ class AreaMoc(Diagnostic): cdo = Utils.cdo temp2 = TempFile.get() - temp = self.data_manager.get_file('ocean', 'vsftmyz', self.startdate, self.member, self.chunk) + temp = self.data_manager.get_file(ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.chunk) handler = Utils.openCdf(temp) if 'i' in handler.dimensions: @@ -149,4 +146,4 @@ class AreaMoc(Diagnostic): nco.ncap2(input=temp2, output=temp2, options='-O -s "coslat[lat]=cos(lat[lat]*3.141592657/180.0)"') nco.ncwa(input=temp2, output=temp2, options='-w coslat -a lat') nco.ncks(input=temp2, output=temp2, options='-O -v vsftmyz,time') - self.send_file(temp2, Domains.ocean, 'vsftmyz', self.startdate, self.member, self.chunk, box=self.box) + self.send_file(temp2, ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.chunk, box=self.box) diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 50961df6910496634cb2c8bdb602d532db7385e3..c4ddfb4132c6d2918ad3a12d6a987df2ba05c759 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -1,10 +1,9 @@ # coding=utf-8 import os from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms class AverageSection(Diagnostic): @@ -28,7 +27,7 @@ class AverageSection(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: Domain + :type domain: ModelingRealm :param box: box to use for the average :type box: Box @@ -65,25 +64,22 @@ class AverageSection(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 5: - raise Exception('You must specify the variable and the box to average') - if num_options > 6: - raise Exception('You must specify between 5 and 6 parameters for the section average diagnostic') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticIntOption('min_lon'), + DiagnosticIntOption('max_lon'), + DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticDomainOption('domain', ModelingRealms.ocean)) + options = cls.process_options(options, options_available) box = Box() - box.min_lon = int(options[2]) - box.max_lon = int(options[3]) - box.min_lat = int(options[4]) - box.max_lat = int(options[5]) - if num_options >= 6: - domain = Domain(options[6]) - else: - domain = Domains.ocean - + box.min_lon = options['min_lon'] + box.max_lon = options['max_lon'] + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(AverageSection(diags.data_manager, startdate, member, chunk, domain, variable, box)) + job_list.append(AverageSection(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index 3ff6a65b1837b3a13f068ea36637ff6efa1abb0c..ca140c2233fc82e7ee73ed2ca84472361584704b 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -4,6 +4,7 @@ from autosubmit.config.log import Log from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.constants import Models +from earthdiagnostics.modelingrealm import ModelingRealms class ConvectionSites(Diagnostic): @@ -84,7 +85,8 @@ class ConvectionSites(Diagnostic): else: raise Exception("Input grid {0} not recognized".format(self.model_version)) - mlotst_file = self.data_manager.get_file('ocean', 'mlotst', self.startdate, self.member, self.chunk) + mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, + self.chunk) output = TempFile.get() self.mlotst_handler = Utils.openCdf(mlotst_file) @@ -113,7 +115,7 @@ class ConvectionSites(Diagnostic): self.mlotst_handler.close() handler.close() - self.send_file(output, 'ocean', 'site', self.startdate, self.member, self.chunk) + self.send_file(output, ModelingRealms.ocean, 'site', self.startdate, self.member, self.chunk) Log.info('Finished convection sites for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index 95701ab806f7106acbc81932ccf1d44d3a287986..899c89ccae5d703a3d46fcc043833d1978145d4e 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -2,11 +2,11 @@ import numpy as np from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticBoolOption, DiagnosticIntOption, \ + DiagnosticDomainOption from earthdiagnostics.box import Box from earthdiagnostics.utils import Utils -from earthdiagnostics.variable import Domain -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class CutSection(Diagnostic): @@ -30,7 +30,7 @@ class CutSection(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: str + :type domain: Domain :param zonal: specifies if section is zonal or meridional :type zonal: bool :param value: value of the section's coordinate @@ -71,22 +71,16 @@ class CutSection(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable, coordinate and coordinate value') - if num_options > 4: - raise Exception('You must specify between 3 and 4 parameters for the interpolation diagnostic') - variable = options[1] - zonal = options[2].lower() == 'true' - value = int(options[3]) - if num_options >= 4: - domain = Domain(options[4]) - else: - domain = Domains.ocean + options_available = (DiagnosticOption('variable'), + DiagnosticBoolOption('zonal'), + DiagnosticIntOption('value'), + DiagnosticDomainOption('domain', ModelingRealms.ocean)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(CutSection(diags.data_manager, startdate, member, chunk, domain, variable, zonal, value)) + job_list.append(CutSection(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['zonal'], options['value'])) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index 93bdb6076718f9ecd5077bff3d6ac7bdc586e831..c7eb720f6dc659ba78a983ca844e57d80938b7f1 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -5,6 +5,7 @@ from autosubmit.config.log import Log from earthdiagnostics.constants import Models from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms class Gyres(Diagnostic): @@ -92,7 +93,8 @@ class Gyres(Diagnostic): raise Exception("Input grid {0} not recognized".format(self.model_version)) output = TempFile.get() - vsftbarot_file = self.data_manager.get_file('ocean', 'vsftbarot', self.startdate, self.member, self.chunk) + vsftbarot_file = self.data_manager.get_file(ModelingRealms.ocean, 'vsftbarot', self.startdate, + self.member, self.chunk) handler_original = Utils.openCdf(vsftbarot_file) self.var_vsftbarot = handler_original.variables['vsftbarot'] @@ -143,7 +145,7 @@ class Gyres(Diagnostic): handler.close() handler_original.close() - self.send_file(output, 'ocean', 'gyre', self.startdate, self.member, self.chunk) + self.send_file(output, ModelingRealms.ocean, 'gyre', self.startdate, self.member, self.chunk) Log.info('Finished gyres for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) def _gyre(self, site, invert=False): diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 5d639e3b9cc2b90daee4a385e27f5fa0d2a6bf26..a82a4da64bca7383255f8e859a11ff9929fcafd0 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -6,9 +6,9 @@ from autosubmit.config.log import Log from earthdiagnostics import cdftools from earthdiagnostics.constants import Basins from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticIntOption from earthdiagnostics.box import Box -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class HeatContent(Diagnostic): @@ -71,19 +71,18 @@ class HeatContent(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the basin, mixed layer option and minimum and maximum depth to use') - if num_options > 4: - raise Exception('You must specify 4 parameters for the heat content diagnostic') - basin = Basins.parse(options[1]) - mixed_layer = int(options[2]) - box = Box(True) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) + options_available = (DiagnosticBasinOption('basin'), + DiagnosticIntOption('mixed_layer', None, -1, 1), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth')) + options = cls.process_options(options, options_available) + box = Box(False) + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(HeatContent(diags.data_manager, startdate, member, chunk, basin, mixed_layer, box)) + job_list.append(HeatContent(diags.data_manager, startdate, member, chunk, + options['basin'], options['mixed_layer'], box)) return job_list def compute(self): @@ -91,9 +90,11 @@ class HeatContent(Diagnostic): Runs the diagnostic """ nco = Utils.nco - temperature_file = self.data_manager.get_file('ocean', 'thetao', self.startdate, self.member, self.chunk) + temperature_file = self.data_manager.get_file(ModelingRealms.ocean, 'thetao', self.startdate, + self.member, self.chunk) if self.mxloption != 0: - mlotst_file = self.data_manager.get_file('ocean', 'mlotst', self.startdate, self.member, self.chunk) + mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', self.startdate, + self.member, self.chunk) nco.ncks(input=mlotst_file, output=temperature_file, options='-A -v mlotst') para = list() @@ -119,19 +120,19 @@ class HeatContent(Diagnostic): shell_output = cdftools.run('cdfheatc', options=para, input=temperature_file) - ohcsum_temp = TempFile.get() - ohcvmean_temp = TempFile.get() - nco.ncks(input=temperature_file, output=ohcsum_temp, options='-O -v time') - shutil.copy(ohcsum_temp, ohcvmean_temp) + heatcsum_temp = TempFile.get() + heatcvmean_temp = TempFile.get() + nco.ncks(input=temperature_file, output=heatcsum_temp, options='-O -v time') + shutil.copy(heatcsum_temp, heatcvmean_temp) - ohcsum_handler = Utils.openCdf(ohcsum_temp) - thc = ohcsum_handler.createVariable('ohcsum', float, 'time') + heatcsum_handler = Utils.openCdf(heatcsum_temp) + thc = heatcsum_handler.createVariable('heatcsum', float, 'time') thc.standard_name = "integral_of_sea_water_potential_temperature_expressed_as_heat_content" thc.long_name = "Total heat content" thc.units = "J" - ohcvmean_handler = Utils.openCdf(ohcvmean_temp) - uhc = ohcvmean_handler.createVariable('ohcvmean', float, 'time') + heatcvmean_handler = Utils.openCdf(heatcvmean_temp) + uhc = heatcvmean_handler.createVariable('heatcvmean', float, 'time') uhc.standard_name = "integral_of_sea_water_potential_temperature_expressed_as_heat_content" uhc.long_name = "Heat content per unit volume" uhc.units = "J*m^-3" @@ -156,8 +157,8 @@ class HeatContent(Diagnostic): elif line.startswith('TIME : '): Log.info(line) - ohcsum_handler.close() - ohcvmean_handler.close() + heatcsum_handler.close() + heatcvmean_handler.close() if self.box.min_depth == 0: # For cdftools, this is all levels @@ -165,9 +166,9 @@ class HeatContent(Diagnostic): else: box_save = self.box - Utils.setminmax(ohcsum_temp, 'ohcsum') - self.send_file(ohcsum_temp, Domains.ocean, 'ohcsum', self.startdate, self.member, self.chunk, - box=box_save, region=self.basin.fullname, rename_var='ohcsum') - Utils.setminmax(ohcvmean_temp, 'ohcvmean') - self.send_file(ohcvmean_temp, Domains.ocean, 'ohcvmean', self.startdate, self.member, self.chunk, - box=box_save, region=self.basin.fullname, rename_var='ohcvmean') + Utils.setminmax(heatcsum_temp, 'heatcsum') + self.send_file(heatcsum_temp, ModelingRealms.ocean, 'heatcsum', self.startdate, self.member, self.chunk, + box=box_save, region=self.basin.fullname, rename_var='heatcsum') + Utils.setminmax(heatcvmean_temp, 'heatcvmean') + self.send_file(heatcvmean_temp, ModelingRealms.ocean, 'heatcvmean', self.startdate, self.member, self.chunk, + box=box_save, region=self.basin.fullname, rename_var='heatcvmean') diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 65f86cb16e64bbc3cc77bb460d64bb688fb39c3f..cf398972bbcc26b89a42192c48746a57baa38187 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -3,9 +3,9 @@ import numpy as np from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class HeatContentLayer(Diagnostic): @@ -61,22 +61,18 @@ class HeatContentLayer(Diagnostic): :param options: minimum depth, maximum depth, basin=Global :type options: list[str] """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the minimum and maximum depth to use') - if num_options > 3: - raise Exception('You must specify between 2 and 3 parameters for the heat content layer diagnostic') + options_available = (DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) + box = Box(True) - box.min_depth = int(options[1]) - box.max_depth = int(options[2]) - if len(options) > 3: - basin = Basins.parse(options[3]) - else: - basin = Basins.Global + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() handler = Utils.openCdf('mesh_zgr.nc') - mask = Utils.get_mask(basin) + mask = Utils.get_mask(options['basin']) if 'e3t' in handler.variables: mask = handler.variables['e3t'][:] * mask @@ -161,7 +157,8 @@ class HeatContentLayer(Diagnostic): nco = Utils.nco results = TempFile.get() - thetao_file = self.data_manager.get_file(Domains.ocean, 'thetao', self.startdate, self.member, self.chunk) + thetao_file = self.data_manager.get_file(ModelingRealms.ocean, 'thetao', + self.startdate, self.member, self.chunk) handler = Utils.openCdf(thetao_file) heatc_sl = np.sum(handler.variables['thetao'][:, self.min_level:self.max_level, :] * self.weight, 1) @@ -172,10 +169,10 @@ class HeatContentLayer(Diagnostic): nco.ncks(input=thetao_file, output=results, options='-O -v lon,lat,time') Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False, True) handler_results = Utils.openCdf(results) - handler_results.createVariable('ohc', float, ('time', 'j', 'i'), fill_value=1.e20) + handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) handler_results.sync() - handler_results.variables['ohc'][:] = heatc_sl + handler_results.variables['heatc'][:] = heatc_sl handler_results.close() - Utils.setminmax(results, 'ohc') - self.send_file(results, Domains.ocean, 'ohc', self.startdate, self.member, self.chunk, box=self.box) + Utils.setminmax(results, 'heatc') + self.send_file(results, ModelingRealms.ocean, 'heatc', self.startdate, self.member, self.chunk, box=self.box) diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 5401dffa4aeb18afe59869eefc1466a1a6895fe9..e1f0ff48297cbd07d572e4ccaf02e3b571ab317b 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -4,9 +4,9 @@ import threading import os from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, Domains +from earthdiagnostics.modelingrealm import ModelingRealms class Interpolate(Diagnostic): @@ -31,7 +31,7 @@ class Interpolate(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: str + :type domain: Domain :param model_version: model version :type model_version: str """ @@ -79,27 +79,18 @@ class Interpolate(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the grid and variable to interpolate') - if num_options > 4: - raise Exception('You must specify between 2 and 4 parameters for the interpolation diagnostic') - target_grid = options[1] - variable = options[2] - if num_options >= 3: - domain = Domain(options[3]) - else: - domain = Domains.ocean - if num_options >= 4: - invert_lat = bool(options[4].lower()) - else: - invert_lat = False + options_available = (DiagnosticOption('target_grid'), + DiagnosticOption('variable'), + DiagnosticDomainOption('domain', ModelingRealms.ocean), + DiagnosticBoolOption('invert_lat', False)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append( - Interpolate(diags.data_manager, startdate, member, chunk, domain, variable, target_grid, - diags.config.experiment.model_version, invert_lat)) + Interpolate(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['target_grid'], + diags.config.experiment.model_version, options['invert_lat'])) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index a80e8da834d2800dc8af9cf29601aded87b8fa95..68a04f5817dac95412b05bf64a16bdfaab5ce3d9 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -1,11 +1,10 @@ # coding=utf-8 from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils, TempFile import numpy as np -from earthdiagnostics.variable import Domain -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms class InterpolateCDO(Diagnostic): @@ -28,7 +27,7 @@ class InterpolateCDO(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: Domain + :type domain: ModelingRealm :param model_version: model version :type model_version: str """ @@ -71,39 +70,26 @@ class InterpolateCDO(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to interpolate') - if num_options > 3: - raise Exception('You must specify between 1 and 3 parameters for the interpolation with CDO diagnostic') - variable = options[1] - - if num_options >= 3: - target_grid = options[2] - else: - target_grid = diags.config.experiment.atmos_grid.lower() - - target_grid = cls._translate_ifs_grids_to_cdo_names(target_grid) - - if num_options >= 3: - domain = Domain(options[3]) - else: - domain = Domains.ocean + options_available = (DiagnosticOption('variable'), + DiagnosticOption('target_grid', diags.config.experiment.atmos_grid.lower()), + DiagnosticDomainOption('domain', ModelingRealms.ocean)) + options = cls.process_options(options, options_available) + target_grid = cls._translate_ifs_grids_to_cdo_names(options['target_grid']) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append( - InterpolateCDO(diags.data_manager, startdate, member, chunk, domain, variable, target_grid, - diags.config.experiment.model_version)) + job_list.append(InterpolateCDO(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], target_grid, + diags.config.experiment.model_version)) return job_list @classmethod def _translate_ifs_grids_to_cdo_names(cls, target_grid): - if target_grid.startswith('T159L'): - target_grid = 't106' - if target_grid.startswith('T255L'): - target_grid = 't170' - if target_grid.startswith('T511L'): - target_grid = 't340' + if target_grid.upper().startswith('T159L'): + target_grid = 't159grid' + if target_grid.upper().startswith('T255L'): + target_grid = 't255grid' + if target_grid.upper().startswith('T511L'): + target_grid = 't511grid' return target_grid def compute(self): diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index fe212894032b88ac6f379b51d114e2f04311ae52..18f27417203fdfcb9ba029da3940f373a54e4fb3 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -5,9 +5,10 @@ import os from autosubmit.config.log import Log from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption +from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class MaxMoc(Diagnostic): @@ -66,30 +67,27 @@ class MaxMoc(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the box to use') - if num_options > 5: - raise Exception('You must specify between 4 and 5 parameters for area moc diagnostic') + options_available = (DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) box = Box() - box.min_lat = int(options[1]) - box.max_lat = int(options[2]) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) - if num_options > 4: - basin = Basins.parse(options[5]) - else: - basin = Basins.Global + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() - for startdate in diags.startdates: - for member in diags.members: + for startdate in diags.config.experiment.startdates: + for member in diags.config.experiment.members: years = diags.config.experiment.get_full_years(startdate) if len(years) == 0: Log.user_warning('No complete years are available with the given configuration. ' 'MaxMoc can not be computed') for year in years: - job_list.append(MaxMoc(diags.data_manager, startdate, member, year, basin, box)) + job_list.append(MaxMoc(diags.data_manager, startdate, member, year, options['basin'], box)) return job_list def compute(self): @@ -98,7 +96,7 @@ class MaxMoc(Diagnostic): """ nco = Utils.nco - temp = self.data_manager.get_year(Domains.ocean, 'vsftmyz', self.startdate, self.member, self.year) + temp = self.data_manager.get_year(ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.year) handler = Utils.openCdf(temp) if 'i' in handler.dimensions: @@ -157,8 +155,8 @@ class MaxMoc(Diagnostic): var.valid_max = 1000. var[0] = maximum handler.close() - self.send_file(temp, Domains.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency='yr', year=self.year) + self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, + frequency=Frequencies.yearly, year=self.year) handler = self._create_output_file(temp) var = handler.createVariable('vsftmyzmaxlat', float, ('time',)) @@ -168,8 +166,8 @@ class MaxMoc(Diagnostic): var.valid_max = 90. var[0] = max_lat handler.close() - self.send_file(temp, Domains.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency='yr', year=self.year) + self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, + frequency=Frequencies.yearly, year=self.year) handler = self._create_output_file(temp) var = handler.createVariable('vsftmyzmaxlev', float, ('time',)) @@ -179,8 +177,8 @@ class MaxMoc(Diagnostic): var.valid_max = 10000. var[0] = max_lev handler.close() - self.send_file(temp, Domains.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency='yr', year=self.year) + self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, + frequency=Frequencies.yearly, year=self.year) handler = self._create_output_file(temp) var = handler.createVariable('vsftmyzmin', float, ('time',)) @@ -190,8 +188,8 @@ class MaxMoc(Diagnostic): var.valid_max = 1000. var[0] = minimum handler.close() - self.send_file(temp, Domains.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency='yr', year=self.year) + self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, + frequency=Frequencies.yearly, year=self.year) handler = self._create_output_file(temp) var = handler.createVariable('vsftmyzminlat', float, ('time',)) @@ -201,8 +199,8 @@ class MaxMoc(Diagnostic): var.valid_max = 90. var[0] = min_lat handler.close() - self.send_file(temp, Domains.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency='yr', year=self.year) + self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, + frequency=Frequencies.yearly, year=self.year) handler = self._create_output_file(temp) var = handler.createVariable('vsftmyzminlev', float, ('time',)) @@ -212,8 +210,8 @@ class MaxMoc(Diagnostic): var.valid_max = 10000. var[0] = min_lev handler.close() - self.send_file(temp, Domains.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency='yr', year=self.year) + self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, + frequency=Frequencies.yearly, year=self.year) def _create_output_file(self, temp): handler = netCDF4.Dataset(temp, 'w') diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index f5af3f53dfe51fa1cc792da24b76b2f5e0a3180d..e880090106360a5d8ef90650e2436aad1fded493 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -4,7 +4,7 @@ import os from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics import cdftools from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class MixedLayerHeatContent(Diagnostic): @@ -67,8 +67,10 @@ class MixedLayerHeatContent(Diagnostic): """ Runs the diagnostic """ - temperature_file = self.data_manager.get_file(Domains.ocean, 'thetao', self.startdate, self.member, self.chunk) - mlotst_file = self.data_manager.get_file(Domains.ocean, 'mlotst', self.startdate, self.member, self.chunk) + temperature_file = self.data_manager.get_file(ModelingRealms.ocean, 'thetao', + self.startdate, self.member, self.chunk) + mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', + self.startdate, self.member, self.chunk) Utils.nco.ncks(input=mlotst_file, output=temperature_file, options='-A -v mlotst') @@ -79,4 +81,4 @@ class MixedLayerHeatContent(Diagnostic): Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlheatc': 'ohcvsumlotst'}, False, True) Utils.setminmax(temp, 'ohcvsumlotst') - self.send_file(temp, Domains.ocean, 'ohcvsumlotst', self.startdate, self.member, self.chunk) + self.send_file(temp, ModelingRealms.ocean, 'ohcvsumlotst', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index 2fda1b3f990f05559dd8041b2353bc00e70d36a1..0127fe64df69f8bc0078663bd472dbe0aa6913f9 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -3,7 +3,7 @@ import os from earthdiagnostics import cdftools from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class MixedLayerSaltContent(Diagnostic): @@ -65,8 +65,9 @@ class MixedLayerSaltContent(Diagnostic): """ Runs the diagnostic """ - salinity_file = self.data_manager.get_file(Domains.ocean, 'so', self.startdate, self.member, self.chunk) - mlotst_file = self.data_manager.get_file(Domains.ocean, 'mlotst', self.startdate, self.member, self.chunk) + salinity_file = self.data_manager.get_file(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) + mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', + self.startdate, self.member, self.chunk) Utils.nco.ncks(input=mlotst_file, output=salinity_file, options='-A -v mlotst') @@ -76,4 +77,4 @@ class MixedLayerSaltContent(Diagnostic): Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlsaltc': 'scvsummlotst'}, False, True) Utils.setminmax(temp, 'scvsummlotst') - self.send_file(temp, Domains.ocean, 'scvsummlotst', self.startdate, self.member, self.chunk) + self.send_file(temp, ModelingRealms.ocean, 'scvsummlotst', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index 459651d2acde77d03e10bb54f3ca808ace62e224..ee070c19e9bb6b06af376a650700649aa79a7cf8 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -6,7 +6,7 @@ from earthdiagnostics import cdftools from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class Moc(Diagnostic): @@ -70,7 +70,7 @@ class Moc(Diagnostic): """ temp = TempFile.get() - input_file = self.data_manager.get_file(Domains.ocean, 'vo', self.startdate, self.member, self.chunk) + input_file = self.data_manager.get_file(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) Log.debug('Computing MOC') cdftools.run('cdfmoc', input=input_file, output=temp) @@ -107,4 +107,4 @@ class Moc(Diagnostic): options='-O -x -v zomsfglo,zomsfatl,zomsfpac,zomsfinp,zomsfind,zomsfinp0') Utils.setminmax(temp, 'vsftmyz') - self.send_file(temp, Domains.ocean, 'vsftmyz', self.startdate, self.member, self.chunk) + self.send_file(temp, ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index 0cabb6799188ed229faaf87d0a635a91d37285a1..072bfb875b0db305a08263a2405d71e85aa9d50d 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -2,7 +2,7 @@ from earthdiagnostics import cdftools from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class Psi(Diagnostic): @@ -65,9 +65,9 @@ class Psi(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - input_file_u = self.data_manager.get_file(Domains.ocean, 'uo', self.startdate, self.member, self.chunk) - input_file_v = self.data_manager.get_file(Domains.ocean, 'vo', self.startdate, self.member, self.chunk) + input_file_u = self.data_manager.get_file(ModelingRealms.ocean, 'uo', self.startdate, self.member, self.chunk) + input_file_v = self.data_manager.get_file(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) cdftools.run('cdfpsi', input=[input_file_u, input_file_v], output=temp, options='-mean -mask') Utils.rename_variable(temp, 'sobarstf', 'vsftbarot') Utils.setminmax(temp, 'vsftbarot') - self.send_file(temp, Domains.ocean, 'vsftbarot', self.startdate, self.member, self.chunk) + self.send_file(temp, ModelingRealms.ocean, 'vsftbarot', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 6e46264e3930d3f7179d9241b8e9b107e54de4d1..604216b4c7372bc1ce4b3df660fa3e8b53a38a51 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,13 +1,15 @@ # coding=utf-8 import netCDF4 import os -from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption from earthdiagnostics.utils import Utils, TempFile +# noinspection PyUnresolvedReferences import earthdiagnostics.cdftoolspython as cdftoolspython import numpy as np -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.constants import Basins class Siasiesiv(Diagnostic): @@ -68,15 +70,14 @@ class Siasiesiv(Diagnostic): :type options: list[str] :return: """ - if len(options) != 2: - raise Exception('You must specify the basin for the siasiesiv diagnostic (and nothing else)') - basin = Basins.parse(options[1]) + options_available = (DiagnosticBasinOption('basin', Basins.Global), ) + options = cls.process_options(options, options_available) - mask = Utils.get_mask(basin) + mask = Utils.get_mask(options['basin']) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, basin, mask)) + job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, options['basin'], mask)) mesh_handler = Utils.openCdf('mesh_hgr.nc') Siasiesiv.e1t = np.asfortranarray(mesh_handler.variables['e1t'][0, :]) Siasiesiv.e2t = np.asfortranarray(mesh_handler.variables['e2t'][0, :]) @@ -89,13 +90,13 @@ class Siasiesiv(Diagnostic): """ Runs the diagnostic """ - sit_file = self.data_manager.get_file(Domains.seaIce, 'sit', self.startdate, self.member, self.chunk) + sit_file = self.data_manager.get_file(ModelingRealms.seaIce, 'sit', self.startdate, self.member, self.chunk) sit_handler = Utils.openCdf(sit_file) sit = np.asfortranarray(sit_handler.variables['sit'][:]) timesteps = sit_handler.dimensions['time'].size sit_handler.close() - sic_file = self.data_manager.get_file(Domains.seaIce, 'sic', self.startdate, self.member, self.chunk) + sic_file = self.data_manager.get_file(ModelingRealms.seaIce, 'sic', self.startdate, self.member, self.chunk) sic_handler = Utils.openCdf(sic_file) sic = np.asfortranarray(sic_handler.variables['sic'][:]) sic_handler.close() @@ -110,18 +111,24 @@ class Siasiesiv(Diagnostic): print ex self.send_file(self._extract_variable_and_rename(sit_file, result[4, :], 'sivols', '10^9 m3'), - Domains.seaIce, 'sivols', self.startdate, self.member, self.chunk, region=self.basin.fullname) + ModelingRealms.seaIce, 'sivols', self.startdate, self.member, self.chunk, + region=self.basin.fullname) self.send_file(self._extract_variable_and_rename(sit_file, result[5, :], 'siareas', '10^9 m2'), - Domains.seaIce, 'siareas', self.startdate, self.member, self.chunk, region=self.basin.fullname) + ModelingRealms.seaIce, 'siareas', self.startdate, self.member, self.chunk, + region=self.basin.fullname) self.send_file(self._extract_variable_and_rename(sit_file, result[7, :], 'siextents', '10^9 m2'), - Domains.seaIce, 'siextents', self.startdate, self.member, self.chunk, region=self.basin.fullname) + ModelingRealms.seaIce, 'siextents', self.startdate, self.member, self.chunk, + region=self.basin.fullname) self.send_file(self._extract_variable_and_rename(sit_file, result[0, :], 'sivoln', '10^9 m3'), - Domains.seaIce, 'sivoln', self.startdate, self.member, self.chunk, region=self.basin.fullname) + ModelingRealms.seaIce, 'sivoln', self.startdate, self.member, self.chunk, + region=self.basin.fullname) self.send_file(self._extract_variable_and_rename(sit_file, result[1, :], 'siarean', '10^9 m2'), - Domains.seaIce, 'siarean', self.startdate, self.member, self.chunk, region=self.basin.fullname) + ModelingRealms.seaIce, 'siarean', self.startdate, self.member, self.chunk, + region=self.basin.fullname) self.send_file(self._extract_variable_and_rename(sit_file, result[3, :], 'siextentn', '10^9 m2'), - Domains.seaIce, 'siextentn', self.startdate, self.member, self.chunk, region=self.basin.fullname) + ModelingRealms.seaIce, 'siextentn', self.startdate, self.member, self.chunk, + region=self.basin.fullname) @staticmethod def _extract_variable_and_rename(reference_file, values, cmor_name, units): @@ -130,15 +137,9 @@ class Siasiesiv(Diagnostic): os.remove(temp) handler = netCDF4.Dataset(temp, 'w') - # Create dimensions - handler.createDimension('time') - handler.createDimension('bnds', 2) - - # Copy time variable - - Utils.copy_variable(reference_handler, handler, 'time') - Utils.copy_variable(reference_handler, handler, 'time_bnds') - Utils.copy_variable(reference_handler, handler, 'leadtime') + Utils.copy_variable(reference_handler, handler, 'time', add_dimensions=True) + Utils.copy_variable(reference_handler, handler, 'time_bnds', add_dimensions=True, must_exist=False) + Utils.copy_variable(reference_handler, handler, 'leadtime', add_dimensions=True, must_exist=False) reference_handler.close() new_var = handler.createVariable(cmor_name, float, 'time', fill_value=1.0e20) diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 7783ab245f5ac14dec5a62f6fdbb8de87c5238d6..4564bb0aae96cd23d279796624a7b2802bfec0b4 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -1,9 +1,9 @@ # coding=utf-8 from earthdiagnostics import cdftools from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class VerticalMean(Diagnostic): @@ -64,23 +64,21 @@ class VerticalMean(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to average vertically') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticIntOption('min_depth', -1), + DiagnosticIntOption('max_depth', -1)) + options = cls.process_options(options, options_available) box = Box() - if num_options >= 2: - box.min_depth = float(options[2]) - if num_options >= 3: - box.max_depth = float(options[3]) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(VerticalMean(diags.data_manager, startdate, member, chunk, - variable, box)) + options['variable'], box)) return job_list def compute(self): @@ -88,7 +86,8 @@ class VerticalMean(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - variable_file = self.data_manager.get_file('ocean', self.variable, self.startdate, self.member, self.chunk) + variable_file = self.data_manager.get_file(ModelingRealms.ocean, self.variable, self.startdate, self.member, + self.chunk) handler = Utils.openCdf(variable_file) if self.box.min_depth is None: @@ -105,6 +104,6 @@ class VerticalMean(Diagnostic): cdftools.run('cdfvertmean', input=variable_file, output=temp, options=[self.variable, 'T', lev_min, lev_max, '-debug']) Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) - self.send_file(temp, Domains.ocean, self.variable + 'vmean', self.startdate, self.member, self.chunk, + self.send_file(temp, ModelingRealms.ocean, self.variable + 'vmean', self.startdate, self.member, self.chunk, box=self.box, rename_var='{0}_vert_mean'.format(self.variable)) diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 5d43196b262926a32c0da3f6911e662d5cebe943..3f280356c7e5e3f46e3b199b887cdd337357c1d1 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -1,9 +1,9 @@ # coding=utf-8 from earthdiagnostics import cdftools from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticFloatOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class VerticalMeanMeters(Diagnostic): @@ -63,21 +63,20 @@ class VerticalMeanMeters(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to average vertically') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticFloatOption('min_depth', -1), + DiagnosticFloatOption('max_depth', -1)) + options = cls.process_options(options, options_available) + box = Box(True) - if num_options >= 2: - box.min_depth = float(options[2]) - if num_options >= 3: - box.max_depth = float(options[3]) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(VerticalMeanMeters(diags.data_manager, startdate, member, chunk, variable, box)) + job_list.append(VerticalMeanMeters(diags.data_manager, startdate, member, chunk, options['variable'], box)) return job_list def compute(self): @@ -85,7 +84,8 @@ class VerticalMeanMeters(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - variable_file = self.data_manager.get_file('ocean', self.variable, self.startdate, self.member, self.chunk) + variable_file = self.data_manager.get_file(ModelingRealms.ocean, self.variable, self.startdate, self.member, + self.chunk) handler = Utils.openCdf(variable_file) if self.box.min_depth is None: @@ -102,5 +102,5 @@ class VerticalMeanMeters(Diagnostic): cdftools.run('cdfvertmean', input=variable_file, output=temp, options=[self.variable, 'T', lev_min, lev_max, '-debug']) Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) - self.send_file(temp, Domains.ocean, self.variable + 'vmean', self.startdate, self.member, self.chunk, + self.send_file(temp, ModelingRealms.ocean, self.variable + 'vmean', self.startdate, self.member, self.chunk, box=self.box, rename_var='{0}_vert_mean'.format(self.variable)) diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 5df474194dbcfc818ce629d0930a25fe6f5b5bfc..33fe83d967c36795be04cb9ec7d2c2786480f12b 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -1,9 +1,11 @@ # coding=utf-8 from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption, \ + DiagnosticIntOption +from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, Variable, VarType +from earthdiagnostics.variable_type import VariableType import numpy as np @@ -35,7 +37,7 @@ class ClimatologicalPercentile(Diagnostic): self.num_bins = num_bins self._bins = None self.percentiles = np.array([0.1, 0.25, 0.33, 0.5, 0.66, 0.75, 0.9]) - self.cmor_var = Variable.get_variable(variable, silent=True) + self.cmor_var = data_manager.variable_list.get_variable(variable, silent=True) if self.cmor_var and self.cmor_var.valid_max and self.cmor_var.valid_min: self.max_value = float(self.cmor_var.valid_max) self.min_value = float(self.cmor_var.valid_min) @@ -63,22 +65,15 @@ class ClimatologicalPercentile(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable (and its domain) and the leadtimes you want to compute ' - 'the percentiles on') - if num_options > 4: - raise Exception('You must specify between three and 4 parameters for the climatological percentiles') - domain = Domain(options[1]) - variable = options[2] - leadtimes = [int(i) for i in options[3].split('-')] - if num_options > 3: - num_bins = int(options[4]) - else: - num_bins = 2000 + options_available = (DiagnosticDomainOption('domain'), + DiagnosticOption('variable'), + DiagnosticListIntOption('leadtimes'), + DiagnosticIntOption('bins', 2000)) + options = cls.process_options(options, options_available) job_list = list() - job_list.append(ClimatologicalPercentile(diags.data_manager, domain, variable, leadtimes, num_bins, + job_list.append(ClimatologicalPercentile(diags.data_manager, options['domain'], options['variable'], + options['leadtimes'], options['bins'], diags.config.experiment)) return job_list @@ -115,8 +110,8 @@ class ClimatologicalPercentile(Diagnostic): handler.close() - self.send_file(temp, self.domain, self.variable + '_percentiles', None, None, frequency='clim', - rename_var='percent', vartype=VarType.STATISTIC) + self.send_file(temp, self.domain, self.variable + '_percentiles', None, None, frequency=Frequencies.climatology, + rename_var='percent', vartype=VariableType.STATISTIC) def _calculate_percentiles(self, distribution): Log.debug('Calculating percentiles') @@ -189,7 +184,8 @@ class ClimatologicalPercentile(Diagnostic): Log.debug('Discretizing realization {0}', realization) def calculate_histogram(time_series): - histogram, self._bins = np.histogram(time_series, bins=self.num_bins, range=(self.min_value, self.max_value)) + histogram, self._bins = np.histogram(time_series, bins=self.num_bins, + range=(self.min_value, self.max_value)) return histogram var = handler.variables[self.variable] diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index d746911e9879978f3471b3b1751e6f3b493741ed..45b7652c16da2689003c8e71dd8caa378a2ee37a 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -3,11 +3,11 @@ import shutil from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticIntOption +from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, VarType +from earthdiagnostics.variable_type import VariableType from calendar import monthrange -import numpy as np class MonthlyPercentile(Diagnostic): @@ -58,23 +58,15 @@ class MonthlyPercentile(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable (and its domain) to average vertically and ' - 'the percentil you want') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - - domain = Domain(options[1]) - variable = options[2] - percentile = int(options[3]) - if percentile < 0 or percentile > 100: - raise Exception('Percentile value must be in the interval [0,100]') + options_available = (DiagnosticOption('domain'), + DiagnosticDomainOption('variable'), + DiagnosticIntOption('percentile', None, 0, 100)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(MonthlyPercentile(diags.data_manager, startdate, member, chunk, - variable, domain, percentile)) + options['variable'], options['domain'], options['percentile'])) return job_list def compute(self): @@ -119,8 +111,9 @@ class MonthlyPercentile(Diagnostic): Log.debug('Computing percentile') Utils.cdo.monpctl(str(self.percentile), input=[variable_file, monmin_file, monmax_file], output=temp) Utils.rename_variable(temp, 'lev', 'ensemble', False, True) - self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, self.member, - self.chunk, frequency='mon', rename_var=self.variable, vartype=VarType.STATISTIC) + self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, + self.member, self.chunk, frequency=Frequencies.monthly, rename_var=self.variable, + vartype=VariableType.STATISTIC) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index ae8acb004b6ecbad6c41e67e0b00d3e3a4629d59..de75cd022e101d2dd1b2eef996399811872eeb62 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -1,12 +1,13 @@ # coding=utf-8 import os -from autosubmit.date.chunk_date_lib import parse_date, add_months, chunk_start_date, chunk_end_date, date2str +from autosubmit.date.chunk_date_lib import parse_date, add_months, chunk_start_date, chunk_end_date from earthdiagnostics.datamanager import DataManager, NetCDFFile from earthdiagnostics.utils import TempFile, Utils from datetime import datetime -from earthdiagnostics.variable import Variable, VarType +from earthdiagnostics.variable import VariableManager +from earthdiagnostics.variable_type import VariableType class THREDDSManager(DataManager): @@ -27,10 +28,11 @@ class THREDDSManager(DataManager): if not self.config.data_dir: raise Exception('Can not find model data') - if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size !=1 : + if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size != 1: raise Exception('For obs and recon data chunk_size must be always 1') - def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, vartype=VarType.MEAN): + # noinspection PyUnusedLocal + def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, vartype=VariableType.MEAN): aggregation_path = self.get_var_url(variable, startdate, frequency, None, vartype) startdate = parse_date(startdate) @@ -49,8 +51,42 @@ class THREDDSManager(DataManager): Utils.cdo.selmonth(selected_months, input=thredds_subset, output=temp) return temp + def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param domain: CMOR domain + :type domain: str + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) + + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', 'standard') + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', 'standard') + + thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) + return thredds_subset.check() + def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VarType.MEAN): + vartype=VariableType.MEAN): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy @@ -69,7 +105,9 @@ class THREDDSManager(DataManager): :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType :return: path to the copy created on the scratch folder :rtype: str """ @@ -81,10 +119,9 @@ class THREDDSManager(DataManager): thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) return thredds_subset.download() - def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False, vartype=VarType.MEAN): + diagnostic=None, cmorized=False, vartype=VariableType.MEAN): """ Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge with already existing ones as needed @@ -122,12 +159,13 @@ class THREDDSManager(DataManager): :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool - + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType """ if cmorized: raise ValueError('cmorized is not supported in THREDDS manager') original_var = var - cmor_var = Variable.get_variable(var) + cmor_var = VariableManager().get_variable(var) var = self._get_final_var_name(box, var) if rename_var and rename_var != var: @@ -139,7 +177,7 @@ class THREDDSManager(DataManager): frequency = self.config.frequency filepath = self.get_file_path(startdate, domain, var, frequency, vartype, box, grid) - netcdf_file = NetCDFFile(filepath, filetosend, domain, var, cmor_var) + netcdf_file = NetCDFFile(filepath, filetosend, domain, var, cmor_var, self.config.data_convention, region) if diagnostic: netcdf_file.add_diagnostic_history(diagnostic) else: @@ -157,13 +195,15 @@ class THREDDSManager(DataManager): :param var: file's var :type var: str :param frequency: file's frequency - :type frequency: str + :type frequency: Frequency :param box: file's box :type box: Box :param grid: file's grid :type grid: str :return: path to the file :rtype: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType """ if not frequency: frequency = self.config.frequency @@ -185,11 +225,11 @@ class THREDDSManager(DataManager): folder_path = os.path.join(self.config.data_dir, self.config.data_type, self.experiment.institute.lower(), self.experiment.model.lower(), - self.frequency_folder_name(frequency, vartype), + frequency.folder_name(vartype), var_folder) return folder_path - def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VarType.MEAN): + def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN): """ Ge a file containing all the data for one year for one variable :param domain: variable's domain @@ -206,19 +246,34 @@ class THREDDSManager(DataManager): :type grid: str :param box: variable's box :type box: Box + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType :return: """ aggregation_path = self.get_var_url(var, startdate, None, box, vartype) thredds_subset = THREDDSSubset(aggregation_path, var, datetime(year, 1, 1), datetime(year+1, 1, 1)) return thredds_subset.download() - def get_var_url(self, var, startdate, frequency, box, vartype): + """ + Get url for dataset + :param var: variable to retrieve + :type var: str + :param startdate: startdate to retrieve + :type startdate: str + :param frequency: frequency to get: + :type frequency: Frequency | None + :param box: box to get + :type box: Box + :param vartype: type of variable + :type vartype: VariableType + :return: + """ if not frequency: frequency = self.config.frequency var = self._get_final_var_name(box, var) full_path = os.path.join(self.server_url, 'dodsC', self.config.data_type, self.experiment.institute, - self.experiment.model, self.frequency_folder_name(frequency, vartype)) + self.experiment.model, frequency.folder_name(vartype)) if self.config.data_type == 'exp': full_path = os.path.join(full_path, var, self._get_file_name(startdate, var)) else: @@ -234,7 +289,7 @@ class THREDDSManager(DataManager): return '{0}.nc'.format(var) def link_file(self, domain, var, startdate, member, chunk=None, grid=None, box=None, - frequency=None, year=None, date_str=None, move_old=False, vartype=VarType.MEAN): + frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ Creates the link of a given file from the CMOR repository. @@ -258,6 +313,8 @@ class THREDDSManager(DataManager): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType :return: path to the copy created on the scratch folder :rtype: str """ @@ -291,6 +348,15 @@ class THREDDSSubset: url = self.get_url() return self._download_url(url) + def check(self): + # noinspection PyBroadException + try: + self.handler = Utils.openCdf(self.get_url()) + self.handler.close() + return True + except Exception: + return False + def _read_metadata(self): self.var_dimensions = self.handler.variables[self.var].dimensions for dimension in self.var_dimensions: @@ -317,9 +383,10 @@ class THREDDSSubset: time_end += 1 self.dimension_indexes['time'] = (time_start, time_end) - def _download_url(self, url): + @staticmethod + def _download_url(url): temp = TempFile.get() - Utils.execute_shell_command(['nccopy', url, temp]) + Utils.execute_shell_command(['nccopy', '-s', '-d', '-4', url, temp]) if not Utils.check_netcdf_file(temp): raise THREDDSError('Can not retrieve {0} from server'.format(url)) return temp @@ -337,7 +404,8 @@ class THREDDSSubset: return '{0}?{1}{2}'.format(self.thredds_path, dimensions_slice, var_slice) - def _get_slice_index(self, index_tuple): + @staticmethod + def _get_slice_index(index_tuple): return '[{0[0]}:1:{0[1]}]'.format(index_tuple) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 7e545c6d507cffdcdac456b70709e8ecbd26749f..f8248ad19ce3e28413f806d66d100c4ac99b32de 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -165,6 +165,7 @@ class Utils(object): original_handler.close() new_handler.close() + # noinspection PyPep8Naming @staticmethod def convert_to_ASCII_if_possible(string, encoding='ascii'): if isinstance(string, basestring): @@ -286,7 +287,8 @@ class Utils(object): Log.log.log(log_level, line) output.append(line) if process.returncode != 0: - raise Utils.ExecutionError('Error executing {0}\n Return code: {1}', ' '.join(command), process.returncode) + raise Utils.ExecutionError('Error executing {0}\n Return code: {1}'.format(' '.join(command), + process.returncode)) return output _cpu_count = None diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index faf7c74a601680593b1ea80c47952248249b1365..30ae1154e28e7ccc54fe402ab1a3a4d1658c8033 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -1,32 +1,38 @@ # coding=utf-8 import csv +import json +import openpyxl import os from autosubmit.config.log import Log from earthdiagnostics.constants import Basins +from earthdiagnostics.frequency import Frequency +from earthdiagnostics.modelingrealm import ModelingRealm -class Variable(object): - """ - Class to characterize a CMOR variable. It also contains the static method to make the match between thje original - name and the standard name. Requires cmor_table.csv to work. - """ - _dict_variables = None - - def __init__(self, line): - self.short_name = line[1].strip() - self.standard_name = line[2].strip() - self.long_name = line[3].strip() - self.domain = Domain(line[4].strip()) - self.basin = Basins.parse(line[5]) - self.units = line[6].strip() - self.valid_min = line[7].strip() - self.valid_max = line[8].strip() - self.grid = line[9].strip() - - @classmethod - def get_variable(cls, original_name, silent=False): +class VariableJsonException(Exception): + pass + + +class SingletonType(type): + def __call__(cls, *args): + try: + return cls.__instance + except AttributeError: + cls.__instance = super(SingletonType, cls).__call__(*args) + return cls.__instance + + +class VariableManager(object): + __metaclass__ = SingletonType + + def __init__(self): + self._cmor_tables_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cmor_tables') + self._aliases_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'variable_alias') + self._dict_variables = {} + + def get_variable(self, original_name, silent=False): """ Returns the cmor variable instance given a variable name @@ -38,93 +44,331 @@ class Variable(object): :rtype: Variable """ try: - return cls._dict_variables[original_name.lower()] + return self._dict_aliases[original_name.lower()][1] except KeyError: if not silent: Log.warning('Variable {0} is not defined in the CMOR table. Please add it'.format(original_name)) return None - @classmethod - def load_variables(cls): + def get_all_variables(self): + """ + Returns all variables + + :return: CMOR variable list + :rtype: set[Variable] + """ + return set(self._dict_variables.values()) + + def get_variable_and_alias(self, original_name, silent=False): + """ + Returns the cmor variable instance given a variable name + + :param original_name: original variable's name + :type original_name: str + :param silent: if True, omits log warning when variable is not found + :type silent: bool + :return: CMOR variable + :rtype: Variable + """ + try: + return self._dict_aliases[original_name.lower()] + except KeyError: + if not silent: + Log.warning('Variable {0} is not defined in the CMOR table. Please add it'.format(original_name)) + return None, None + + def load_variables(self, table_name): """ - Loads the cmor_table.csv and creates the variables dictionary + Loads the CMOR csv and creates the variables dictionary """ - Variable._dict_variables = dict() - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cmor_table.csv'), 'rb') as csvfile: + self._dict_variables = dict() + self._load_variable_list(table_name) + self._load_missing_defaults() + self._load_known_aliases(table_name) + self._construct_aliases_dict() + + def _load_variable_list(self, table_name): + + json_folder = self._get_json_folder(table_name) + if os.path.isdir(json_folder): + self._load_json(json_folder) + return + + xlsx_path = self._get_xlsx_path(table_name) + if os.path.isfile(xlsx_path): + self._load_xlsx(table_name) + return + + csv_path = self._get_csv_path(table_name) + if os.path.isfile(csv_path): + self._load_file(table_name) + return + + raise Exception('Data convention {0} unknown'.format(table_name)) + + def _get_csv_path(self, table_name): + csv_table_path = os.path.join(self._cmor_tables_folder, '{0}.csv'.format(table_name)) + return csv_table_path + + def _get_json_folder(self, table_name): + json_folder = os.path.join(self._cmor_tables_folder, '{0}/Tables'.format(table_name)) + return json_folder + + def _load_file(self, csv_table_path, default=False): + with open(self._get_csv_path(csv_table_path), 'rb') as csvfile: reader = csv.reader(csvfile, dialect='excel') for line in reader: if line[0] == 'Variable': continue - var = Variable(line) - if not var.short_name: + var = Variable() + var.parse_csv(line) + if not var.short_name or var.short_name.lower() in self._dict_variables: continue - for old_name in line[0].split(':'): - Variable._dict_variables[old_name.lower()] = var - Variable._dict_variables[var.short_name.lower()] = var + var.default = default + + self._dict_variables[var.short_name.lower()] = var + def _load_json(self, json_folder): + for file_name in os.listdir(json_folder): + if file_name in ('CMIP6_grids.json', 'CMIP6_formula_terms.json'): + continue + json_data = open(os.path.join(json_folder, file_name)).read() + data = json.loads(json_data) + if 'variable_entry' in data: + Log.debug('Parsing file {0}'.format(file_name)) + table = CMORTable(data['Header']['table_id'][6:], + Frequency(data['Header']['frequency']), + data['Header']['table_date']) + self._load_json_variables(data['variable_entry'], table) -class Domain(object): + def _load_json_variables(self, json_data, table): + for short_name in json_data.keys(): + if short_name.lower() in self._dict_variables: + self._dict_variables[short_name.lower()].tables.append(table) + continue + variable = Variable() + try: + variable.parse_json(json_data[short_name], short_name) + variable.tables.append(table) + self._dict_variables[variable.short_name.lower()] = variable + except VariableJsonException: + Log.error('Could not read variable {0}'.format(short_name)) - def __init__(self, domain_name): - domain_name = domain_name.lower() - if domain_name == 'seaice': - self.name = 'seaIce' - elif domain_name == 'landice': - self.name = 'landIce' - elif domain_name in ['ocean', 'atmos', 'land']: - self.name = domain_name + def _load_known_aliases(self, table_name): + self._load_alias_csv('default') + self._load_alias_csv(table_name) + + def _load_alias_csv(self, filename): + file_path = self._get_aliases_csv_path(filename) + if not os.path.isfile(file_path): + return + + with open(file_path, 'rb') as csvfile: + reader = csv.reader(csvfile, dialect='excel') + for line in reader: + if line[0] == 'Aliases': + continue + + aliases = line[0].split(':') + if line[1] not in aliases: + aliases.append(line[1]) + + cmor_vars = [] + for alias in aliases: + if alias.lower() in self._dict_variables: + cmor_vars.append(self._dict_variables[alias.lower()]) + if len(cmor_vars) == 0: + Log.error('Aliases {0} could not be mapped to any variable'.format(aliases)) + continue + elif len(cmor_vars) > 1: + non_default = [var for var in cmor_vars if not var.default] + if len(non_default) == 1: + for default in [var for var in cmor_vars if var not in non_default]: + del self._dict_variables[default.short_name.lower()] + cmor_vars = non_default + + else: + Log.error('Aliases {0} can be be mapped to multiple variables ' + '[{1}]'.format(aliases, ', '.join(map(str, cmor_vars)))) + continue + cmor_var = cmor_vars[0] + + for alias in aliases: + if alias != cmor_var.short_name and alias in self._dict_variables: + Log.error('Alias {0} for variable {1} is already a different ' + 'variable!'.format(alias, cmor_var.short_name)) + continue + alias_object = VariableAlias(alias) + if line[2]: + alias_object.basin = Basins.parse(line[2]) + if line[3]: + alias_object.grid = line[3] + cmor_var.known_aliases.append(alias_object) + + def _get_aliases_csv_path(self, filename): + csv_table_path = os.path.join(self._aliases_folder, '{0}.csv'.format(filename)) + return csv_table_path + + def _construct_aliases_dict(self): + self._dict_aliases = {} + for cmor_var_name in self._dict_variables: + cmor_var = self._dict_variables[cmor_var_name] + if cmor_var_name not in cmor_var.known_aliases: + cmor_var.known_aliases.append(VariableAlias(cmor_var_name)) + for alias in cmor_var.known_aliases: + self._dict_aliases[alias.alias] = (alias, cmor_var) + + def _get_xlsx_path(self, table_name): + xlsx_table_path = os.path.join(self._cmor_tables_folder, '{0}.xlsx'.format(table_name)) + return xlsx_table_path + + def _load_xlsx(self, table_name): + xlsx_table_path = os.path.join(self._cmor_tables_folder, '{0}.xlsx'.format(table_name)) + excel = openpyxl.load_workbook(xlsx_table_path, True) + + table_data = {} + data_sheet = excel.worksheets[0] + for row in data_sheet.rows: + if row[1].value in excel.sheetnames: + table_data[row[1].value] = (Frequency(row[2].value), row[4].value[4:-1]) + + for sheet_name in excel.sheetnames: + sheet = excel.get_sheet_by_name(sheet_name) + if sheet['A1'].value != 'Priority': + continue + table_frequency, table_date = table_data[sheet.title] + table = CMORTable(sheet.title, table_frequency, table_date) + for row in sheet.rows: + if row[0].value == 'Priority' or not row[5].value: + continue + + if row[5].value.lower() in self._dict_variables: + self._dict_variables[row[5].value.lower()].tables.append(table) + continue + + var = Variable() + var.priority = row[0].value + var.short_name = row[5].value + var.standard_name = row[6].value + var.long_name = row[1].value + + self._process_modelling_realm(var, row[12].value) + + var.units = row[2].value + var.tables.append(table) + self._dict_variables[var.short_name.lower()] = var + + def _process_modelling_realm(self, var, value): + if value is None: + value = '' + modelling_realm = value.split(' ') + if len(modelling_realm) > 1: + Log.warning('Multiple modeling realms assigned to variable {0}: {1}. ' + 'We wil use first ({1[0]}) as modelling realm'.format(var.short_name, modelling_realm)) + if not modelling_realm[0]: + Log.warning('Variable {0} has no modeling realm defined'.format(var.short_name)) else: - raise ValueError('Domain {0} not recognized!'.format(domain_name)) + var.domain = ModelingRealm(modelling_realm[0]) + + def _load_missing_defaults(self): + self._load_file('default', True) - def __eq__(self, other): - return other.__class__ == Domain and self.name == other.name + +class Variable(object): + """ + Class to characterize a CMOR variable. It also contains the static method to make the match between thje original + name and the standard name. Requires data _convetion to be available in cmor_tables to work. + """ def __str__(self): - return self.name + return '{0} ({1})'.format(self.standard_name, self.short_name) - def get_table_name(self, frequency): - """ - Returns the table name for a domain-frequency pair - :param frequency: variable's frequency - :type frequency: str - :return: variable's table name - :rtype: str - """ - if frequency in ('mon', 'clim'): - if self.name == 'seaIce': - prefix = 'OI' - elif self.name == 'landIce': - prefix = 'LI' - else: - prefix = self.name[0].upper() - table_name = prefix + frequency - elif frequency == '6hr': - table_name = '6hrPlev' + def __init__(self): + self.short_name = None + self.standard_name = None + self.long_name = None + self.units = None + self.valid_min = None + self.valid_max = None + self.grid = None + self.priority = None + self.default = False + self.domain = None + self.known_aliases = [] + self.tables = [] + + def parse_json(self, json_var, key): + + if 'out_name' in json_var: + self.short_name = json_var['out_name'] else: - table_name = 'day' - return table_name + raise VariableJsonException('Variable has no out name defined'.format(key)) + self.standard_name = json_var['standard_name'] + self.long_name = json_var['long_name'] + domain = json_var['modeling_realm'].split(' ') + if len(domain) > 1: + Log.warning('Multiple modeling realms assigned to variable {0}: {1}. ' + 'We wil use first ({1[0]}) as domain'.format(self.short_name, domain)) + if not domain[0]: + Log.warning('Variable {0} has no modeling realm defined'.format(self.short_name)) + else: + self.domain = ModelingRealm(domain[0]) -class Domains(object): - seaIce = Domain('seaice') - ocean = Domain('ocean') - landIce = Domain('landIce') - atmos = Domain('atmos') - land = Domain('land') + self.valid_min = json_var['valid_min'] + self.valid_max = json_var['valid_max'] + self.units = json_var['units'] + def parse_csv(self, var_line): + self.short_name = var_line[1].strip() + self.standard_name = var_line[2].strip() + self.long_name = var_line[3].strip() + self.domain = ModelingRealm(var_line[4].strip()) + self.basin = Basins.parse(var_line[5]) + self.units = var_line[6].strip() + self.valid_min = var_line[7].strip() + self.valid_max = var_line[8].strip() + self.grid = var_line[9].strip() + for table in var_line[10].strip().split(':'): + if table: + self.tables.append(table) -class VarType(object): - MEAN = 1 - STATISTIC = 2 + def get_table(self, frequency, data_convention): + for table in self.tables: + if table.frequency == frequency: + return table + table_name = self.domain.get_table_name(frequency, data_convention) + return CMORTable(table_name, frequency, 'December 2013') - @staticmethod - def to_str(vartype): - if vartype == VarType.MEAN: - return 'mean' - elif vartype == VarType.STATISTIC: - return 'statistics' - else: - raise ValueError('Variable type {0} not supported'.format(vartype)) + +class VariableAlias(object): + """ + Class to characterize a CMOR variable. It also contains the static method to make the match between thje original + name and the standard name. Requires data _convetion to be available in cmor_tables to work. + """ + + def __init__(self, alias): + self.alias = alias + self.basin = None + self.grid = None + + def __str__(self): + string = self.alias + if self.basin: + string += ' Basin: {0}'.format(self.basin) + if self.grid: + string += ' Grid: {0}'.format(self.grid) + return string + + +class CMORTable(object): + def __init__(self, name, frequency, date): + self.name = name + self.frequency = Frequency.parse(frequency) + self.date = date + + def __str__(self): + return self.name diff --git a/earthdiagnostics/variable_alias/cmip6.csv b/earthdiagnostics/variable_alias/cmip6.csv new file mode 100644 index 0000000000000000000000000000000000000000..6aee91f502a827957a7411b8657e52e8d0344f37 --- /dev/null +++ b/earthdiagnostics/variable_alias/cmip6.csv @@ -0,0 +1,3 @@ +Aliases,Shortname,Basin,Grid +iiceconc:soicecov:ileadfra,siconc,, +ci,siconc,,ifs diff --git a/earthdiagnostics/variable_alias/default.csv b/earthdiagnostics/variable_alias/default.csv new file mode 100644 index 0000000000000000000000000000000000000000..24683fe3fed92fba78ad1a7f9daedabc3db6ae72 --- /dev/null +++ b/earthdiagnostics/variable_alias/default.csv @@ -0,0 +1,294 @@ +Aliases,Shortname,Basin,Grid +iiceages:siage:iice_otd,ageice,, +al,al,, +bgfrcsal,bgfrcsal,, +bgfrctem,bgfrctem,, +bgfrcvol,bgfrcvol,, +bgheatco,bgheatco,, +bgsaline,bgsaline,, +bgsaltco,bgsaltco,, +bgtemper,bgtemper,, +bgvole3t,bgvole3t,, +bgvolssh,bgvolssh,, +bld,bld,, +iicebome:iocewflx,bmelt,, +sobowlin,bowlin,, +cc,cl,, +hcc,clh,, +lcc,cll,, +mcc,clm,, +ciwc,cli,, +tcc,clt,, +tcw,clwvi,, +iicedive:sidive,divice,, +e,evspsbl,, +fal,fal,, +sowaflep,fatmosocean,, +sowaflcd,fdilution,, +sophtldf,fhbasindif,, +iowaflup,ficeocean,, +sorunoff,friver,, +sowaflup,fupward,, +gwd,gwd,, +ibgheatco,hcicega,, +sbgheatco,hcsnga,, +heatc,heatc,, +sohtatl:hfnortha,hfbasin,Atl, +sohtind,hfbasin,Ind, +sohtipc,hfbasin,IndPac, +sohtpac,hfbasin,Pac, +sophtadv,hfbasinadv,, +sophteiv,hfbasinba,, +qt_oce:sohefldo:qt,hfds,, +slhf,hfls,, +sshf,hfss,, +sophtove,htovovrt,, +q,hus,, +soicealb,ialb,, +ibgfrcsfx,ibgfrcsfx,, +ibgfrcvol,ibgfrcvol,, +ibghfxbog,ibghfxbog,, +ibghfxbom,ibghfxbom,, +ibghfxdhc,ibghfxdhc,, +ibghfxdif,ibghfxdif,, +ibghfxdyn,ibghfxdyn,, +ibghfxin,ibghfxin,, +ibghfxopw,ibghfxopw,, +ibghfxout,ibghfxout,, +ibghfxres,ibghfxres,, +ibghfxsnw,ibghfxsnw,, +ibghfxspr,ibghfxspr,, +ibghfxsub,ibghfxsub,, +ibghfxsum,ibghfxsum,, +ibghfxthd,ibghfxthd,, +ibgsfxbog,ibgsfxbogga,, +ibgsfxbom,ibgsfxbomga,, +ibgsfxbri,ibgsfxbriga,, +ibgsfxdyn,ibgsfxdynga,, +ibgsfx,ibgsfxga,, +ibgsfxopw,ibgsfxopwga,, +ibgsfxres,ibgsfxresga,, +ibgsfxsni,ibgsfxsniga,, +ibgsfxsum,ibgsfxsumga,, +ibgvfxbog,ibgvfxbogga,, +ibgvfxbom,ibgvfxbomga,, +ibgvfxdyn,ibgvfxdynga,, +ibgvfx,ibgvfxga,, +ibgvfxopw,ibgvfxopwga,, +ibgvfxres,ibgvfxresga,, +ibgvfxsni,ibgvfxsniga,, +ibgvfxsnw,ibgvfxsnwga,, +ibgvfxspr,ibgvfxsprga,, +ibgvfxsub,ibgvfxsubga,, +ibgvfxsum,ibgvfxsumga,, +ibgvolgrm,ibgvolgrm,, +ibrinvol,ibrinvol,, +sibricat,ibrinvolcat,, +iicebopr,iicebopr,, +iicecolf,iicecolf,, +iicedypr,iicedypr,, +iice_etd,iiceetd,, +iicelapr,iicelapr,, +iicenflx,iicenflx,, +iicesflx,iicesflx,, +iiceshea,iiceshea,, +iicesipr,iicesipr,, +iicfsbri,iicfsbri,, +iicfseqv,iicfseqv,, +ioceflxb,ioceflxb,, +iocehebr,iocehebr,, +iocesafl,iocesafl,, +iocesflx,iocesflx,, +iocetflx,iocetflx,, +iocwnsfl,iocwnsfl,, +isstempe,isstempe,, +scmastot,masso,, +mldkz5,mldkz5,, +somxl010:mldr10_1,mlotst,, +swvl1,mrlsl1,, +swvl2,mrlsl2,, +swvl3,mrlsl3,, +swvl4,mrlsl4,, +ro,mrro,, +tp:precip,pr,, +cp,prc,, +lsp,prs,, +isnowpre,prsn,, +sf:snowpre,prsn,, +tcwv,prw,, +msl,psl,, +qns_ice,qnsice,, +qt_ice,qtice,, +strd,rlds,, +strc:str,rls,, +ttr,rlut,, +ttrc,rlutcs,, +ssrd,rsds,, +tsr,rsdt,, +soshfldo,rsntds,, +ssr,rss,, +ssrc,rsscs,, +tsrc,rsut,, +saltc,saltc,, +sosalflx,sfs,, +NArea,siarean,, +SArea,siareas,, +iice_itd:siconc_cat:siconcat,siccat,, +ibgarea,sicga,, +NExnsidc,siextentn,, +SExnsidc,siextents,, +iiceprod,sigr,, +iiceheco,siheco,, +ibgsaltco,sisaltcga,, +iicethic:sithic,sit,, +iice_hid:sithic_cat:sithicat,sitcat,, +iicetemp,sitemp,, +ibgtemper,sitempga,, +iicevelo:sivelo,sivelo,, +iicevelu:sivelu,sivelu,, +iicevelv:sivelv,sivelv,, +ibgvoltot,sivolga,, +sivoln:NVolume,sivoln,, +sivols:SVolume,sivols,, +sivolu,sivolu,, +sostatl,sltbasin,, +sostind,sltbasin,, +sostipc,sltbasin,, +sostpac,sltbasin,, +sopstadv,sltbasinadv,, +sopsteiv,sltbasinba,, +sopstldf,sltbasindif,, +sltnortha,sltnortha,, +sopstove,sltovovrt,, +zosalatl,sltzmean,Atl, +zosalglo,sltzmean,Glob, +zosalind,sltzmean,Ind, +zosalipc,sltzmean,IndPac, +zosalpac,sltzmean,Pac, +asn,snal,, +iice_hsd:snthicat,sndcat,, +isnoheco,snheco,, +sd,snld,, +smlt,snm,, +isnowthi,snthic,, +sbgvoltot,snvolga,, +snvolu,snvolu,, +vosaline:mean_3Dsosaline,so,, +scsaltot,soga,, +soleaeiw,soleaeiw,, +soleahtw,soleahtw,, +somixhgt,somixhgt,, +sosaline:isssalin:mean_sosaline,sos,, +sothedep,sothedep,, +src,src,, +zosrfatl,srfzmean,Atl, +zosrfglo,srfzmean,Glob, +zosrfind,srfzmean,Ind, +zosrfipc,srfzmean,IndPac, +zosrfpac,srfzmean,Pac, +rsn,srho,, +iicesali:iice_std,ssi,, +salincat,ssicat,, +ibgsaline,ssiga,, +iicestre,streng,, +so20chgt,t20d,, +t,ta,, +t2m,tas,, +mx2t,tasmax,, +mn2t,tasmin,, +ewss,tauu,, +utau_ice:iocestru:iicestru,strairx,, +sozotaux,tauuo,, +nsss,tauv,, +vtau_ice:iocestrv:iicestrv,strairy,, +sozotauy:sometauy,tauvo,, +d2m,tdps,, +votemper:mean_3Dsosstsst,thetao,, +sctemtot,thetaoga,, +iicesume,tmelt,, +sosstsst:mean_sosstsst,tos,, +sstk,tos,,ifs +tossq,tossq,, +zotematl,toszmean,Atl, +zotemglo,toszmean,Glob, +zotemind,toszmean,Ind, +zotemipc,toszmean,IndPac, +zotempac,toszmean,Pac, +skt,ts,, +iicesurt:soicetem:sistem,tsice,, +istl1,tsice,, +stl1,tsl1,, +stl2,tsl2,, +stl3,tsl3,, +stl4,tsl4,, +tsn,tsn,, +u,ua,, +u10m,uas,, +vozocrtx,uo,, +uos,uos,, +v,va,, +v10m,vas,, +vomecrty,vo,, +vos,vos,, +voddmavs,voddmavs,, +vozoeivu,voeivu,, +vomeeivv,voeivv,, +voveeivw,voeivz,, +scvoltot,volo,, +votkeavm,votkeavm,, +votkeavt,votkeavt,, +votkeevd,votkeevd,, +votkeevm,votkeevm,, +sobarstf,vsftbarot,, +zomsfatl,vsftmyz,Atl, +zomsfglo,vsftmyz,Glob, +zomsfind,vsftmyz,Ind, +zomsfipc:zomsfinp,vsftmyz,IndPac, +zomsfpac,vsftmyz,Pac, +zomsfeiv,vsftmyzba,, +w,wa,, +z,zg,, +vovecrtz,zo,, +sossheigh:sossheig:mean_sossheig,zos,, +scsshtot,zosga,, +scsshste,zossga,, +zossq,zossq,, +scsshtst,zostoga,, +ohc,heatc,, +ohcsum,heatcsum,, +ohcvmean,heatcvmean,, +transix,transix,, +transiy,transiy,, +windsp,sfcWind,, +vsfsit,vsfsit,, +sfdsi,sfdsi,, +hfsithermds,hfsithermds,, +u2o,uosq,, +v2o,vosq,, +vozomatr,umo,, +vomematr,vmo,, +sozohetr,hfx,, +somehetr,hfy,, +uto,uothetao,, +vto,vothetao,, +uso,uoso,, +vso,voso,, +wfo,wfo,, +emp_oce,evsmpr,, +emp_ice,evsmpr,, +qsr_oce,rsntds,, +qns_oce,rlds,, +qsr_ice,rsdssi,, +qns_ice,rldssi,, +sfx,sfx,, +taum,taum,, +zfull,zfull,, +zhalf,zhalf,, +pbo,pbo,, +thkcello,thkcello,, +ficeberg,ficeberg,, +wo,wo,, +w2o,wosq,, +difvho,difvho,, +vovematr,wmo,, +qtr_ice,qtr,, diff --git a/earthdiagnostics/variable_alias/primavera.csv b/earthdiagnostics/variable_alias/primavera.csv new file mode 100644 index 0000000000000000000000000000000000000000..23d010ad7e7045cdcef49588202a4b1d8d3ea231 --- /dev/null +++ b/earthdiagnostics/variable_alias/primavera.csv @@ -0,0 +1,3 @@ +Aliases,Shortname,Basin,Grid +iiceconc:siconc:soicecov:ileadfra,siconc,, +ci,siconc,,ifs \ No newline at end of file diff --git a/earthdiagnostics/variable_alias/specs.csv b/earthdiagnostics/variable_alias/specs.csv new file mode 100644 index 0000000000000000000000000000000000000000..1aebaab3c330db12114aedb4ac2198d944e6692a --- /dev/null +++ b/earthdiagnostics/variable_alias/specs.csv @@ -0,0 +1,3 @@ +Aliases,Shortname,Basin,Grid +siconc:soicecov,sic,, +ci,sic,,ifs \ No newline at end of file diff --git a/earthdiagnostics/variable_type.py b/earthdiagnostics/variable_type.py new file mode 100644 index 0000000000000000000000000000000000000000..4b3f17daa0b515582f1870cbbc7f05dacfa3379a --- /dev/null +++ b/earthdiagnostics/variable_type.py @@ -0,0 +1,13 @@ +# coding=utf-8 +class VariableType(object): + MEAN = 1 + STATISTIC = 2 + + @staticmethod + def to_str(vartype): + if vartype == VariableType.MEAN: + return 'mean' + elif vartype == VariableType.STATISTIC: + return 'statistics' + else: + raise ValueError('Variable type {0} not supported'.format(vartype)) diff --git a/launch_diags.sh b/launch_diags.sh index 561d2df2da477693aefcdf11b2be135f3c52a5a6..cdeaff799e92e8363c03ca386781417ffc1b43e1 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -10,12 +10,12 @@ set -xv PATH_TO_CONF_FILE=~/earthdiagnostics/diags.conf PATH_TO_DIAGNOSTICS=~/earthdiagnostics -PATH_TO_VIRTUALENV=/shared/earth/ClimatePrediction/EarthDiagnostics/bin +PATH_TO_VIRTUALENV=~jvegas/virtualenvs/diags/bin module purge module load NCO/4.5.4-foss-2015a -module load CDO/1.6.9-foss-2015a -module load CDFTOOLS/3.0a1-foss-2015a +module load CDO/1.7.2-foss-2015a +module load CDFTOOLS/3.0a2-foss-2015a source ${PATH_TO_VIRTUALENV}/activate diff --git a/setup.py b/setup.py index 17f4b07a9ae63ec51cd0ca6709eaac3e080e2598..bd67a0b34a4472916733eeeac2848b16cd2cd7da 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,8 @@ setup( author_email='javier.vegas@bsc.es', url='http://www.bsc.es/projects/earthscience/autosubmit/', keywords=['climate', 'weather', 'diagnostic'], - install_requires=['numpy', 'netCDF4', 'autosubmit', 'cdo', 'pygrib', 'nco', 'cfunits>=1.1.4', 'coverage', 'pyproj'], + install_requires=['numpy', 'netCDF4', 'autosubmit', 'cdo', 'pygrib', 'nco', 'cfunits>=1.1.4', 'coverage', 'pyproj', + 'openpyxl'], packages=find_packages(), include_package_data=True, scripts=['bin/earthdiags'] diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 7a15d906c9002de2c6185d56ed22c90d2d2af217..3c07e90196c414109ffe592d381dccdc45d463ff 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -1,9 +1,9 @@ # coding=utf-8 from test_data_manager import TestConversion -from test.unit.test_variable import TestVariable +# from test.unit.test_variable import TestVariable from test_constants import TestBasin, TestBasins from test_box import TestBox -from test_diagnostic import TestDiagnostic +from test_diagnostic import * from test_cdftools import TestCDFTools from test_utils import TestTempFile, TestUtils from test_psi import TestPsi diff --git a/test/unit/test_averagesection.py b/test/unit/test_averagesection.py index 01d32184a19c4686015f9aeb24e95347c860ea41..78554e5e495ab6b28a7d5c90e7d7537628abf1b7 100644 --- a/test/unit/test_averagesection.py +++ b/test/unit/test_averagesection.py @@ -5,7 +5,7 @@ from earthdiagnostics.box import Box from earthdiagnostics.ocean.averagesection import AverageSection from mock import Mock -from earthdiagnostics.variable import Domains, Domain +from earthdiagnostics.modelingrealm import ModelingRealms class TestAverageSection(TestCase): @@ -21,18 +21,22 @@ class TestAverageSection(TestCase): self.box.max_lon = 0 self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.psi = AverageSection(self.data_manager, '20000101', 1, 1, Domains.ocean, 'var', self.box) + self.psi = AverageSection(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', self.box) def test_generate_jobs(self): jobs = AverageSection.generate_jobs(self.diags, ['psi', 'var', '0', '0', '0', '0']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', self.box)) - self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', self.box)) + self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + self.box)) + self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + self.box)) jobs = AverageSection.generate_jobs(self.diags, ['psi', 'var', '0', '0', '0', '0', 'ocean']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', self.box)) - self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', self.box)) + self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + self.box)) + self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + self.box)) with self.assertRaises(Exception): AverageSection.generate_jobs(self.diags, ['psi']) diff --git a/test/unit/test_cdftools.py b/test/unit/test_cdftools.py index 0d0fadb85cec7192220fde39d2d72856b94976c4..9ebdc65f9289ea0a217f046800e258cde95904e6 100644 --- a/test/unit/test_cdftools.py +++ b/test/unit/test_cdftools.py @@ -14,14 +14,8 @@ class TestCDFTools(TestCase): # noinspection PyTypeChecker def test_run(self): + # noinspection PyUnusedLocal def mock_exists(path, access=None): - """ - Function for os.path.exists mock - :param path: path to check - :type path: str - :return: true if path does not start with 'bad' - :rtype: bool - """ return not os.path.basename(path.startswith('bad')) with mock.patch('os.path.exists') as exists_mock: diff --git a/test/unit/test_cutsection.py b/test/unit/test_cutsection.py index 6f4d4aa276b24d58fb267ddecef37ca03c64e6a6..8cfb3cdb4e4f6ffd56e58e064522d01c7784d7ec 100644 --- a/test/unit/test_cutsection.py +++ b/test/unit/test_cutsection.py @@ -5,7 +5,7 @@ from earthdiagnostics.box import Box from earthdiagnostics.ocean.cutsection import CutSection from mock import Mock -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class TestCutSection(TestCase): @@ -21,18 +21,22 @@ class TestCutSection(TestCase): self.box.max_lon = 0 self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.psi = CutSection(self.data_manager, '20000101', 1, 1, Domains.atmos, 'var', True, 0) + self.psi = CutSection(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', True, 0) def test_generate_jobs(self): jobs = CutSection.generate_jobs(self.diags, ['psi', 'var', 'true', '10']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', True, 10)) - self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', True, 10)) + self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + True, 10)) + self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + True, 10)) jobs = CutSection.generate_jobs(self.diags, ['psi', 'var', 'false', '0', 'atmos']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', False, 0)) - self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', False, 0)) + self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + False, 0)) + self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + False, 0)) with self.assertRaises(Exception): CutSection.generate_jobs(self.diags, ['psi']) diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index 117e8e22d77e32fb7ffd2bd37321db2b538362ba..31ba6fa9c1562d05ed669a618f76a8f0b8925583 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -1,7 +1,9 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import * from unittest import TestCase +from earthdiagnostics.modelingrealm import ModelingRealms + class TestDiagnostic(TestCase): @@ -46,3 +48,212 @@ class TestDiagnostic(TestCase): def test_repr(self): self.assertEquals(self.diagnostic.__repr__(), str(self.diagnostic)) + def test_empty_process_options(self): + self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) + + # def test_empty_process_options(self): + # self.assertEqual(len(cls.process_options(('diag_name', ), tuple())), 0) + + +class TestDiagnosticOption(TestCase): + + def test_good_default_value(self): + diag = DiagnosticOption('option', 'default') + self.assertEqual('default', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticOption('option') + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('default', diag.parse('')) + + def test_parse_value(self): + diag = DiagnosticOption('option') + self.assertEqual('value', diag.parse('value')) + + +class TestDiagnosticFloatOption(TestCase): + def test_float_default_value(self): + diag = DiagnosticFloatOption('option', 3.0) + self.assertEqual(3.0, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticFloatOption('option', '3') + self.assertEqual(3.0, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticFloatOption('option', 'default') + with self.assertRaises(ValueError): + self.assertEqual('default', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticFloatOption('option') + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('default', diag.parse('')) + + def test_parse_value(self): + diag = DiagnosticFloatOption('option') + self.assertEqual(3.25, diag.parse('3.25')) + + +class TestDiagnosticDomainOption(TestCase): + def test_domain_default_value(self): + diag = DiagnosticDomainOption('option', ModelingRealms.ocean) + self.assertEqual(ModelingRealms.ocean, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticDomainOption('option', 'atmos') + self.assertEqual(ModelingRealms.atmos, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticDomainOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticDomainOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticDomainOption('option') + self.assertEqual(ModelingRealms.seaIce, diag.parse('seaice')) + + +class TestDiagnosticIntOption(TestCase): + def test_int_default_value(self): + diag = DiagnosticIntOption('option', 3) + self.assertEqual(3, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticIntOption('option', '3') + self.assertEqual(3, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticIntOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticIntOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticIntOption('option') + self.assertEqual(3, diag.parse('3')) + + def test_parse_bad_value(self): + diag = DiagnosticIntOption('option') + with self.assertRaises(ValueError): + diag.parse('3.5') + + def test_good_low_limit(self): + diag = DiagnosticIntOption('option', None, 0) + self.assertEqual(1, diag.parse('1')) + + def test_bad_low_limit(self): + diag = DiagnosticIntOption('option', None, 0) + with self.assertRaises(DiagnosticOptionError): + diag.parse('-1') + + def test_good_high_limit(self): + diag = DiagnosticIntOption('option', None, None, 0) + self.assertEqual(-1, diag.parse('-1')) + + def test_bad_high_limit(self): + diag = DiagnosticIntOption('option', None, None, 0) + with self.assertRaises(DiagnosticOptionError): + diag.parse('1') + + +class TestDiagnosticBoolOption(TestCase): + def test_bool_default_value(self): + diag = DiagnosticBoolOption('option', True) + self.assertEqual(True, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticBoolOption('option', 'False') + self.assertEqual(False, diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticBoolOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_True(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('true')) + + def test_parse_true(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('true')) + + def test_parse_t(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('t')) + + def test_parse_yes(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('YES')) + + def test_parse_bad_value(self): + diag = DiagnosticBoolOption('option') + self.assertFalse(diag.parse('3.5')) + + +class TestDiagnosticComplexStrOption(TestCase): + def test_complex_default_value(self): + diag = DiagnosticComplexStrOption('option', 'default&.str&;&.working') + self.assertEqual('default str, working', diag.parse('')) + + def test_simple_default_value(self): + diag = DiagnosticComplexStrOption('default str, working', 'default str, working') + self.assertEqual('default str, working', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticComplexStrOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticComplexStrOption('option') + self.assertEqual('complex string, for testing', diag.parse('complex&.string&;&.for&.testing')) + + +class TestDiagnosticListIntOption(TestCase): + def test_tuple_default_value(self): + diag = DiagnosticListIntOption('option', (3,)) + self.assertEqual((3,), diag.parse('')) + + def test_list_default_value(self): + diag = DiagnosticListIntOption('option', [3]) + self.assertEqual([3], diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticListIntOption('option', '3-4') + self.assertEqual([3, 4], diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticListIntOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticListIntOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticListIntOption('option') + self.assertEqual([3, 2], diag.parse('3-2')) + + def test_parse_single_value(self): + diag = DiagnosticListIntOption('option') + self.assertEqual([3], diag.parse('3')) + + def test_parse_bad_value(self): + diag = DiagnosticListIntOption('option') + with self.assertRaises(ValueError): + diag.parse('3.5') + + diff --git a/test/unit/test_interpolate.py b/test/unit/test_interpolate.py index 480a230d86ea5bf6f58a9db10a4559f09ca4ba66..1aa264cf24c8e53009805abc900cb94e3aeb0645 100644 --- a/test/unit/test_interpolate.py +++ b/test/unit/test_interpolate.py @@ -4,7 +4,7 @@ from unittest import TestCase from earthdiagnostics.ocean.interpolate import Interpolate from mock import Mock -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class TestInterpolate(TestCase): @@ -17,29 +17,29 @@ class TestInterpolate(TestCase): self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.diags.config.experiment.model_version = 'model_version' - self.interpolate = Interpolate(self.data_manager, '20000101', 1, 1, Domains.atmos, 'var', 'grid', + self.interpolate = Interpolate(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', 'grid', 'model_version', False) def test_generate_jobs(self): jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', 'grid', + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'grid', 'model_version', False)) - self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', 'grid', + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'grid', 'model_version', False)) jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', 'grid', + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'model_version', False)) - self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', 'grid', + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', 'model_version', False)) jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', 'grid', + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'model_version', True)) - self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', 'grid', + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', 'model_version', True)) with self.assertRaises(Exception): diff --git a/test/unit/test_maxmoc.py b/test/unit/test_maxmoc.py index f31c58222b182c49d5e7635e0c0d708eb48be17e..99c2fca6da18126d818ec305728a900ef5d4bb1b 100644 --- a/test/unit/test_maxmoc.py +++ b/test/unit/test_maxmoc.py @@ -23,8 +23,8 @@ class TestMaxMoc(TestCase): def test_generate_jobs(self): self.diags = Mock() self.diags.model_version = 'model_version' - self.diags.startdates = ('20010101',) - self.diags.members = (0,) + self.diags.config.experiment.startdates = ('20010101',) + self.diags.config.experiment.members = (0,) self.diags.config.experiment.get_full_years.return_value = (2000, 2001) jobs = MaxMoc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0']) diff --git a/test/unit/test_monthlymean.py b/test/unit/test_monthlymean.py index 128d411fe59f0c68201838df9c6a2e12db53d32b..91e42a5387c9cf2df305f0f31321aa52caa3cd5d 100644 --- a/test/unit/test_monthlymean.py +++ b/test/unit/test_monthlymean.py @@ -2,10 +2,11 @@ from unittest import TestCase from earthdiagnostics.box import Box +from earthdiagnostics.frequency import Frequencies from earthdiagnostics.general.monthlymean import MonthlyMean from mock import Mock -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class TestMonthlyMean(TestCase): @@ -21,26 +22,30 @@ class TestMonthlyMean(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 - self.mixed = MonthlyMean(self.data_manager, '20000101', 1, 1, Domains.ocean, 'var', 'freq', '') + self.mixed = MonthlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') def test_generate_jobs(self): jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'ocean']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', 'day', '')) - self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', 'day', '')) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + Frequencies.daily, '')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + Frequencies.daily, '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'atmos', 'freq']) + jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'atmos', 'monthly']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', 'freq', '')) - self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', 'freq', '')) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + Frequencies.monthly, '')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + Frequencies.monthly, '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'seaice', 'freq', 'grid']) + jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'seaice', 'mon', 'grid']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, Domains.seaIce, 'var', 'freq', - 'grid')) - self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, Domains.seaIce, 'var', 'freq', - 'grid')) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', + Frequencies.monthly, 'grid')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, 'var', + Frequencies.monthly, 'grid')) with self.assertRaises(Exception): MonthlyMean.generate_jobs(self.diags, ['psi']) diff --git a/test/unit/test_rewrite.py b/test/unit/test_rewrite.py index 13adc6857340a57bfd0de0c65a6d021897d90537..f125947f4c2f82c3c15eb5d0b0cb5e8360419f56 100644 --- a/test/unit/test_rewrite.py +++ b/test/unit/test_rewrite.py @@ -5,7 +5,7 @@ from earthdiagnostics.box import Box from earthdiagnostics.general.rewrite import Rewrite from mock import Mock -from earthdiagnostics.variable import Domains +from earthdiagnostics.modelingrealm import ModelingRealms class TestRewrite(TestCase): @@ -21,19 +21,19 @@ class TestRewrite(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 - self.mixed = Rewrite(self.data_manager, '20000101', 1, 1, Domains.atmos, 'var', 'grid') + self.mixed = Rewrite(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', 'grid') def test_generate_jobs(self): jobs = Rewrite.generate_jobs(self.diags, ['psi', 'var', 'atmos']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', 'original')) - self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', 'original')) + self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'original')) + self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'original')) jobs = Rewrite.generate_jobs(self.diags, ['psi', 'var', 'ocean', 'grid']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', 'grid')) - self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', 'grid')) + self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'grid')) + self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'grid')) with self.assertRaises(Exception): Rewrite.generate_jobs(self.diags, ['psi']) diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index ffc3d037acb845a4ed1ef315849029ab053400fe..7f3e0b8472157c518f517af25cbafdb1a4812bf5 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -1,28 +1,29 @@ # coding=utf-8 -from unittest import TestCase +# from unittest import TestCase +# +# from earthdiagnostics.variable import Variable +# from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.variable import Variable, Domains - -class TestVariable(TestCase): - - def test__init__(self): - variable = Variable('alias:alias2,name,standard_name,long_name,ocean,basin,units,' - 'valid_min,valid_max,grid'.split(',')) - self.assertEqual(variable.short_name, 'name') - self.assertEqual(variable.standard_name, 'standard_name') - self.assertEqual(variable.long_name, 'long_name') - self.assertEqual(variable.domain, Domains.ocean) - self.assertEqual(variable.basin, None) - self.assertEqual(variable.units, 'units') - self.assertEqual(variable.valid_min, 'valid_min') - self.assertEqual(variable.valid_max, 'valid_max') - self.assertEqual(variable.grid, 'grid') - - def test_get_variable(self): - Variable._dict_variables = dict() - variable = Variable('alias:alias2,name,standard_name,long_name,atmos,basin,units,valid_min,' - 'valid_max,grid'.split(',')) - Variable._dict_variables['var'] = variable - self.assertIs(Variable.get_variable('var'), variable) - self.assertIsNone(Variable.get_variable('novar')) +# class TestVariable(TestCase): +# +# def test__init__(self): +# variable = Variable('alias:alias2,name,standard_name,long_name,ocean,basin,units,' +# 'valid_min,valid_max,grid'.split(',')) +# self.assertEqual(variable.short_name, 'name') +# self.assertEqual(variable.standard_name, 'standard_name') +# self.assertEqual(variable.long_name, 'long_name') +# self.assertEqual(variable.domain, Domains.ocean) +# self.assertEqual(variable.basin, None) +# self.assertEqual(variable.units, 'units') +# self.assertEqual(variable.valid_min, 'valid_min') +# self.assertEqual(variable.valid_max, 'valid_max') +# self.assertEqual(variable.grid, 'grid') +# +# def test_get_variable(self): +# Variable._dict_variables = dict() +# variable = Variable('alias:alias2,name,standard_name,long_name,atmos,basin,units,valid_min,' +# 'valid_max,grid'.split(',')) +# Variable._dict_variables['var'] = variable +# self.assertIs(Variable.get_variable('var'), variable) +# self.assertIsNone(Variable.get_variable('novar'))