From fab7addcc939e35766b4ca79ae7b3b520b8f718e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 20 Nov 2017 13:33:10 +0100 Subject: [PATCH 001/168] Codacy config --- .codacy.yml | 36 +++++ .editorconfig | 26 ++++ .prospector.yml | 28 ++++ .pylintrc | 407 ++++++++++++++++++++++++++++++++++++++++++++++++ README | 3 +- environment.yml | 30 ++++ 6 files changed, 529 insertions(+), 1 deletion(-) create mode 100644 .codacy.yml create mode 100644 .editorconfig create mode 100644 .prospector.yml create mode 100644 .pylintrc create mode 100644 environment.yml diff --git a/.codacy.yml b/.codacy.yml new file mode 100644 index 00000000..3dc3bb8c --- /dev/null +++ b/.codacy.yml @@ -0,0 +1,36 @@ +# codacy configuration file + +--- + +engines: + coverage: + enabled: true + exclude_paths: [ + 'tests', + ] + metrics: + enabled: true + duplication: + enabled: true + prospector: + enabled: true + pylint: + enabled: true + python_version: 3 + +exclude_paths: [ + 'doc/sphinx/**', + 'esmvaltool/doc/sphinx/**', + # cmor tables + 'esmvaltool/interface_scripts/cmip*-cmor-tables/**', + # old stuff + 'backend/**', + 'diag_scripts/**', + 'interface_data/**', + 'interface_scripts/**', + 'main.py', + 'nml/**', + 'plot_scripts/**', + 'reformat_scripts/**', + 'variable_defs/**', +] diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..2a7a00f3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# EditorConfig is awesome: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +# Set default charset +charset = utf-8 + +# Matches multiple files with brace expansion notation + +# 4 space indentation +[*.{py,java,r,R}] +indent_size = 4 + +# 2 space indentation +[*.{js,json,yml,html,xml,ncl}] +indent_size = 2 + +[*.{md,Rmd}] +trim_trailing_whitespace = false + diff --git a/.prospector.yml b/.prospector.yml new file mode 100644 index 00000000..58d4946d --- /dev/null +++ b/.prospector.yml @@ -0,0 +1,28 @@ +# prospector configuration file + +--- + +output-format: grouped + +strictness: veryhigh +doc-warnings: true +test-warnings: true +member-warnings: false + +pyroma: + run: true + +pep8: + full: true + +pep257: + # see http://pep257.readthedocs.io/en/latest/error_codes.html + disable: [ + # For short descriptions it makes sense not to end with a period: + D400, # First line should end with a period + # Disable because not part of PEP257 official convention: + D203, # 1 blank line required before class docstring + D212, # Multi-line docstring summary should start at the first line + D213, # Multi-line docstring summary should start at the second line + D404, # First word of the docstring should not be This + ] diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..4094c1f4 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,407 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=79 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format=LF + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,FIX-ME,XXX,TODO + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_,logger + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=yes + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec,optparse + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/README b/README index f247d009..79c3a704 100644 --- a/README +++ b/README @@ -1,3 +1,4 @@ +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Grade) This tool is a set of diagnostics used at BSC-ES department for NEMO and EC-EARTH models postprocessing. They are based on CDO, NCO and CDFTOOLS 3.0. For CDFTOOLS, a custom build is required. @@ -9,4 +10,4 @@ Check the Earth Diagnostics documentation in PDF format in EarthDiagnostics.pdf CONTACT ======= -For any doubts or suggestions, contact javier.vegas@bsc.es \ No newline at end of file +For any doubts or suggestions, contact javier.vegas@bsc.es diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..3a3ca170 --- /dev/null +++ b/environment.yml @@ -0,0 +1,30 @@ +--- + +name: earthdiagnostics +channels: +- conda-forge +#- defaults # in case of missing packages +#- Clyde_Fare #for scientific python + + +dependencies: +- iris +- netcdf4 +- numpy +- cdo +- nco +- python-cdo +- coverage +- pygrib +- psutil +- six +- cf_units +- openpyxl +- mock +- cmake + +- pip: + - bscearth.utils + - futures + - nco + - exrex -- GitLab From 9ee9ca9dc7c4dfe921d3985899a5a938eb69fc39 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 20 Nov 2017 14:25:07 +0100 Subject: [PATCH 002/168] fixed codacy config --- .codacy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codacy.yml b/.codacy.yml index 3dc3bb8c..c0f0e505 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -16,7 +16,7 @@ engines: enabled: true pylint: enabled: true - python_version: 3 + python_version: 2 exclude_paths: [ 'doc/sphinx/**', -- GitLab From 87875b93307937098a43fa98ffe914a3c8eba824 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 20 Nov 2017 14:26:52 +0100 Subject: [PATCH 003/168] Changed readme extension --- README => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.md (100%) diff --git a/README b/README.md similarity index 100% rename from README rename to README.md -- GitLab From 70992937cd85ffccf6bcc8d72b4017eacfbd10a2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 20 Nov 2017 14:28:15 +0100 Subject: [PATCH 004/168] Updated readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 79c3a704..b8f950dd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Codacy Badge](https://api.codacy.com/project/badge/Grade/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Grade) + This tool is a set of diagnostics used at BSC-ES department for NEMO and EC-EARTH models postprocessing. They are based on CDO, NCO and CDFTOOLS 3.0. For CDFTOOLS, a custom build is required. @@ -10,4 +11,4 @@ Check the Earth Diagnostics documentation in PDF format in EarthDiagnostics.pdf CONTACT ======= -For any doubts or suggestions, contact javier.vegas@bsc.es +For any doubts or suggestions, contact javier.vegas[at]bsc.es -- GitLab From ae60b16e98ce97011eb41d8dac45c988219a46ab Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 20 Nov 2017 14:31:43 +0100 Subject: [PATCH 005/168] Updated max line length for pylint --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 4094c1f4..db7741b9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -99,7 +99,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme [FORMAT] # Maximum number of characters on a single line. -max-line-length=79 +max-line-length=120 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ -- GitLab From 5803bcc31b875d64389226954b4ff08805973d59 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Mon, 20 Nov 2017 16:00:27 +0100 Subject: [PATCH 006/168] Fixed environment yml --- earthdiagnostics/ocean/siasiesiv.py | 3 ++- environment.yml | 5 ++--- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 21adfc0f..bd2c41c8 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -2,7 +2,7 @@ import os # noinspection PyUnresolvedReferences -import earthdiagnostics.cdftoolspython as cdftoolspython + import netCDF4 import numpy as np from bscearth.utils.log import Log @@ -117,6 +117,7 @@ class Siasiesiv(Diagnostic): """ Runs the diagnostic """ + import earthdiagnostics.cdftoolspython as cdftoolspython sit_handler = Utils.openCdf(self.sit.local_file) sit = np.asfortranarray(sit_handler.variables[self.sit_varname][:]) timesteps = sit_handler.dimensions['time'].size diff --git a/environment.yml b/environment.yml index 3a3ca170..8de32f66 100644 --- a/environment.yml +++ b/environment.yml @@ -3,9 +3,6 @@ name: earthdiagnostics channels: - conda-forge -#- defaults # in case of missing packages -#- Clyde_Fare #for scientific python - dependencies: - iris @@ -22,9 +19,11 @@ dependencies: - openpyxl - mock - cmake +- cfunits - pip: - bscearth.utils - futures - nco - exrex + - xxhash diff --git a/setup.py b/setup.py index 0e68a089..a9ff71b6 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( keywords=['climate', 'weather', 'diagnostic'], setup_requires=['pyproj'], install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo>=1.3.4', 'nco>=0.0.3', 'iris>=1.12.0', 'coverage', - 'pygrib', 'openpyxl', 'mock', 'futures', 'cf_units', 'cfunits', 'xxhash', 'six', 'psutil', + 'pygrib', 'openpyxl', 'mock', 'futures', 'cf_units', 'xxhash', 'six', 'psutil', 'exrex'], packages=find_packages(), include_package_data=True, -- GitLab From bdaaac2579a7f171e5a311d9250b6904a12aca31 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Mon, 20 Nov 2017 18:21:24 +0100 Subject: [PATCH 007/168] Setting automated tests --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..61b71586 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,8 @@ +test: + script: + - module purge + - module load Miniconda2 + - conda env update -n environment.yml + - source activate earthdiagnostics + - unittest test + -- GitLab From 9bf7ae39f4d17b9abd8770ccd93f2299bab304df Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Mon, 20 Nov 2017 18:29:55 +0100 Subject: [PATCH 008/168] Added coverage support for codacy --- .gitlab-ci.yml | 6 ++++-- environment.yml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61b71586..74c23b89 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,10 @@ test: script: - - module purge - - module load Miniconda2 + - echo $USER - conda env update -n environment.yml + - pip install codacy-coverage - source activate earthdiagnostics - unittest test + - coverage xml + - python-codacy-coverage -r coverage.xml diff --git a/environment.yml b/environment.yml index 8de32f66..772ab619 100644 --- a/environment.yml +++ b/environment.yml @@ -20,6 +20,7 @@ dependencies: - mock - cmake - cfunits +- coverage - pip: - bscearth.utils -- GitLab From dcc9c8bac4f964d6a36db15918496b7523eed2cf Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 09:48:34 +0100 Subject: [PATCH 009/168] Update to gitlab-runner --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74c23b89..9085bfeb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ test: script: - echo $USER + - echo $PATH - conda env update -n environment.yml - pip install codacy-coverage - source activate earthdiagnostics -- GitLab From 99ce3bd09954b7f662c757a0620eadab295ba2c8 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 09:52:17 +0100 Subject: [PATCH 010/168] Update to gitlab-runner --- .gitlab-ci.yml | 2 +- environment.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9085bfeb..0b55374d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,9 @@ test: script: - echo $USER + - export PATH="$HOME/miniconda2/bin:$PATH" - echo $PATH - conda env update -n environment.yml - - pip install codacy-coverage - source activate earthdiagnostics - unittest test - coverage xml diff --git a/environment.yml b/environment.yml index 772ab619..4fa44d13 100644 --- a/environment.yml +++ b/environment.yml @@ -28,3 +28,4 @@ dependencies: - nco - exrex - xxhash + - codacy-coverage -- GitLab From bec61fce46b36e6638d25117d684d2b281dacd0d Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 10:08:22 +0100 Subject: [PATCH 011/168] Update to gitlab-runner --- .coveragerc | 2 ++ .gitlab-ci.yml | 2 +- .prospector.yml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..398ff08a --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +branch = True diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b55374d..987d6ec3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ test: - echo $PATH - conda env update -n environment.yml - source activate earthdiagnostics - - unittest test + - echo $PWD - coverage xml - python-codacy-coverage -r coverage.xml diff --git a/.prospector.yml b/.prospector.yml index 58d4946d..b5652ca3 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -25,4 +25,5 @@ pep257: D212, # Multi-line docstring summary should start at the first line D213, # Multi-line docstring summary should start at the second line D404, # First word of the docstring should not be This + E501, # Line-length, already controlled by pylint ] -- GitLab From d713e456ea7345d7aee0d784bc92513e11a6938b Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 10:11:29 +0100 Subject: [PATCH 012/168] Fixed environment error --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 987d6ec3..27e0c7d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ test: - echo $USER - export PATH="$HOME/miniconda2/bin:$PATH" - echo $PATH - - conda env update -n environment.yml + - conda env update -f environment.yml - source activate earthdiagnostics - echo $PWD - coverage xml -- GitLab From 6b6d5da9ae1b592f7d87dd4946ae7bea3a5fe9b5 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 10:18:16 +0100 Subject: [PATCH 013/168] Getting gitlabb-ci to work --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27e0c7d7..f0b6fe74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ test: - conda env update -f environment.yml - source activate earthdiagnostics - echo $PWD + - coverage run - coverage xml - python-codacy-coverage -r coverage.xml -- GitLab From 08e008b56c8e19ba404133ebb908330386f88107 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 10:30:47 +0100 Subject: [PATCH 014/168] Getting gitlabb-ci to work --- .coveragerc | 3 +++ .gitlab-ci.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 398ff08a..e41ecaeb 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,5 @@ [run] branch = True +source = earthdiagnostics + + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0b6fe74..cc29806a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ test: - conda env update -f environment.yml - source activate earthdiagnostics - echo $PWD - - coverage run + - coverage run -m unittest discover - coverage xml - python-codacy-coverage -r coverage.xml -- GitLab From b08e625c68abd24437a6947ffcc85852a3f5ba74 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 11:36:00 +0100 Subject: [PATCH 015/168] Fixed tests --- test/unit/general/test_simplify_dimensions.py | 15 ++++++++------- test/unit/general/test_verticalmeanmetersiris.py | 15 ++++++++------- test/unit/ocean/test_interpolatecdo.py | 6 ++++-- test/unit/ocean/test_siasiesiv.py | 3 ++- test/unit/statistics/test_daysoverpercentile.py | 5 +++-- test/unit/test_config.py | 7 +------ test/unit/test_diagnostic.py | 2 +- 7 files changed, 27 insertions(+), 26 deletions(-) diff --git a/test/unit/general/test_simplify_dimensions.py b/test/unit/general/test_simplify_dimensions.py index 429ad6f2..48ac6f7c 100644 --- a/test/unit/general/test_simplify_dimensions.py +++ b/test/unit/general/test_simplify_dimensions.py @@ -19,6 +19,7 @@ class TestSimplifyDimensions(TestCase): self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.diags.config.experiment.startdates = ['20010101', ] self.diags.config.frequency = Frequencies.monthly + self.diags.config.data_convention = 'convention' self.box = Box() self.box.min_depth = 0 @@ -32,24 +33,24 @@ class TestSimplifyDimensions(TestCase): jobs = SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], SimplifyDimensions(self.data_manager, '20010101', 0, 0, - ModelingRealms.atmos, 'var', '')) + ModelingRealms.atmos, 'var', '', 'convention')) self.assertEqual(jobs[1], SimplifyDimensions(self.data_manager, '20010101', 0, 1, - ModelingRealms.atmos, 'var', '')) + ModelingRealms.atmos, 'var', '', 'convention')) jobs = SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], SimplifyDimensions(self.data_manager, '20010101', 0, 0, - ModelingRealms.atmos, 'var', 'grid')) + ModelingRealms.atmos, 'var', 'grid', 'convention')) self.assertEqual(jobs[1], SimplifyDimensions(self.data_manager, '20010101', 0, 1, - ModelingRealms.atmos, 'var', 'grid')) + ModelingRealms.atmos, 'var', 'grid', 'convention')) with self.assertRaises(DiagnosticOptionError): SimplifyDimensions.generate_jobs(self.diags, ['diagnostic']) - with self.assertRaises(DiagnosticOptionError): - SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid', 'extra']) + with self.assertRaises(DiagnosticOptionError): + SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid', 'extra']) def test_str(self): - mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid') + mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'convention') self.assertEquals(str(mixed), 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Grid: grid') diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py index cd2876fe..179b54fa 100644 --- a/test/unit/general/test_verticalmeanmetersiris.py +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -1,7 +1,7 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError from earthdiagnostics.box import Box from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris from earthdiagnostics.frequency import Frequencies @@ -27,14 +27,14 @@ class TestVerticalMeanMetersIris(TestCase): def fake_parse(self, value): if not value: raise DiagnosticOptionError - return value + return [value] - @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) def test_generate_jobs(self): box = Box(True) - jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var']) + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box)) @@ -45,14 +45,14 @@ class TestVerticalMeanMetersIris(TestCase): box.min_depth = 0 box.max_depth = 100 - jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box)) self.assertEqual(jobs[1], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', box)) - jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100', 'seaIce']) + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', box)) @@ -63,7 +63,8 @@ class TestVerticalMeanMetersIris(TestCase): VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(DiagnosticOptionError): - VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100', 'seaIce', 'extra']) + VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', '0', '100', 'seaIce', + 'extra']) def test_str(self): box = Box(True) diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py index 04b08552..ba238f28 100644 --- a/test/unit/ocean/test_interpolatecdo.py +++ b/test/unit/ocean/test_interpolatecdo.py @@ -24,9 +24,11 @@ class TestInterpolate(TestCase): raise DiagnosticOptionError return value.split('-') - @patch('earthdiagnostics.ocean.interpolatecdo.InterpolateCDO._compute_weights') + @patch('earthdiagnostics.ocean.interpolatecdo.InterpolateCDO.compute_weights') + @patch('earthdiagnostics.ocean.interpolatecdo.InterpolateCDO.get_sample_grid_file') @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) - def test_generate_jobs(self, mock_weights): + @patch('os.remove') + def test_generate_jobs(self, mock_weights, mock_grid_file, mock_remove): mock_weights.return_value = None jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var']) diff --git a/test/unit/ocean/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py index 00f7c68c..2457b72d 100644 --- a/test/unit/ocean/test_siasiesiv.py +++ b/test/unit/ocean/test_siasiesiv.py @@ -15,7 +15,8 @@ class TestSiasiesiv(TestCase): self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.mask = Mock() - self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask) + self.var_manager = Mock() + self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask, self.var_manager) def test_str(self): self.assertEquals(str(self.psi), 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global') diff --git a/test/unit/statistics/test_daysoverpercentile.py b/test/unit/statistics/test_daysoverpercentile.py index cc225cd6..26e0506a 100644 --- a/test/unit/statistics/test_daysoverpercentile.py +++ b/test/unit/statistics/test_daysoverpercentile.py @@ -13,14 +13,15 @@ class TestDaysOverPercentile(TestCase): self.data_manager = Mock() self.diags = Mock() self.diags.config.experiment.get_chunk_list.return_value = (('20011101', 0, 0), ('20011101', 0, 1)) + self.diags.config.experiment.startdates = ('20001101', '20011101') def test_generate_jobs(self): jobs = DaysOverPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '2000', '2001', '11']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, - 2000, 11)) + '20001101', 11)) self.assertEqual(jobs[1], DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, - 2001, 11)) + '20011101', 11)) with self.assertRaises(Exception): DaysOverPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '2000', '2001']) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 419be475..a1c8d25b 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -273,6 +273,7 @@ class TestReportConfig(TestCase): self.assertEquals(config.path, 'new_path') def test_priority(self): + self.mock_parser.add_value('REPORT', 'MAXIMUM_PRIORITY', 3) config = ReportConfig(self.mock_parser) self.assertEquals(config.maximum_priority, 3) @@ -294,12 +295,6 @@ class TestExperimentConfig(TestCase): self.assertEquals(config.atmos_timestep, 6) self.assertEquals(config.ocean_timestep, 6) - def test_cmor_version_required(self): - self.mock_parser.add_value('CMOR', 'VERSION', '20001101') - self.mock_parser.add_value('EXPERIMENT', 'DATA_CONVENTION', 'Primavera') - config = ExperimentConfig(self.mock_parser) - self.assertEquals(config.path, 'new_path') - def test_startdates(self): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '20001101 20011101') config = ExperimentConfig(self.mock_parser) diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index ae9921fa..e3cdcd83 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -271,7 +271,7 @@ class TestDiagnosticVariableListOption(TestCase): var_manager_mock = Mock() var_manager_mock.get_variable.side_effect = (self.get_var_mock('var1'), self.get_var_mock('var2')) diag = DiagnosticVariableListOption(var_manager_mock, 'variables') - self.assertEqual(['var1', 'var2'], diag.parse('var1-var2')) + self.assertEqual(['var1', 'var2'], diag.parse('var1:var2')) def test_parse_one(self): var_manager_mock = Mock() -- GitLab From 491febd211c478d1a83c4d04a0ba2802cdb9a11a Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 12:08:44 +0100 Subject: [PATCH 016/168] Fixed run test script --- .coveragerc | 2 ++ test/run_test.py | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.coveragerc b/.coveragerc index e41ecaeb..6c77e847 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,4 +2,6 @@ branch = True source = earthdiagnostics +[html] +title = Coverage report for EarthDiagnostics diff --git a/test/run_test.py b/test/run_test.py index 59eec7e9..765057a5 100644 --- a/test/run_test.py +++ b/test/run_test.py @@ -3,16 +3,15 @@ Script to run the tests for EarthDiagnostics and generate the code coverage report """ + +import os +work_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +os.chdir(work_path) +print(work_path) import coverage import unittest -import os -work_path = os.path.abspath('.') -source_path = os.path.join(work_path, '..', 'earthdiagnostics', '*') -print(source_path) -cov = coverage.Coverage(include=source_path) -cov.set_option("run:branch", True) -cov.set_option("html:title", 'Coverage report for EarthDiagnostics') +cov = coverage.Coverage() cov.start() suite = unittest.TestLoader().discover('.') unittest.TextTestRunner(verbosity=2).run(suite) -- GitLab From c6b130cd6ee9dfc38cf4865331e6035110d02117 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 15:04:41 +0100 Subject: [PATCH 017/168] Added test for cmormananger --- earthdiagnostics/cmorizer.py | 2 +- earthdiagnostics/cmormanager.py | 29 ++++++++++---------- test/unit/test_cmormanager.py | 47 +++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 test/unit/test_cmormanager.py diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 4bfde9de..97c19717 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -1,7 +1,6 @@ # coding=utf-8 import glob import os -import pygrib import shutil import uuid from datetime import datetime @@ -338,6 +337,7 @@ class Cmorizer(object): def _get_atmos_timestep(self, gribfile): Log.info('Getting timestep...') + import pygrib grib_handler = pygrib.open(gribfile) dates = set() try: diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index cfa67cae..265fc677 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -23,11 +23,14 @@ class CMORManager(DataManager): def __init__(self, config): super(CMORManager, self).__init__(config) self._dic_cmorized = dict() + self.find_model_data() + self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid) + + def find_model_data(self): data_folders = self.config.data_dir.split(':') experiment_folder = self.experiment.model.lower() if experiment_folder.startswith('ec-earth'): experiment_folder = 'ecearth' - self.config.data_dir = None for data_folder in data_folders: if os.path.isdir(os.path.join(data_folder, self.experiment.expid)): @@ -38,14 +41,12 @@ class CMORManager(DataManager): self.config.data_dir = test_folder break - test_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) + test_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) if os.path.isdir(os.path.join(test_folder, self.experiment.expid)): self.config.data_dir = test_folder break - if not self.config.data_dir: raise Exception('Can not find model data') - self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, @@ -74,8 +75,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param vartype: - :param domain: CMOR domain + :param vartype: + :param domain: CMOR domain :type domain: Domain :param var: variable name :type var: str @@ -104,8 +105,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param year: - :param diagnostic: + :param year: + :param diagnostic: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -137,8 +138,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param diagnostic: - :param region: + :param diagnostic: + :param region: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -179,8 +180,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param diagnostic: - :param year: + :param diagnostic: + :param year: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -215,7 +216,7 @@ class CMORManager(DataManager): grid=None, year=None, date_str=None): """ Returns the path to a concrete file - :param cmor_var: + :param cmor_var: :param startdate: file's startdate :type startdate: str :param member: file's member @@ -332,7 +333,7 @@ class CMORManager(DataManager): """ Creates the link of a given file from the CMOR repository. - :param cmor_var: + :param cmor_var: :param move_old: :param date_str: :param year: if frequency is yearly, this parameter is used to give the corresponding year diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py new file mode 100644 index 00000000..9c5fbb5f --- /dev/null +++ b/test/unit/test_cmormanager.py @@ -0,0 +1,47 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.cmormanager import CMORManager +from earthdiagnostics.modelingrealm import ModelingRealms +from mock import Mock +import tempfile +import os +import shutil + + +class TestCMORManager(TestCase): + + def setUp(self): + self.config = Mock() + self.config.data_convention = 'specs' + self.config.experiment.expid = 'expid' + self.config.experiment.model = 'model' + self.config.experiment.atmos_timestep = 6 + self.config.experiment.ocean_timestep = 6 + + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + self.config.data_dir = self.tmp_dir + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_find_data(self): + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'expid')) + + def test_find_data_fail(self): + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + with self.assertRaises(Exception): + CMORManager(self.config) + + def test_find_data_with_model(self): + os.makedirs(os.path.join(self.tmp_dir, 'model', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'model', 'expid')) + + def test_get_varfolder(self): + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.get_varfolder(ModelingRealms.ocean, 'var'), + 'var_f6h') + -- GitLab From 7cb043d7f86bda3537cc996515edaeec573002fb Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 24 Nov 2017 18:04:38 +0100 Subject: [PATCH 018/168] Pisces developments --- earthdiagnostics/cmorizer.py | 2 +- earthdiagnostics/ocean/__init__.py | 27 +++---- earthdiagnostics/ocean/regionmean.py | 10 ++- earthdiagnostics/ocean/regionsum.py | 30 ++++--- earthdiagnostics/utils.py | 9 +++ earthdiagnostics/variable_alias/primavera.csv | 78 +++++++++++++++++++ earthdiagnostics/work_manager.py | 3 +- 7 files changed, 132 insertions(+), 27 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 4bfde9de..be5ff6b8 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -363,7 +363,7 @@ class Cmorizer(object): Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) - Utils.rename_variables(filename, Cmorizer.ALT_COORD_NAMES, False, True) + Utils.rename_variables(filename, self.alt_coord_names, False, True) self._remove_valid_limits(filename) self._add_common_attributes(filename, frequency) self._update_time_variables(filename) diff --git a/earthdiagnostics/ocean/__init__.py b/earthdiagnostics/ocean/__init__.py index 854d90d9..c867bdc5 100644 --- a/earthdiagnostics/ocean/__init__.py +++ b/earthdiagnostics/ocean/__init__.py @@ -2,25 +2,26 @@ """ Module containing the diagnostics related to the ocean output """ -from earthdiagnostics.ocean.heatcontent import HeatContent -from earthdiagnostics.ocean.moc import Moc from earthdiagnostics.ocean.areamoc import AreaMoc -from earthdiagnostics.ocean.maxmoc import MaxMoc -from earthdiagnostics.ocean.psi import Psi -from earthdiagnostics.ocean.gyres import Gyres +from earthdiagnostics.ocean.averagesection import AverageSection from earthdiagnostics.ocean.convectionsites import ConvectionSites from earthdiagnostics.ocean.cutsection import CutSection -from earthdiagnostics.ocean.averagesection import AverageSection +from earthdiagnostics.ocean.gyres import Gyres +from earthdiagnostics.ocean.heatcontent import HeatContent +from earthdiagnostics.ocean.heatcontentlayer import HeatContentLayer from earthdiagnostics.ocean.interpolate import Interpolate from earthdiagnostics.ocean.interpolatecdo import InterpolateCDO -from earthdiagnostics.ocean.verticalmeanmeters import VerticalMeanMeters -from earthdiagnostics.ocean.verticalmean import VerticalMean -from earthdiagnostics.ocean.mixedlayersaltcontent import MixedLayerSaltContent -from earthdiagnostics.ocean.siasiesiv import Siasiesiv -from earthdiagnostics.ocean.heatcontentlayer import HeatContentLayer +from earthdiagnostics.ocean.mask_land import MaskLand +from earthdiagnostics.ocean.maxmoc import MaxMoc from earthdiagnostics.ocean.mixedlayerheatcontent import MixedLayerHeatContent +from earthdiagnostics.ocean.mixedlayersaltcontent import MixedLayerSaltContent +from earthdiagnostics.ocean.moc import Moc +from earthdiagnostics.ocean.mxl import Mxl +from earthdiagnostics.ocean.psi import Psi from earthdiagnostics.ocean.regionmean import RegionMean +from earthdiagnostics.ocean.regionsum import RegionSum from earthdiagnostics.ocean.rotation import Rotation -from earthdiagnostics.ocean.mxl import Mxl +from earthdiagnostics.ocean.siasiesiv import Siasiesiv from earthdiagnostics.ocean.verticalgradient import VerticalGradient -from earthdiagnostics.ocean.mask_land import MaskLand +from earthdiagnostics.ocean.verticalmean import VerticalMean +from earthdiagnostics.ocean.verticalmeanmeters import VerticalMeanMeters diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index edc5e212..8bfce0de 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -53,6 +53,9 @@ class RegionMean(Diagnostic): self.grid = grid self.declared = {} + self.lat_name = 'lat' + self.lon_name = 'lon' + def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable @@ -123,6 +126,10 @@ class RegionMean(Diagnostic): handler = Utils.openCdf(variable_file) self.save3d &= 'lev' in handler.dimensions + if "latitude" in handler.variables: + self.lat_name = 'latitude' + if "longitude" in handler.variables: + self.lon_name = 'longitude' handler.close() cdfmean_options = [self.variable, self.grid_point, 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] @@ -158,7 +165,8 @@ class RegionMean(Diagnostic): levels = '' temp2 = TempFile.get() - Utils.nco.ncks(input=mean_file, output=temp2, options=('-v {0},lat,lon{1}'.format(original_name, levels),)) + Utils.nco.ncks(input=mean_file, output=temp2, + options=('-v {0},{2.lat_name},{2.lon_name}{1}'.format(original_name, levels, self),)) handler = Utils.openCdf(temp2) var_handler = handler.variables[original_name] if hasattr(var_handler, 'valid_min'): diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index 52b49f91..0f4c59a9 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -38,7 +38,7 @@ class RegionSum(Diagnostic): "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid_point, box, save3d, basin, - variance, grid): + grid): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -49,10 +49,12 @@ class RegionSum(Diagnostic): self.box = box self.save3d = save3d self.basin = basin - self.variance = variance self.grid = grid self.declared = {} + self.lat_name = 'lat' + self.lon_name = 'lon' + def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable @@ -91,7 +93,7 @@ class RegionSum(Diagnostic): for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(RegionSum(diags.data_manager, startdate, member, chunk, options['domain'], options['variable'], options['grid_point'], box, - options['save3D'], options['basin'], options['variance'], options['grid'])) + options['save3D'], options['basin'], options['grid'])) return job_list def request_data(self): @@ -118,25 +120,30 @@ class RegionSum(Diagnostic): handler = Utils.openCdf(variable_file) self.save3d &= 'lev' in handler.dimensions + if "latitude" in handler.variables: + self.lat_name = 'latitude' + if "longitude" in handler.variables: + self.lon_name = 'longitude' + handler.close() - cdfmean_options = [self.variable, self.grid_point, 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] - if self.variance: - cdfmean_options += ['-var'] + cdfmean_options = ['-v', self.variable, '-p', self.grid_point, + '-zoom', 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] if self.basin != Basins().Global: cdfmean_options.append('-M') cdfmean_options.append('mask_regions.3d.nc') cdfmean_options.append(self.basin.name) - cdftools.run('cdfsum', input=variable_file, output=mean_file, options=cdfmean_options) + cdftools.run('cdfsum', input=variable_file, input_option='-f', output=mean_file, options=cdfmean_options) Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) - self.send_var('mean', False, mean_file) - self.send_var('mean', True, mean_file) + self.send_var(False, mean_file) + self.send_var(True, mean_file) os.remove(mean_file) - def send_var(self, var, threed, mean_file): + def send_var(self, threed, mean_file): + var = 'sum' if threed: if not self.save3d: return False @@ -149,7 +156,8 @@ class RegionSum(Diagnostic): levels = '' temp2 = TempFile.get() - Utils.nco.ncks(input=mean_file, output=temp2, options=('-v {0},lat,lon{1}'.format(original_name, levels),)) + Utils.nco.ncks(input=mean_file, output=temp2, + options=('-v {0},{2.lat_name},{2.lon_name}{1}'.format(original_name, levels, self),)) handler = Utils.openCdf(temp2) var_handler = handler.variables[original_name] if hasattr(var_handler, 'valid_min'): diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 7c70b56e..ccc503ea 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -239,6 +239,11 @@ class Utils(object): if old_name in handler.variables: if new_name not in handler.variables: handler.renameVariable(old_name, new_name) + for var in handler.variables: + if hasattr(var, 'coordinates') and " {0} ".format(old_name) in var.coordinates: + new_coordinates = var.coordinates.replace(" {0} ".format(old_name), + " {0} ".format(new_name)) + var.coordinates = Utils.convert_to_ASCII_if_possible(new_coordinates) elif must_exist: raise Exception("Variable {0} does not exist in file {1}".format(old_name, filepath)) handler.sync() @@ -538,6 +543,10 @@ class Utils(object): original_var = source.variables[variable] new_var = destiny.createVariable(new_name, original_var.datatype, translated_dimensions) Utils.copy_attributes(new_var, original_var) + if hasattr(new_var, 'coordinates'): + coords = [new_names[coord] if coord in new_names else coord for coord in new_var.coordinates.split(' ')] + new_var.coordinates = Utils.convert_to_ASCII_if_possible(' '.join(coords)) + new_var[:] = original_var[:] @staticmethod diff --git a/earthdiagnostics/variable_alias/primavera.csv b/earthdiagnostics/variable_alias/primavera.csv index db38cd5c..c64d2303 100644 --- a/earthdiagnostics/variable_alias/primavera.csv +++ b/earthdiagnostics/variable_alias/primavera.csv @@ -7,3 +7,81 @@ hflx_rain_cea,hfrainds,, hflx_cal_cea,hfibthermds2d,, rain,prra,, calving,ficeberg2d,, +alk,talk,, +oxygen,o2,, +calcite,calc,, +po4,po4,, +poc,poc,, +silicate,si,, +nanophy,nanophy,, +microzoo,zmicro,, +doc,dissoc,, +diaphy,phydiat,, +mesozoo,zmeso,, +dsi,dsi,, +dissfe,dfe,, +bfe,bfe,, +goc,goc,, +sfe,sfe,, +dfe,dfe,, +micrzoo,zmicro,, +nfe,nfe,, +nchl,nchl,, +dchl,chldiat,, +nitrate,no3,, +ammonium,nh4,, +pno3tot,pno3tot,, +psiltot,psiltot,, +palktot,palktot,, +pfertot,pfertot,, +tcflx,tcflx,, +tcflxcum,tcflxcum,, +c-export,c-export,, +tintpp,tintpp,, +tnfix,tnfix,, +tdenit,tdenit,, +intppnew,intppnew,, +inttppnew,inttppnew,, +inttpbfe,pbfe,, +intdic,intdic,, +o2min,o2min,, +zo2min,zo2min,, +intnfix,intpn2,, +intppphy,intppphy,, +intppphy2,intppdiat,, +ppphy ,ppphy ,, +ppphy2 ,pdi,, +intpp,intpp,, +inttpp,inttpp,, +intpbfe,intpbfe,, +intpbsi,intpbsi,, +intpbcal,intpbcal,, +cflx,cflx,, +remin,remin,, +denit,denit,, +nfix,nfix,, +sdenit,sdenit,, +dpco2,dpco2,, +epc100,epc100,, +expc,expc,, +par,par,, +lnnut,lnnut,, +ldnut,ldnut,, +lnfe,lnfe,, +ldfe,limfediat,, +lnlight,lnlight,, +ldlight,ldlight,, +graz1,graz1,, +graz2,graz2,, +mumax,mumax,, +mun,mun,, +mud,mud,, +ppnewn,ppnewn,, +ppnewd,ppnewd,, +alb_ice,sialb,, +qsr3d,rsdo,, +hflx_rnf_cea,hfrunoffds2d,, +hflx_rain_cea,hfrainds,, +hflx_cal_cea,hfibthermds2d,, +rain,prra,, +calving,ficeberg2d,, diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 33576f7c..95fe29b6 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -54,7 +54,7 @@ class WorkManager(object): if len(self.jobs) == 0: Log.result('No diagnostics to run') return True - + start_time = datetime.datetime.now() Log.info("Starting to compute at {0}", start_time) self.threads = Utils.available_cpu_count() @@ -258,6 +258,7 @@ class WorkManager(object): Diagnostic.register(HeatContentLayer) Diagnostic.register(HeatContent) Diagnostic.register(RegionMean) + Diagnostic.register(RegionSum) Diagnostic.register(Rotation) Diagnostic.register(VerticalGradient) -- GitLab From f551c2f46e7e8d76e8f291582cf0255c3529fcfc Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 29 Nov 2017 09:31:48 +0100 Subject: [PATCH 019/168] Added test for paths in specs format --- earthdiagnostics/cmormanager.py | 6 ++ test/unit/ocean/test_interpolatecdo.py | 4 +- test/unit/test_cmormanager.py | 131 +++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 265fc677..d66690e3 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -240,6 +240,12 @@ class CMORManager(DataManager): :param cmor_var: variable instance describing the selected variable :type cmor_var: Variable """ + + options = sum(x is not None for x in (chunk, year, date_str)) + if options == 0: + raise ValueError('You must provide chunk, year or date_str') + elif options > 1: + raise ValueError('You must provide only one parameter in chunk, year or date_str') if not frequency: frequency = self.config.frequency diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py index ba238f28..b7b87cf1 100644 --- a/test/unit/ocean/test_interpolatecdo.py +++ b/test/unit/ocean/test_interpolatecdo.py @@ -28,8 +28,10 @@ class TestInterpolate(TestCase): @patch('earthdiagnostics.ocean.interpolatecdo.InterpolateCDO.get_sample_grid_file') @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) @patch('os.remove') - def test_generate_jobs(self, mock_weights, mock_grid_file, mock_remove): + @patch('earthdiagnostics.utils.TempFile.get') + def test_generate_jobs(self, mock_weights, mock_grid_file, mock_remove, mock_get): mock_weights.return_value = None + mock_get.return_value = 'path_to_weights' jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var']) self.assertEqual(len(jobs), 2) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 9c5fbb5f..6e563344 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -13,10 +13,19 @@ class TestCMORManager(TestCase): def setUp(self): self.config = Mock() self.config.data_convention = 'specs' + self.config.data_type = 'exp' self.config.experiment.expid = 'expid' self.config.experiment.model = 'model' + self.config.experiment.experiment_name = 'expname' + self.config.experiment.institute = 'institute' + self.config.experiment.member_count_start = 0 self.config.experiment.atmos_timestep = 6 self.config.experiment.ocean_timestep = 6 + self.config.experiment.chunk_size = 12 + self.config.experiment.calendar = 'standard' + + self.config.cmor.initialization_number = 1 + self.config.cmor.version = '' self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) @@ -40,8 +49,130 @@ class TestCMORManager(TestCase): cmor_manager = CMORManager(self.config) self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'model', 'expid')) + def test_find_data_with_ecearth_fix(self): + self.config.experiment.model = 'EC-Earth' + os.makedirs(os.path.join(self.tmp_dir, 'ecearth', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'ecearth', 'expid')) + + def test_find_data_with_type_and_model(self): + os.makedirs(os.path.join(self.tmp_dir, 'exp', 'model', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'exp', 'model', 'expid')) + def test_get_varfolder(self): cmor_manager = CMORManager(self.config) self.assertEqual(cmor_manager.get_varfolder(ModelingRealms.ocean, 'var'), 'var_f6h') + def test_get_file_path_specs(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_version(self): + self.config.cmor.version = 'version' + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/version/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_grid(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency, 'grid') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/grid/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_year(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'year' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'year' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, year='1998') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/year/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_1998.nc')) + + frequency.frequency = 'other' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'other' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + + def test_get_file_path_raise_incomaptible_date_info(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'monthly' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, date_str='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + None, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998', date_str='1998') + + def test_get_file_path_specs_date_str(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + frequency.frequency = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, date_str='date_str') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_date_str.nc')) + -- GitLab From d1561e452a9247029a80e9a8dbfbab1b73107ed7 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 29 Nov 2017 09:37:04 +0100 Subject: [PATCH 020/168] Removed unused code --- earthdiagnostics/cmormanager.py | 31 ------------------- earthdiagnostics/constants.py | 4 +-- earthdiagnostics/diagnostic.py | 1 - earthdiagnostics/general/dailymean.py | 1 - earthdiagnostics/general/module.py | 1 - earthdiagnostics/general/monthlymean.py | 1 - earthdiagnostics/general/relink.py | 2 -- earthdiagnostics/general/scale.py | 1 - earthdiagnostics/general/select_levels.py | 1 - .../general/simplify_dimensions.py | 3 +- earthdiagnostics/ocean/mask_land.py | 3 +- test/unit/general/test_relinkall.py | 2 -- test/unit/ocean/test_vertical_gradient.py | 2 -- 13 files changed, 4 insertions(+), 49 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index d66690e3..b39ecd73 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -602,37 +602,6 @@ class CMORManager(DataManager): self.create_link(domain, cmorfile, frequency, var, "", False, vartype=VariableType.MEAN) - def _fix_ij_swap(self, cmorfile): - return - original_handler = Utils.openCdf(cmorfile) - if original_handler.dimensions['i'].size < original_handler.dimensions['j'].size: - temp = TempFile.get() - new_handler = Utils.openCdf(temp, 'w') - for attribute in original_handler.ncattrs(): - original = getattr(original_handler, attribute) - setattr(new_handler, attribute, - Utils.convert_to_ASCII_if_possible(original)) - for dimension in original_handler.dimensions.keys(): - if dimension == 'i': - new_name = 'j' - elif dimension == 'j': - new_name = 'i' - else: - new_name = dimension - new_handler.createDimension(new_name, original_handler.dimensions[dimension].size) - for variable in original_handler.variables.keys(): - original_var = original_handler.variables[variable] - translated_dimensions = Utils._translate(original_var.dimensions, - {'i': 'j', 'j': 'i'}) - new_var = new_handler.createVariable(variable, original_var.datatype, - translated_dimensions) - Utils.copy_attributes(new_var, original_var) - new_var[:] = original_var[:] - original_handler.close() - new_handler.close() - Utils.move_file(temp, cmorfile, save_hash=True) - Log.debug('File {0} translated', cmorfile) - def _get_startdate_path(self, startdate): """ Returns the path to the startdate's CMOR folder diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index 229a61b0..d7935838 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -2,8 +2,6 @@ """ Contains the enumeration-like classes used by the diagnostics """ -import netCDF4 - from singleton import SingletonType @@ -133,7 +131,7 @@ class Basins(object): def get_available_basins(self, handler): """ - + :param handler: :type handler: netCDF4.Dataset """ diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 0b1dbf57..c49dea84 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -245,7 +245,6 @@ class Diagnostic(Publisher): def _updated_request(self, request): if self.status != DiagnosticStatus.WAITING: return - from datafile import LocalStatus if request.local_status == LocalStatus.FAILED: self.message = 'Required file {0} is not available'.format(request.remote_file) self.status = DiagnosticStatus.FAILED diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py index 7fb4736e..1411b0ed 100644 --- a/earthdiagnostics/general/dailymean.py +++ b/earthdiagnostics/general/dailymean.py @@ -5,7 +5,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm class DailyMean(Diagnostic): diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index f72aa5f5..e8cf6d85 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -1,7 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm import numpy as np diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index dca5e730..6062d075 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -5,7 +5,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm class MonthlyMean(Diagnostic): diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 60c69f4c..b2545e1b 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -1,8 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption, \ DiagnosticVariableOption -from earthdiagnostics.modelingrealm import ModelingRealm -from earthdiagnostics.variable import VariableManager class Relink(Diagnostic): diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 116a978d..46b7af47 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,7 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils -from earthdiagnostics.modelingrealm import ModelingRealm import math diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index 1d2fb9ca..69931b87 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -1,7 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption, DiagnosticIntOption -from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.box import Box diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index c903af0a..c9c39e21 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -3,7 +3,6 @@ import numpy as np from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption -from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.utils import Utils, TempFile @@ -127,7 +126,7 @@ class SimplifyDimensions(Diagnostic): '{0}_vertices'.format(self.lon_name), '{0}_vertices'.format(self.lat_name)): continue Utils.copy_variable(handler, new_file, var, new_names={'i': self.lon_name, 'j': self.lat_name}) - + self._create_var(self.lon_name, lon_values, handler, new_file) self._create_var(self.lat_name, lat_values, handler, new_file) handler.close() diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index a7af9aaa..24370378 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -25,6 +25,7 @@ class MaskLand(Diagnostic): """ alias = 'maskland' + "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, mask, grid): Diagnostic.__init__(self, data_manager) @@ -83,7 +84,7 @@ class MaskLand(Diagnostic): mask_file.close() return mask - "Diagnostic alias for the configuration file" + def request_data(self): self.var_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, diff --git a/test/unit/general/test_relinkall.py b/test/unit/general/test_relinkall.py index cf8c9a16..cda393c8 100644 --- a/test/unit/general/test_relinkall.py +++ b/test/unit/general/test_relinkall.py @@ -6,8 +6,6 @@ from earthdiagnostics.box import Box from earthdiagnostics.general.relinkall import RelinkAll from mock import Mock, patch -from earthdiagnostics.modelingrealm import ModelingRealms - class TestRelinkAll(TestCase): diff --git a/test/unit/ocean/test_vertical_gradient.py b/test/unit/ocean/test_vertical_gradient.py index e2d9d22d..892160a2 100644 --- a/test/unit/ocean/test_vertical_gradient.py +++ b/test/unit/ocean/test_vertical_gradient.py @@ -1,8 +1,6 @@ # coding=utf-8 from unittest import TestCase from earthdiagnostics.ocean.verticalgradient import VerticalGradient -from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption from mock import Mock, patch -- GitLab From 015082a5e79766fedcf774fdf8176a336d115800 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 29 Nov 2017 12:55:07 +0100 Subject: [PATCH 021/168] Added tests for PRIMAVERA paths --- earthdiagnostics/cmormanager.py | 3 +- test/unit/test_cmormanager.py | 96 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index b39ecd73..e6ffeebb 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -306,6 +306,8 @@ class CMORManager(DataManager): if self.config.cmor.version: folder_path = os.path.join(folder_path, self.config.cmor.version) else: + if not self.config.cmor.version: + raise ValueError('CMOR version is mandatory for PRIMAVERA and CMIP6') if not grid: if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: grid = self.config.cmor.default_ocean_grid @@ -598,7 +600,6 @@ class CMORManager(DataManager): if not filename.endswith('.nc') or filename.startswith('.'): return cmorfile = os.path.join(filepath, filename) - self._fix_ij_swap(cmorfile) self.create_link(domain, cmorfile, frequency, var, "", False, vartype=VariableType.MEAN) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 6e563344..20b30d30 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -26,6 +26,9 @@ class TestCMORManager(TestCase): self.config.cmor.initialization_number = 1 self.config.cmor.version = '' + self.config.cmor.default_ocean_grid = 'ocean_grid' + self.config.cmor.default_atmos_grid = 'atmos_grid' + self.config.cmor.activity = 'activity' self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) @@ -176,3 +179,96 @@ class TestCMORManager(TestCase): 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_date_str.nc')) + def test_get_file_path_primavera(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' + 'r2i1p1f1/Omon/var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_198901-198912.nc')) + + def test_get_file_path_no_version_primavera(self): + self._configure_primavera() + self.config.cmor.version = '' + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 0, frequency) + + def _configure_primavera(self): + self.config.data_convention = 'primavera' + self.config.cmor.version = 'version' + + def test_get_file_path_primavera_grid(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency, 'grid') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' + 'Omon/var/grid/version/' + 'var_Omon_model_expname_r2i1p1f1_grid_198901-198912.nc')) + + def test_get_file_path_primavera_year(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'year' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'year' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, year='1998') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/' + 'var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_1998.nc')) + + frequency.frequency = 'other' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'other' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + + def test_get_file_path_primavera_date_str(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + frequency.frequency = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, date_str='date_str') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' + 'Omon/var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_date_str.nc')) -- GitLab From 7977fa13f633c50664eb0431650fb645f1e34a67 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 29 Nov 2017 13:14:14 +0100 Subject: [PATCH 022/168] Added test for file_exists --- earthdiagnostics/cmormanager.py | 15 +++------------ test/unit/test_cmormanager.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index e6ffeebb..9b6e0d1d 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -54,21 +54,12 @@ class CMORManager(DataManager): cmor_var = self.variable_list.get_variable(var) filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) - # noinspection PyBroadException if possible_versions is None: - # noinspection PyBroadException - try: - return os.path.isfile(filepath) - except Exception: - return False + return os.path.isfile(filepath) else: for version in possible_versions: - # noinspection PyBroadException - try: - if os.path.isfile(filepath.replace(self.config.cmor.version, version)): - return True - except Exception: - pass + if os.path.isfile(filepath.replace(self.config.cmor.version, version)): + return True return False def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 20b30d30..e56deea5 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -3,6 +3,7 @@ from unittest import TestCase from earthdiagnostics.cmormanager import CMORManager from earthdiagnostics.modelingrealm import ModelingRealms from mock import Mock +import mock import tempfile import os import shutil @@ -272,3 +273,22 @@ class TestCMORManager(TestCase): os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' 'Omon/var/ocean_grid/version/' 'var_Omon_model_expname_r2i1p1f1_ocean_grid_date_str.nc')) + + def test_file_exists(self): + with mock.patch('os.path.isfile') as isfile: + cmor_manager = CMORManager(self.config) + isfile.return_value = True + self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1)) + isfile.return_value = False + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1)) + + + def test_file_exists_multiple_versions(self): + with mock.patch('os.path.isfile') as isfile: + cmor_manager = CMORManager(self.config) + isfile.return_value = True + self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + possible_versions=('version1', 'version2'))) + isfile.return_value = False + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + possible_versions=('version1', 'version2'))) -- GitLab From c6b210c380ea2cbe0f87424b9f41ef69a79d1db3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 11:06:55 +0100 Subject: [PATCH 023/168] Added mask option to scale --- earthdiagnostics/general/scale.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 46b7af47..da35ebad 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,7 +1,10 @@ # coding=utf-8 +import math + +import numpy as np + from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils -import math class Scale(Diagnostic): @@ -32,7 +35,7 @@ class Scale(Diagnostic): "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, value, offset, domain, variable, grid, - min_limit, max_limit, frequency): + min_limit, max_limit, frequency, apply_mask): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -45,6 +48,7 @@ class Scale(Diagnostic): self.min_limit = min_limit self.max_limit = max_limit self.frequency = frequency + self.apply_mask = apply_mask self.original_values = None @@ -76,14 +80,16 @@ class Scale(Diagnostic): DiagnosticOption('grid', ''), DiagnosticFloatOption('min_limit', float('nan')), DiagnosticFloatOption('max_limit', float('nan')), - DiagnosticListFrequenciesOption('frequencies', [diags.config.frequency])) + DiagnosticListFrequenciesOption('frequencies', [diags.config.frequency]), + DiagnosticBoolOption('apply_mask', False)) options = cls.process_options(options, options_available) job_list = list() for frequency in options['frequencies']: for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Scale(diags.data_manager, startdate, member, chunk, options['value'], options['offset'], options['domain'], options['variable'], - options['grid'], options['min_limit'], options['max_limit'], frequency)) + options['grid'], options['min_limit'], options['max_limit'], frequency, + options['apply_mask'])) return job_list def request_data(self): @@ -101,10 +107,17 @@ class Scale(Diagnostic): variable_file = self.variable_file.local_file handler = Utils.openCdf(variable_file) - var_handler = handler.variables[self.variable] - self.original_values = var_handler[:] + var = handler.variables[self.variable] + self.original_values = var[:] + if self.apply_mask: + mask = Utils.get_mask(Basins().Global).astype(float) + mask[mask == 0] = np.nan + var[:] = mask * var[:] if self._check_limits(): - var_handler[:] = self.original_values * self.value + self.offset + values = self.original_values * self.value + self.offset + if self.apply_mask: + values[np.isnan(values)] = 0 + var[:] = values handler.close() self.corrected.set_local_file(self.variable_file.local_file, self) -- GitLab From 9c1fbc23a5f9e2c4d66d4b0526d5bcef6720cc83 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 11:37:57 +0100 Subject: [PATCH 024/168] Added meteofrance path convention --- earthdiagnostics/cmormanager.py | 13 ++++++++++++- launch_diags.sh | 6 +++--- test/unit/test_cmormanager.py | 30 +++++++++++++++++++++++++----- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 9b6e0d1d..5c7298bd 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -284,6 +284,8 @@ class CMORManager(DataManager): self.experiment.experiment_name, self._get_member_str(member), grid, time_bound) + elif self.config.data_convention in ('meteofrance',): + file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, startdate[0:6], self._get_member_str(member)) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name @@ -296,7 +298,8 @@ class CMORManager(DataManager): folder_path = os.path.join(folder_path, self._get_member_str(member)) if self.config.cmor.version: folder_path = os.path.join(folder_path, self.config.cmor.version) - else: + + elif self.config.data_convention in ('primavera', 'cmip6'): if not self.config.cmor.version: raise ValueError('CMOR version is mandatory for PRIMAVERA and CMIP6') if not grid: @@ -311,6 +314,12 @@ class CMORManager(DataManager): folder_path = os.path.join(self._get_startdate_path(startdate), self._get_member_str(member), table_name, var, grid, self.config.cmor.version) + elif self.config.data_convention == 'meteofrance': + folder_path = os.path.join(self.config.data_dir, + 'H{0}'.format(chr(64 + int(startdate[4:6]))), + startdate[0:4]) + else: + raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return folder_path def _get_chunk_time_bounds(self, startdate, chunk): @@ -617,6 +626,8 @@ class CMORManager(DataManager): template = 'r{0}i{1}p1' elif self.config.data_convention in ('primavera', 'cmip6'): template = 'r{0}i{1}p1f1' + elif self.config.data_convention == 'meteofrance': + return str(member) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) diff --git a/launch_diags.sh b/launch_diags.sh index 477a1ca4..bcfa6b07 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -21,6 +21,6 @@ set -xv source activate diags -export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} -cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ -./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} + export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} + cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ + ./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index e56deea5..258fc791 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -1,12 +1,14 @@ # coding=utf-8 +import os +import shutil +import tempfile from unittest import TestCase + +import mock +from mock import Mock + from earthdiagnostics.cmormanager import CMORManager from earthdiagnostics.modelingrealm import ModelingRealms -from mock import Mock -import mock -import tempfile -import os -import shutil class TestCMORManager(TestCase): @@ -215,6 +217,9 @@ class TestCMORManager(TestCase): self.config.data_convention = 'primavera' self.config.cmor.version = 'version' + def _configure_meteofrance(self): + self.config.data_convention = 'meteofrance' + def test_get_file_path_primavera_grid(self): self._configure_primavera() cmor_manager = CMORManager(self.config) @@ -292,3 +297,18 @@ class TestCMORManager(TestCase): isfile.return_value = False self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, possible_versions=('version1', 'version2'))) + + def test_get_file_path_meteofrance(self): + self._configure_meteofrance() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'day' + file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 1, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'HA/2011/soicecov_day_201101_16.nc')) -- GitLab From 071765a5c614b63c21594a0722541c08772fa3d1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 15:11:15 +0100 Subject: [PATCH 025/168] Fixed meteofrance format --- earthdiagnostics/cmorizer.py | 2 +- earthdiagnostics/cmormanager.py | 10 +++++++--- test/unit/test_cmormanager.py | 7 ++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 97c19717..baf5f3de 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -363,7 +363,7 @@ class Cmorizer(object): Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) - Utils.rename_variables(filename, Cmorizer.ALT_COORD_NAMES, False, True) + Utils.rename_variables(filename, self.alt_coord_names, False, True) self._remove_valid_limits(filename) self._add_common_attributes(filename, frequency) self._update_time_variables(filename) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 5c7298bd..27261987 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -285,7 +285,8 @@ class CMORManager(DataManager): self._get_member_str(member), grid, time_bound) elif self.config.data_convention in ('meteofrance',): - file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, startdate[0:6], self._get_member_str(member)) + time_bound = self._get_chunk_time_bounds(startdate, chunk) + file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self._get_member_str(member)) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name @@ -315,7 +316,7 @@ class CMORManager(DataManager): table_name, var, grid, self.config.cmor.version) elif self.config.data_convention == 'meteofrance': - folder_path = os.path.join(self.config.data_dir, + folder_path = os.path.join(self.config.data_dir, self.experiment.experiment_name, 'H{0}'.format(chr(64 + int(startdate[4:6]))), startdate[0:4]) else: @@ -331,7 +332,10 @@ class CMORManager(DataManager): separator = '_' else: separator = '-' - time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, + if self.config.data_convention == 'meteofrance': + time_bound = "{0:04}{1:02}".format(chunk_start.year, chunk_start.month) + else: + time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, chunk_end.month, separator) return time_bound diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 258fc791..8367545b 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -311,4 +311,9 @@ class TestCMORManager(TestCase): file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 1, frequency) self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'HA/2011/soicecov_day_201101_16.nc')) + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_16.nc')) + + file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 2, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_16.nc')) -- GitLab From 92c42599c74215f49e19b6dbea9140c7a15e8b70 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 15:12:43 +0100 Subject: [PATCH 026/168] Added option meteofrance to config --- earthdiagnostics/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index f3c8d638..f5d4c3c4 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -23,7 +23,7 @@ class Config(object): :param path: path to the conf file :type path: str """ - + def __init__(self, path): parser = ConfigParser() @@ -62,7 +62,8 @@ class Config(object): "Custom mask regions 3D file to use" self.data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', - ('specs', 'primavera', 'cmip6', 'preface'), 'specs', + ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), + 'specs', ignore_case=True) if self.data_convention in ('primavera', 'cmip6'): @@ -130,10 +131,10 @@ class Config(object): :rtype: list(str) """ return self._real_commands - + class CMORConfig(object): - + def __init__(self, parser, var_manager): self.force = parser.get_bool_option('CMOR', 'FORCE', False) self.force_untar = parser.get_bool_option('CMOR', 'FORCE_UNTAR', False) -- GitLab From be9a2b61858a069ae6d31477adf978e1b51ea68b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 15:18:00 +0100 Subject: [PATCH 027/168] Added tables for meteofrance --- earthdiagnostics/cmor_tables/meteofrance.csv | 1 + earthdiagnostics/variable_alias/meteofrance.csv | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 earthdiagnostics/cmor_tables/meteofrance.csv create mode 100644 earthdiagnostics/variable_alias/meteofrance.csv diff --git a/earthdiagnostics/cmor_tables/meteofrance.csv b/earthdiagnostics/cmor_tables/meteofrance.csv new file mode 100644 index 00000000..90f01d91 --- /dev/null +++ b/earthdiagnostics/cmor_tables/meteofrance.csv @@ -0,0 +1 @@ +Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables diff --git a/earthdiagnostics/variable_alias/meteofrance.csv b/earthdiagnostics/variable_alias/meteofrance.csv new file mode 100644 index 00000000..07aef840 --- /dev/null +++ b/earthdiagnostics/variable_alias/meteofrance.csv @@ -0,0 +1,4 @@ +Aliases,Shortname,Basin,Grid +iiceconc:siconc:soicecov:ileadfra,sic,, +ci,sic,,ifs +es,sbl,, \ No newline at end of file -- GitLab From 021d67abaa22dae912fa6c4e3cad912757c74af6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 17:55:58 +0100 Subject: [PATCH 028/168] Added sic ans sit names for meteofrance --- earthdiagnostics/cmor_tables/meteofrance.csv | 2 ++ earthdiagnostics/cmormanager.py | 3 +++ earthdiagnostics/variable_alias/meteofrance.csv | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/cmor_tables/meteofrance.csv b/earthdiagnostics/cmor_tables/meteofrance.csv index 90f01d91..086e2ec0 100644 --- a/earthdiagnostics/cmor_tables/meteofrance.csv +++ b/earthdiagnostics/cmor_tables/meteofrance.csv @@ -1 +1,3 @@ Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables +iiceconc:siconc:soicecov:ileadfra:ci,soicecov,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,,, +iicethic:sithic,sogsit__,sea_ice_thickness,Sea Ice Thickness,seaIce,,m,,,, diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 27261987..66c75082 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -392,6 +392,9 @@ class CMORManager(DataManager): """ # Check if cmorized and convert if not + if self.config.data_convention == 'meteofrance': + return + for startdate, member in self.experiment.get_member_list(): if not self._unpack_cmor_files(startdate, member): self._cmorize_member(startdate, member) diff --git a/earthdiagnostics/variable_alias/meteofrance.csv b/earthdiagnostics/variable_alias/meteofrance.csv index 07aef840..abb99d5e 100644 --- a/earthdiagnostics/variable_alias/meteofrance.csv +++ b/earthdiagnostics/variable_alias/meteofrance.csv @@ -1,4 +1,4 @@ Aliases,Shortname,Basin,Grid -iiceconc:siconc:soicecov:ileadfra,sic,, +iiceconc:siconc:soicecov:ileadfra,soicecov,, ci,sic,,ifs -es,sbl,, \ No newline at end of file +es,sbl,, -- GitLab From dff07441c14411b583e6ea633daf062790f97487 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 1 Dec 2017 17:16:33 +0100 Subject: [PATCH 029/168] Removed cfunits dependency --- earthdiagnostics/utils.py | 16 ++++++++-------- environment.yml | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 7c70b56e..026c1f7d 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -10,6 +10,7 @@ import tarfile import tempfile from contextlib import contextmanager +import cf_units import iris import iris.exceptions import netCDF4 @@ -18,7 +19,6 @@ import six import xxhash from bscearth.utils.log import Log from cdo import Cdo -from cfunits import Units from nco import Nco from earthdiagnostics.constants import Basins @@ -666,15 +666,15 @@ class Utils(object): if hasattr(var_handler, 'calendar'): old_calendar = var_handler.calendar - new_unit = Units(new_units, calendar=calendar) - old_unit = Units(var_handler.units, calendar=old_calendar) - var_handler[:] = Units.conform(var_handler[:], old_unit, new_unit, inplace=True) + new_unit = cf_units.Unit(new_units, calendar=calendar) + old_unit = cf_units.Unit(var_handler.units, calendar=old_calendar) + var_handler[:] = old_unit.convert(var_handler[:], new_unit, inplace=True) if 'valid_min' in var_handler.ncattrs(): - var_handler.valid_min = Units.conform(float(var_handler.valid_min), old_unit, new_unit, - inplace=True) + var_handler.valid_min = old_unit.convert(float(var_handler.valid_min), new_unit, + inplace=True) if 'valid_max' in var_handler.ncattrs(): - var_handler.valid_max = Units.conform(float(var_handler.valid_max), old_unit, new_unit, - inplace=True) + var_handler.valid_max = old_unit.convert(float(var_handler.valid_max), new_unit, + inplace=True) var_handler.units = new_units @staticmethod diff --git a/environment.yml b/environment.yml index 4fa44d13..14d62005 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,6 @@ dependencies: - openpyxl - mock - cmake -- cfunits - coverage - pip: -- GitLab From 18c8b79b9541b2f1e316744f27d7bb5b6fb5b4f3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Dec 2017 10:28:06 +0100 Subject: [PATCH 030/168] Fixed scale tests --- earthdiagnostics/general/scale.py | 10 ++++---- environment.yml | 2 +- test/unit/general/test_scale.py | 42 ++++++++++++++++++------------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index da35ebad..099d6774 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -53,14 +53,14 @@ class Scale(Diagnostic): self.original_values = None def __str__(self): - return 'Scale output Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Scale value: {5} Offset: {6} Variable: {3}:{4} ' \ - 'Frequency: {7}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, - self.value, self.offset, self.frequency) + return 'Scale output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Scale value: {0.value} Offset: {0.offset} Variable: {0.domain}:{0.variable} ' \ + 'Frequency: {0.frequency} Apply mask: {0.apply_mask}'.format(self) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.apply_mask == other.apply_mask and self.value == other.value and self.offset == other.offset @classmethod def generate_jobs(cls, diags, options): diff --git a/environment.yml b/environment.yml index 14d62005..f804b6aa 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: - openpyxl - mock - cmake -- coverage +5- coverage - pip: - bscearth.utils diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py index e7697cc2..b2fcd0d0 100644 --- a/test/unit/general/test_scale.py +++ b/test/unit/general/test_scale.py @@ -1,12 +1,12 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError -from earthdiagnostics.box import Box -from earthdiagnostics.general.scale import Scale -from earthdiagnostics.frequency import Frequencies from mock import Mock, patch +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.scale import Scale from earthdiagnostics.modelingrealm import ModelingRealms @@ -32,40 +32,48 @@ class TestScale(TestCase): jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', '', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', '', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.monthly)) + 0, 100, Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.monthly)) + 0, 100, Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.three_hourly)) + 0, 100, Frequencies.three_hourly, False)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly, False)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', + True]) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly, True)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.three_hourly)) + 0, 100, Frequencies.three_hourly, True)) with self.assertRaises(DiagnosticOptionError): Scale.generate_jobs(self.diags, ['diagnostic']) - with self.assertRaises(DiagnosticOptionError): - Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', - 'extra']) + with self.assertRaises(DiagnosticOptionError): + Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', 'True', + 'extra']) def test_str(self): mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, - Frequencies.three_hourly) + Frequencies.three_hourly, False) self.assertEquals(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' - 'Variable: atmos:var Frequency: 3hr') + 'Variable: atmos:var Frequency: 3hr Apply mask: False') -- GitLab From 51b317d0a6ea94f64acbc559b97461b87954bf68 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Dec 2017 10:37:41 +0100 Subject: [PATCH 031/168] Fixed member meteofrance --- earthdiagnostics/cmormanager.py | 2 +- test/unit/test_cmormanager.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 66c75082..a0237d94 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -634,7 +634,7 @@ class CMORManager(DataManager): elif self.config.data_convention in ('primavera', 'cmip6'): template = 'r{0}i{1}p1f1' elif self.config.data_convention == 'meteofrance': - return str(member) + return '{0:02d}'.format(member) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 8367545b..f536dd64 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -308,12 +308,12 @@ class TestCMORManager(TestCase): frequency = Mock() frequency.__str__ = Mock() frequency.__str__.return_value = 'day' - file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 1, + file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 1, frequency) self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_16.nc')) + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_01.nc')) - file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 2, + file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 2, frequency) self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_16.nc')) + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_01.nc')) -- GitLab From 5c5861196ad76909b8accc001edbcd78f1153c0b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Dec 2017 12:57:32 +0100 Subject: [PATCH 032/168] Add fix for meteofrance var_names --- earthdiagnostics/CDFTOOLS_meteofrance.namlist | 211 ++++++++++++++++++ earthdiagnostics/datafile.py | 5 + 2 files changed, 216 insertions(+) create mode 100644 earthdiagnostics/CDFTOOLS_meteofrance.namlist diff --git a/earthdiagnostics/CDFTOOLS_meteofrance.namlist b/earthdiagnostics/CDFTOOLS_meteofrance.namlist new file mode 100644 index 00000000..8f747650 --- /dev/null +++ b/earthdiagnostics/CDFTOOLS_meteofrance.namlist @@ -0,0 +1,211 @@ + ! Thu Jun 30 16:19:27 2016 + ! Namelist automatically generated by PrintCdfNames + ! Do not edit without changing its name ... + ! ------------------------------------------ + &NAMDIM + CN_X = "i" + , + CN_Y = "j" + , + CN_Z = "lev" + , + CN_T = "time" + + / + &NAMDIMVAR + CN_VLON2D = "lon" + , + CN_VLAT2D = "lat" + , + CN_VDEPTHT = "lev" + , + CN_VDEPTHU = "lev" + , + CN_VDEPTHV = "lev" + , + CN_VDEPTHW = "lev" + , + CN_VTIMEC = "time" + , + CN_MISSING_VALUE = "_FillValue" + + / + &NAMMETRICS + CN_VE1T = "e1t" + , + CN_VE1U = "e1u" + , + CN_VE1V = "e1v" + , + CN_VE1F = "e1f" + , + CN_VE2T = "e2t" + , + CN_VE2U = "e2u" + , + CN_VE2V = "e2v" + , + CN_VE2F = "e2f" + , + CN_VE3T = "e3t" + , + CN_VE3W = "e3w" + , + CN_VFF = "ff" + , + CN_GLAMT = "glamt" + , + CN_GLAMU = "glamu" + , + CN_GLAMV = "glamv" + , + CN_GLAMF = "glamf" + , + CN_GPHIT = "gphit" + , + CN_GPHIU = "gphiu" + , + CN_GPHIV = "gphiv" + , + CN_GPHIF = "gphif" + , + CN_GDEPT = "gdept" + , + CN_GDEPW = "gdepw" + , + CN_HDEPT = "hdept" + , + CN_HDEPW = "hdepw" + + / + &NAMVARS + CN_VOTEMPER = "thetao" + , + CN_VOSALINE = "so" + , + CN_VOZOCRTX = "uo" + , + CN_VOMECRTY = "vo" + , + CN_VOMEEIVV = "vomeeivv" + , + CN_VOVECRTZ = "vovecrtz" + , + CN_SOSSHEIG = "sossheig" + , + CN_SOMXL010 = "mlotst" + , + CN_SOMXLT02 = "somxlt02" + , + CN_SOHEFLDO = "sohefldo" + , + CN_SOLHFLUP = "solhflup" + , + CN_SOSBHFUP = "sosbhfup" + , + CN_SOLWFLDO = "solwfldo" + , + CN_SOSHFLDO = "soshfldo" + , + CN_SOWAFLUP = "sowaflup" + , + CN_SOWAFLCD = "sowaflcd" + , + CN_SOWAFLDP = "sowafldp" + , + CN_IOWAFLUP = "iowaflup" + , + CN_ZOMSFATL = "zomsfatl" + , + CN_ZOMSFGLO = "zomsfglo" + , + CN_ZOMSFPAC = "zomsfpac" + , + CN_ZOMSFINP = "zomsfinp" + , + CN_ZOMSFIND = "zomsfind" + , + CN_ZOISOATL = "zoisoatl" + , + CN_ZOISOGLO = "zoisoglo" + , + CN_ZOISOPAC = "zoisopac" + , + CN_ZOISOINP = "zoisoinp" + , + CN_ZOISOIND = "zoisoind" + , + CN_VOZOUT = "vozout" + , + CN_VOMEVT = "vomevt" + , + CN_VOZOUS = "vozous" + , + CN_VOMEVS = "vomevs" + , + CN_SOZOUT = "sozout" + , + CN_SOMEVT = "somevt" + , + CN_SOZOUS = "sozous" + , + CN_SOMEVS = "somevs" + , + CN_SOZOUTRP = "sozoutrp" + , + CN_SOMEVTRP = "somevtrp" + , + CN_SOICECOV = "soicecov" + , + CN_VOSIGMA0 = "vosigma0" + , + CN_VOSIGMAI = "vosigmai" + , + CN_VOSIGNTR = "vosigntr" + , + CN_VODEPISO = "vodepiso" + , + CN_ISOTHICK = "isothick" + , + CN_IICETHIC = "iicethic" + , + CN_ILEADFRA = "ileadfra" + , + CN_INVCFC = "INVCFC" + , + CN_CFC11 = "CFC11" + , + CN_PENDEP = "pendep" + + / + &NAMBATHY + CN_FBATHYMET = "bathy_meter.nc" + , + CN_FBATHYLEV = "bathy_level.nc" + , + CN_BATHYMET = "Bathymetry" + , + CN_BATHYLEV = "bathy_level" + , + CN_MBATHY = "mbathy" + + / + ! Namelist entry namsqdvar needs manual formating before + ! it can be used as input : put variables names in between ' + ! and separate variables by , + &NAMSQDVAR + NN_SQDVAR = 4, + CN_SQDVAR = "vozocrtx vomecrty vovecrtz sossheig" , + / + &NAMMESHMASK + CN_FZGR = "mesh_zgr.nc" + , + CN_FHGR = "mesh_hgr.nc" + , + CN_FMSK = "mask.nc" + , + CN_FCOO = "coordinates.nc" + , + CN_FBASINS = "new_maskglo.nc", + + / diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 3b38e077..a0a7eb7d 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -511,6 +511,11 @@ class NetCDFFile(DataFile): self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) Utils.copy_file(self.remote_file, self.local_file) + if self.data_convention == 'meteofrance': + self.alt_coord_names = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds', + 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', + 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', + 'y': 'j'} Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY -- GitLab From 3f699a713c1ca8dfd393040f68c1baac8074e01c Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Dec 2017 13:05:12 +0100 Subject: [PATCH 033/168] Fixed environment file --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f804b6aa..14d62005 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: - openpyxl - mock - cmake -5- coverage +- coverage - pip: - bscearth.utils -- GitLab From adb435ab835b8417aa947ea107b317c331c09122 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 09:47:59 +0100 Subject: [PATCH 034/168] Added rename call --- earthdiagnostics/datafile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index a0a7eb7d..300eb7cd 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -516,6 +516,7 @@ class NetCDFFile(DataFile): 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} + Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY -- GitLab From 2c1843b64864ec7db48437dadd72cc2e833337e8 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:06:34 +0100 Subject: [PATCH 035/168] Added ncdumps for debugging --- earthdiagnostics/datafile.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 300eb7cd..3f253b73 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -516,7 +516,11 @@ class NetCDFFile(DataFile): 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} + Log.info('Original file') + Utils.execute_shell_command('ncdump -h {0}'.format(self.local_file), Log.INFO) Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) + Log.info('Renamed file') + Utils.execute_shell_command('ncdump -h {0}'.format(self.local_file), Log.INFO) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY -- GitLab From abc7050ec8b31b274099defa6c9096bdc2104437 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:11:58 +0100 Subject: [PATCH 036/168] More debug info --- earthdiagnostics/datafile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 3f253b73..202dbd2d 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -511,7 +511,9 @@ class NetCDFFile(DataFile): self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) Utils.copy_file(self.remote_file, self.local_file) + Log.warning(self.data_convention) if self.data_convention == 'meteofrance': + Log.info('Converting variable names from meteofrance convention') self.alt_coord_names = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds', 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', -- GitLab From dd75ec5ef8cdf930e97d4a4499247447f86d7201 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:17:03 +0100 Subject: [PATCH 037/168] Now setting data_convention --- earthdiagnostics/datafile.py | 6 ++++-- earthdiagnostics/datamanager.py | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 202dbd2d..4606f58b 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -139,17 +139,19 @@ class DataFile(Publisher): self.dispatch(self) @classmethod - def from_storage(cls, filepath): + def from_storage(cls, filepath, data_convention): file_object = cls() file_object.remote_file = filepath file_object.local_status = LocalStatus.PENDING + file_object.data_convention = data_convention return file_object @classmethod - def to_storage(cls, remote_file): + def to_storage(cls, remote_file, data_convention): new_object = cls() new_object.remote_file = remote_file new_object.storage_status = StorageStatus.PENDING + new_object.data_convention = data_convention return new_object def download(self): diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index b4ab0124..81fd3740 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -29,7 +29,7 @@ class DataManager(object): def _get_file_from_storage(self, filepath): if filepath not in self.requested_files: - self.requested_files[filepath] = NCfile.from_storage(filepath) + self.requested_files[filepath] = NCfile.from_storage(filepath, self.config.data_convention) file_object = self.requested_files[filepath] file_object.local_satatus = LocalStatus.PENDING return self.requested_files[filepath] @@ -37,7 +37,7 @@ class DataManager(object): def _declare_generated_file(self, remote_file, domain, final_var, cmor_var, data_convention, region, diagnostic, grid, var_type, original_var): if remote_file not in self.requested_files: - self.requested_files[remote_file] = NCfile.to_storage(remote_file) + self.requested_files[remote_file] = NCfile.to_storage(remote_file, data_convention) file_object = self.requested_files[remote_file] file_object.diagnostic = diagnostic file_object.var_type = var_type @@ -48,7 +48,6 @@ class DataManager(object): file_object.final_name = final_var file_object.cmor_var = cmor_var file_object.region = region - file_object.data_convention = data_convention file_object.storage_status = StorageStatus.PENDING return file_object -- GitLab From 0712af4a17d5566097f2e30dfeb1cc1d0d787ba0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:28:48 +0100 Subject: [PATCH 038/168] Cleaned debug info --- earthdiagnostics/datafile.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 4606f58b..6c7237bc 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -513,18 +513,13 @@ class NetCDFFile(DataFile): self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) Utils.copy_file(self.remote_file, self.local_file) - Log.warning(self.data_convention) if self.data_convention == 'meteofrance': - Log.info('Converting variable names from meteofrance convention') - self.alt_coord_names = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds', + Log.debug('Converting variable names from meteofrance convention') + self.alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} - Log.info('Original file') - Utils.execute_shell_command('ncdump -h {0}'.format(self.local_file), Log.INFO) Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) - Log.info('Renamed file') - Utils.execute_shell_command('ncdump -h {0}'.format(self.local_file), Log.INFO) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY -- GitLab From 4d725816a0e609cce4ae3bc42e17e349e87af644 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:32:23 +0100 Subject: [PATCH 039/168] Fix check compression --- earthdiagnostics/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 026c1f7d..29d86dda 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -440,7 +440,9 @@ class Utils(object): handler = Utils.openCdf(filetoconvert) if not handler.file_format == 'NETCDF4': is_compressed = False + handler.close() else: + handler.close() ncdump_result = Utils.execute_shell_command('ncdump -hs {0}'.format(filetoconvert), Log.NO_LOG) ncdump_result = ncdump_result[0].replace('\t', '').split('\n') for var in handler.variables: @@ -450,8 +452,6 @@ class Utils(object): if not '{0}:_Shuffle = "true" ;'.format(var) in ncdump_result: is_compressed = False break - - handler.close() return is_compressed -- GitLab From 3eb9e269ed4411e7477ac207a905ed313e9822f2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:40:04 +0100 Subject: [PATCH 040/168] Avoid collisions between time_centered and time_counter --- earthdiagnostics/datafile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 6c7237bc..b28f319a 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -516,7 +516,6 @@ class NetCDFFile(DataFile): if self.data_convention == 'meteofrance': Log.debug('Converting variable names from meteofrance convention') self.alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', - 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) -- GitLab From 51824178c178c588b64671afd8f8313bc8821783 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 11:02:00 +0100 Subject: [PATCH 041/168] Add HDF5 flag --- earthdiagnostics/earthdiags.py | 1 + earthdiagnostics/utils.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 44c7fba5..c8a489b9 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -46,6 +46,7 @@ class EarthDiags(object): def __init__(self, config_file): Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) self.config = Config(config_file) + os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' TempFile.scratch_folder = self.config.scratch_dir cdftools.path = self.config.cdftools_path diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 29d86dda..b91fda94 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -390,7 +390,8 @@ class Utils(object): Log.log.log(log_level, line) output.append(line) if process.returncode != 0: - raise Utils.ExecutionError('Error executing {0}\n Return code: {1}'.format(' '.join(command), process.returncode)) + raise Utils.ExecutionError('Error executing {0}\n Return code: {1}'.format(' '.join(command), + str(process.returncode))) return output _cpu_count = None -- GitLab From 22693537b523e9f4f89d490c74690703646ccb84 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 11:50:07 +0100 Subject: [PATCH 042/168] Now user can specify STARTDATES in the format {START,END,INTERVAL} --- earthdiagnostics/config.py | 37 ++++++++++++++++++++++++++++++++----- test/unit/test_config.py | 21 ++++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index f5d4c3c4..e0dfa50b 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -3,7 +3,7 @@ import os import six from bscearth.utils.config_parser import ConfigParser -from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str +from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str, add_years, add_months, add_days from bscearth.utils.log import Log from earthdiagnostics import cdftools @@ -290,25 +290,52 @@ class ExperimentConfig(object): mem = mem[len(self.member_prefix):] members.append(int(mem)) self.members = members - + self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') startdates = parser.get_list_option('EXPERIMENT', 'STARTDATES') import exrex self.startdates = [] for startdate_pattern in startdates: - for startdate in exrex.generate(startdate_pattern): - self.startdates.append(startdate) + if startdate_pattern[0] == '{' and startdate_pattern[-1] == '}': + self._read_startdates(startdate_pattern[1:-1]) + else: + for startdate in exrex.generate(startdate_pattern): + self.startdates.append(startdate) self.chunk_size = parser.get_int_option('EXPERIMENT', 'CHUNK_SIZE') self.num_chunks = parser.get_int_option('EXPERIMENT', 'CHUNKS') self.chunk_list = parser.get_int_list_option('EXPERIMENT', 'CHUNK_LIST', []) - self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') + self.model = parser.get_option('EXPERIMENT', 'MODEL') self.model_version = parser.get_option('EXPERIMENT', 'MODEL_VERSION', '') self.atmos_grid = parser.get_option('EXPERIMENT', 'ATMOS_GRID', '') self.atmos_timestep = parser.get_int_option('EXPERIMENT', 'ATMOS_TIMESTEP', 6) self.ocean_timestep = parser.get_int_option('EXPERIMENT', 'OCEAN_TIMESTEP', 6) + def _read_startdates(self, pattern): + pattern = pattern.split(',') + start = parse_date(pattern[0].strip()) + end = parse_date(pattern[1].strip()) + interval = pattern[2].strip() + if len(interval) == 1: + factor = 1 + else: + factor = int(interval[0:-1]) + interval = interval[-1].upper() + while start <= end: + self.startdates.append(date2str(start)) + if interval == 'Y': + start = add_years(start, factor) + elif interval == 'M': + start = add_months(start, factor, cal=self.calendar) + elif interval == 'W': + start = add_days(start, factor * 7, cal=self.calendar) + elif interval == 'D': + start = add_days(start, factor, cal=self.calendar) + else: + raise ConfigException('Interval {0} not supported in STARTDATES definition: {1}', interval, pattern) + + def get_chunk_list(self): """ Return a list with all the chunks diff --git a/test/unit/test_config.py b/test/unit/test_config.py index a1c8d25b..b113d8bd 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -306,9 +306,28 @@ class TestExperimentConfig(TestCase): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200[0-2](02|05|08|11)01') config = ExperimentConfig(self.mock_parser) - print(config.startdates) self.assertEquals(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', u'20020801', u'20021101']) + def test_auto_startdates(self): + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,1Y}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,6M}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20010501', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,1W}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7D}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7F}') + with self.assertRaises(ConfigException): + ExperimentConfig(self.mock_parser) -- GitLab From 9dde9f2a337c4910c5e2cc3091aced7e9abdd918 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 15:43:48 +0100 Subject: [PATCH 043/168] Added option to avoid computing volume in siasiesiv --- earthdiagnostics/ocean/siasiesiv.py | 47 ++++++++++++++++++----------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index bd2c41c8..eb55ee26 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,18 +1,19 @@ # coding=utf-8 import os -# noinspection PyUnresolvedReferences - import netCDF4 import numpy as np from bscearth.utils.log import Log from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticBoolOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile +# noinspection PyUnresolvedReferences + + class Siasiesiv(Diagnostic): """ Compute the sea ice extent , area and volume in both hemispheres or a specified region. @@ -34,7 +35,7 @@ class Siasiesiv(Diagnostic): e2t = None gphit = None - def __init__(self, data_manager, startdate, member, chunk, basin, mask, var_manager): + def __init__(self, data_manager, startdate, member, chunk, basin, mask, var_manager, omit_vol): """ :param data_manager: data management object :type data_manager: DataManager @@ -55,6 +56,7 @@ class Siasiesiv(Diagnostic): self.mask = mask self.generated = {} self.var_manager = var_manager + self.omit_volume = omit_vol self.sic_varname = self.var_manager.get_variable('sic').short_name self.sit_varname = self.var_manager.get_variable('sit').short_name @@ -73,7 +75,8 @@ class Siasiesiv(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticBasinOption('basin', Basins().Global), ) + options_available = (DiagnosticBasinOption('basin', Basins().Global), + DiagnosticBoolOption('omit_volume', False)) options = cls.process_options(options, options_available) if options['basin'] is None: @@ -85,7 +88,7 @@ class Siasiesiv(Diagnostic): job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, options['basin'], mask, - diags.config.var_manager)) + diags.config.var_manager, options['omit_volume'])) mesh_handler = Utils.openCdf('mesh_hgr.nc') Siasiesiv.e1t = np.asfortranarray(mesh_handler.variables['e1t'][0, :]) Siasiesiv.e2t = np.asfortranarray(mesh_handler.variables['e2t'][0, :]) @@ -95,17 +98,20 @@ class Siasiesiv(Diagnostic): return job_list def request_data(self): - self.sit = self.request_chunk(ModelingRealms.seaIce, self.sit_varname, - self.startdate, self.member, self.chunk) + if not self.omit_volume: + self.sit = self.request_chunk(ModelingRealms.seaIce, self.sit_varname, + self.startdate, self.member, self.chunk) self.sic = self.request_chunk(ModelingRealms.seaIce, self.sic_varname, self.startdate, self.member, self.chunk) def declare_data_generated(self): - self._declare_var('sivols') + if not self.omit_volume: + self._declare_var('sivols') + self._declare_var('sivoln') + self._declare_var('siareas') self._declare_var('siextents') - self._declare_var('sivoln') self._declare_var('siarean') self._declare_var('siextentn') @@ -118,32 +124,39 @@ class Siasiesiv(Diagnostic): Runs the diagnostic """ import earthdiagnostics.cdftoolspython as cdftoolspython - sit_handler = Utils.openCdf(self.sit.local_file) - sit = np.asfortranarray(sit_handler.variables[self.sit_varname][:]) - timesteps = sit_handler.dimensions['time'].size - sit_handler.close() + sic_handler = Utils.openCdf(self.sic.local_file) Utils.convert_units(sic_handler.variables[self.sic_varname], '1.0') sic = np.asfortranarray(sic_handler.variables[self.sic_varname][:]) + timesteps = sic_handler.dimensions['time'].size sic_handler.close() + if self.omit_volume: + sit = sic + else: + sit_handler = Utils.openCdf(self.sit.local_file) + sit = np.asfortranarray(sit_handler.variables[self.sit_varname][:]) + sit_handler.close() + result = np.empty((8, timesteps)) for t in range(0, timesteps): result[:, t] = cdftoolspython.icediag.icediags(Siasiesiv.e1t, Siasiesiv.e2t, self.mask, Siasiesiv.gphit, sit[t, :], sic[t, :]) - self._extract_variable_and_rename(result[4, :], 'sivols', '10^9 m3') self._extract_variable_and_rename(result[5, :], 'siareas', '10^9 m2') self._extract_variable_and_rename(result[7, :], 'siextents', '10^9 m2') - self._extract_variable_and_rename(result[0, :], 'sivoln', '10^9 m3') self._extract_variable_and_rename(result[1, :], 'siarean', '10^9 m2') self._extract_variable_and_rename(result[3, :], 'siextentn', '10^9 m2') + if not self.omit_volume: + self._extract_variable_and_rename(result[4, :], 'sivols', '10^9 m3') + self._extract_variable_and_rename(result[0, :], 'sivoln', '10^9 m3') + def _extract_variable_and_rename(self, values, cmor_name, units): temp = TempFile.get() - reference_handler = Utils.openCdf(self.sit.local_file) + reference_handler = Utils.openCdf(self.sic.local_file) os.remove(temp) handler = netCDF4.Dataset(temp, 'w') -- GitLab From 5919edda1420925363bed041ac2d7defa34c792a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 15:44:21 +0100 Subject: [PATCH 044/168] Meteofrance convention is no longer creating links --- earthdiagnostics/datamanager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 81fd3740..5f6763e1 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -73,6 +73,8 @@ class DataManager(object): return folder_name def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + if self.config.data_convetion == 'meteofrance': + return freq_str = frequency.folder_name(vartype) if not grid: -- GitLab From b048d52f3f99458fcf7c41fbec13594ec849df7f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 16:56:28 +0100 Subject: [PATCH 045/168] Fixed parser errors and siasiesiv tests --- earthdiagnostics/config.py | 3 +++ earthdiagnostics/ocean/siasiesiv.py | 4 ++-- test/unit/ocean/test_siasiesiv.py | 8 +++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index e93e8159..4cd712f8 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -109,6 +109,9 @@ class Config(object): commands = self._diags.split() self._real_commands = list() for command in commands: + command = command.split() + if command.startswith('#'): + break if command.lower() in self._aliases: added_commands = self._aliases[command.lower()] Log.info('Changing alias {0} for {1}', command, ' '.join(added_commands)) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index eb55ee26..31c7ea48 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -61,8 +61,8 @@ class Siasiesiv(Diagnostic): self.sit_varname = self.var_manager.get_variable('sit').short_name def __str__(self): - return 'Siasiesiv Startdate: {0} Member: {1} Chunk: {2} Basin: {3}'.format(self.startdate, self.member, - self.chunk, self.basin) + return 'Siasiesiv Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Basin: {0.basin} Omit volume: {0.omit_volume}'.format(self) @classmethod def generate_jobs(cls, diags, options): diff --git a/test/unit/ocean/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py index 2457b72d..c580e151 100644 --- a/test/unit/ocean/test_siasiesiv.py +++ b/test/unit/ocean/test_siasiesiv.py @@ -1,9 +1,10 @@ # coding=utf-8 from unittest import TestCase +from mock import Mock + from earthdiagnostics.constants import Basins from earthdiagnostics.ocean.siasiesiv import Siasiesiv -from mock import Mock class TestSiasiesiv(TestCase): @@ -16,7 +17,8 @@ class TestSiasiesiv(TestCase): self.mask = Mock() self.var_manager = Mock() - self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask, self.var_manager) + self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask, self.var_manager, False) def test_str(self): - self.assertEquals(str(self.psi), 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global') + self.assertEquals(str(self.psi), + 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global Omit volume: False') -- GitLab From 6e67e4ea8f14efc65734451979c5bb024d7ce1c2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 17:04:41 +0100 Subject: [PATCH 046/168] Fixed confussion between strip and split --- earthdiagnostics/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 4cd712f8..3ec7d1a5 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -109,7 +109,7 @@ class Config(object): commands = self._diags.split() self._real_commands = list() for command in commands: - command = command.split() + command = command.strip() if command.startswith('#'): break if command.lower() in self._aliases: -- GitLab From 512aec9dec190d33a6df38c1650234a01a6fef15 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 17:55:42 +0100 Subject: [PATCH 047/168] Interpcdo now respects lat and lon names --- earthdiagnostics/ocean/interpolatecdo.py | 40 +++++++++++++----------- earthdiagnostics/ocean/siasiesiv.py | 1 - 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index b8b81620..80721a5b 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -133,18 +133,9 @@ class InterpolateCDO(Diagnostic): def get_sample_grid_file(cls): temp = TempFile.get() - lat_name = 'lat' handler = Utils.openCdf('mask.nc') - for lat_alias in ['lat', 'latitude']: - if lat_alias in handler.variables: - lat_name = lat_alias - break - lon_name = None - for lon_alias in ['lon', 'longitude']: - if lon_alias in handler.variables: - lon_name = lon_alias - break + lat_name, lon_name = cls._get_lat_lon_alias(handler) lon_bnds_name = '{0}_bnds'.format(lon_name) lat_bnds_name = '{0}_bnds'.format(lat_name) @@ -193,6 +184,20 @@ class InterpolateCDO(Diagnostic): Utils.nco.ncks(input=temp, output=temp, options=('-O -x -v gphif,glamf',)) return temp + @classmethod + def _get_lat_lon_alias(cls, handler): + lat_name = None + for lat_alias in ['lat', 'latitude']: + if lat_alias in handler.variables: + lat_name = lat_alias + break + lon_name = None + for lon_alias in ['lon', 'longitude']: + if lon_alias in handler.variables: + lon_name = lon_alias + break + return lat_name, lon_name + @classmethod def _translate_ifs_grids_to_cdo_names(cls, target_grid): if target_grid.upper().startswith('T159L'): @@ -223,20 +228,15 @@ class InterpolateCDO(Diagnostic): 'nav_lat': 'lat', 'nav_lon': 'lon'}, must_exist=False, rename_dimension=True) handler = Utils.openCdf(variable_file) + lat_name, lon_name = self._get_lat_lon_alias(handler) var = handler.variables[self.variable] units = var.units coordinates = list() for dim in var.dimensions: if dim == 'i': - if 'lat' in handler.variables: - coordinates.append('lat') - else: - coordinates.append('latitude') + coordinates.append(lon_name) elif dim == 'j': - if 'lon' in handler.variables: - coordinates.append('lon') - else: - coordinates.append('longitude') + coordinates.append(lat_name) else: coordinates.append(dim) var.coordinates = ' '.join(coordinates) @@ -251,9 +251,13 @@ class InterpolateCDO(Diagnostic): Utils.cdo.remap(','.join((self.grid.split('_')[0], self.weights)), input=variable_file, output=temp) handler = Utils.openCdf(temp) + handler.variables[self.variable].units = units handler.close() + if lat_name != 'lat': + Utils.rename_variables(temp, {'lat': lat_name, 'lon': lon_name}, True, True) + self.regridded.set_local_file(temp) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 31c7ea48..4a33aa98 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -125,7 +125,6 @@ class Siasiesiv(Diagnostic): """ import earthdiagnostics.cdftoolspython as cdftoolspython - sic_handler = Utils.openCdf(self.sic.local_file) Utils.convert_units(sic_handler.variables[self.sic_varname], '1.0') sic = np.asfortranarray(sic_handler.variables[self.sic_varname][:]) -- GitLab From af4c135dd5b0eafa895e93556fbe42c87acc1caf Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 17:58:50 +0100 Subject: [PATCH 048/168] Fixed typo --- earthdiagnostics/datamanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 5f6763e1..b4022bbd 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -73,7 +73,7 @@ class DataManager(object): return folder_name def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): - if self.config.data_convetion == 'meteofrance': + if self.config.data_convention == 'meteofrance': return freq_str = frequency.folder_name(vartype) -- GitLab From c988a78ac48095152b2b7c94e801ae4dfc77e7bc Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 10:05:54 +0100 Subject: [PATCH 049/168] Diagnostic __ne__ now default to not equals --- diags.conf | 4 +--- earthdiagnostics/diagnostic.py | 4 +++- test/unit/test_config.py | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/diags.conf b/diags.conf index d6504e3d..e211e9e6 100644 --- a/diags.conf +++ b/diags.conf @@ -88,9 +88,7 @@ OCEAN_TIMESTEP = 6 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = testing_erainterim -#STARTDATES = 199[3-9]0[1-9]01 199[3-9]1[0-2]01 200[0-9]0[1-9]01 200[0-9]1[0-2]01 201[0-5]0[1-9]01 201[0-5]1[0-2]01 -#STARTDATES = 19840101 19850101 -STARTDATES = 19960104 +STARTDATES = {19960104,20151231,W} {19960107,20151231,W} MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index c49dea84..f038547a 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -43,6 +43,9 @@ class Diagnostic(Publisher): self.consumed_time = datetime.timedelta() self.subjobs = [] + def __ne__(self, other): + return not self == other + def can_skip_run(self): for file_generated in self._generated_files: if file_generated.storage_status != StorageStatus.READY: @@ -51,7 +54,6 @@ class Diagnostic(Publisher): Log.warning('Can not skip diagnostics run when data is going to be modified: {0}'.format(self)) return False - def __repr__(self): return str(self) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index b113d8bd..441e983f 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -323,6 +323,10 @@ class TestExperimentConfig(TestCase): config = ExperimentConfig(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,W}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7D}') config = ExperimentConfig(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) -- GitLab From 4dcfa9c1c20e9650f3d7ec62fac0076acc42bac6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 10:52:13 +0100 Subject: [PATCH 050/168] Updated prospector and ci config --- .gitlab-ci.yml | 15 ++++++++++++--- .prospector.yml | 5 +++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc29806a..d8a25d0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,12 +1,21 @@ -test: +test_python2: script: - echo $USER - export PATH="$HOME/miniconda2/bin:$PATH" - echo $PATH - - conda env update -f environment.yml + - conda env update -f environment.yml -n earthdiagnostics3 python=2 + - source activate earthdiagnostics + - echo $PWD + - coverage run -m unittest discover + +test_python3: + script: + - echo $USER + - export PATH="$HOME/miniconda2/bin:$PATH" + - echo $PATH + - conda env update -f environment.yml -n earthdiagnostics3 python=3 - source activate earthdiagnostics - echo $PWD - coverage run -m unittest discover - coverage xml - python-codacy-coverage -r coverage.xml - diff --git a/.prospector.yml b/.prospector.yml index b5652ca3..3e01f3ca 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -13,7 +13,9 @@ pyroma: run: true pep8: - full: true + disable: [ + E501, # Line-length, already controlled by pylint + ] pep257: # see http://pep257.readthedocs.io/en/latest/error_codes.html @@ -25,5 +27,4 @@ pep257: D212, # Multi-line docstring summary should start at the first line D213, # Multi-line docstring summary should start at the second line D404, # First word of the docstring should not be This - E501, # Line-length, already controlled by pylint ] -- GitLab From c172b9a475aee19dcc40abd3d996816a156660b2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 11:09:18 +0100 Subject: [PATCH 051/168] Sorting issues with gitlab CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8a25d0b..a913fdad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ test_python2: - echo $USER - export PATH="$HOME/miniconda2/bin:$PATH" - echo $PATH - - conda env update -f environment.yml -n earthdiagnostics3 python=2 + - conda env update -f environment.yml -n earthdiagnostics2 python=2 - source activate earthdiagnostics - echo $PWD - coverage run -m unittest discover -- GitLab From a5152758a4b9649bc75a1f6436dd1c0d24c6daa7 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 11:11:48 +0100 Subject: [PATCH 052/168] Updated codacy config --- .codacy.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.codacy.yml b/.codacy.yml index c0f0e505..1da409b7 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -19,18 +19,7 @@ engines: python_version: 2 exclude_paths: [ - 'doc/sphinx/**', - 'esmvaltool/doc/sphinx/**', - # cmor tables - 'esmvaltool/interface_scripts/cmip*-cmor-tables/**', - # old stuff - 'backend/**', - 'diag_scripts/**', - 'interface_data/**', - 'interface_scripts/**', - 'main.py', - 'nml/**', - 'plot_scripts/**', - 'reformat_scripts/**', - 'variable_defs/**', + 'doc/**', + 'test/**', + 'earthdiagnostics/cmor_tables/**', ] -- GitLab From b3b72b45fd5fe57f43903fd01670a51e2920f62f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 11:21:01 +0100 Subject: [PATCH 053/168] Changed version to python 3 --- .codacy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codacy.yml b/.codacy.yml index 1da409b7..445a627b 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -16,7 +16,7 @@ engines: enabled: true pylint: enabled: true - python_version: 2 + python_version: 3 exclude_paths: [ 'doc/**', -- GitLab From f87732bc8c26395d40e2e342d1a8707e58ec2900 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 11:31:08 +0100 Subject: [PATCH 054/168] Code cleaned --- earthdiagnostics/earthdiags.py | 2 +- earthdiagnostics/general/attribute.py | 1 - earthdiagnostics/statistics/daysoverpercentile.py | 6 ++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index c8a489b9..49508aa8 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -9,7 +9,7 @@ from distutils.spawn import find_executable import bscearth.utils.path import netCDF4 import pkg_resources -from bscearth.utils.date import * +from bscearth.utils.log import Log from earthdiagnostics import cdftools from earthdiagnostics.cmormanager import CMORManager diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 1c618578..4259d0a3 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -2,7 +2,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticComplexStrOption, \ DiagnosticDomainOption, DiagnosticVariableOption from earthdiagnostics.utils import Utils -from earthdiagnostics.modelingrealm import ModelingRealm class Attribute(Diagnostic): diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index aad012a4..663618af 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -11,10 +11,12 @@ from bscearth.utils.date import parse_date, add_months from bscearth.utils.log import Log from iris.time import PartialDateTime -from earthdiagnostics.diagnostic import * +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticDomainOption, DiagnosticIntOption, \ + DiagnosticListIntOption, DiagnosticOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.variable_type import VariableType class DaysOverPercentile(Diagnostic): @@ -71,7 +73,7 @@ class DaysOverPercentile(Diagnostic): for forecast_month in options['forecast_month']: job_list.append(DaysOverPercentile(diags.data_manager, options['domain'], options['variable'], options['start_year'], options['end_year'], - startdate, forecast_month)) + startdate, forecast_month)) return job_list def request_data(self): -- GitLab From 5c42fc912ca7a557e811b066ef3b7a6f55ee197e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 11:39:02 +0100 Subject: [PATCH 055/168] Added readthedocs config --- readthedocs.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 readthedocs.yml diff --git a/readthedocs.yml b/readthedocs.yml new file mode 100644 index 00000000..40c09542 --- /dev/null +++ b/readthedocs.yml @@ -0,0 +1,2 @@ +conda: + file: environment.yml \ No newline at end of file -- GitLab From 8314b91d97fb1f4601d3a0f98975cfc45e550a16 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 11:47:18 +0100 Subject: [PATCH 056/168] Added readthedocs badge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b8f950dd..fed74417 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Codacy Badge](https://api.codacy.com/project/badge/Grade/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Grade) - +[![Readthedocs](http://earthdiagnostics.readthedocs.io/en/latest/)](https://readthedocs.org/projects/pip/badge/) +[![Readthedocs](http://earthdiagnostics.readthedocs.io/en/stable/)](https://readthedocs.org/projects/pip/badge/?version=stable) This tool is a set of diagnostics used at BSC-ES department for NEMO and EC-EARTH models postprocessing. They are based on CDO, NCO and CDFTOOLS 3.0. For CDFTOOLS, a custom build is required. -- GitLab From 41a2586e0d2f8ff047824e7591b95ff241ca7adf Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 11:44:39 +0100 Subject: [PATCH 057/168] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fed74417..c9d09041 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Codacy Badge](https://api.codacy.com/project/badge/Grade/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Grade) -[![Readthedocs](http://earthdiagnostics.readthedocs.io/en/latest/)](https://readthedocs.org/projects/pip/badge/) -[![Readthedocs](http://earthdiagnostics.readthedocs.io/en/stable/)](https://readthedocs.org/projects/pip/badge/?version=stable) +[![Readthedocs](https://readthedocs.org/projects/pip/badge/)](http://earthdiagnostics.readthedocs.io/en/latest/) +[![Readthedocs](https://readthedocs.org/projects/pip/badge/?version=stable)](http://earthdiagnostics.readthedocs.io/en/stable/) This tool is a set of diagnostics used at BSC-ES department for NEMO and EC-EARTH models postprocessing. They are based on CDO, NCO and CDFTOOLS 3.0. For CDFTOOLS, a custom build is required. -- GitLab From ed9759d4f2fc0bde6102c5deadba3d097be73e4a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 11:45:47 +0100 Subject: [PATCH 058/168] Update README.md --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c9d09041..0f5541cc 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,18 @@ [![Codacy Badge](https://api.codacy.com/project/badge/Grade/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Grade) -[![Readthedocs](https://readthedocs.org/projects/pip/badge/)](http://earthdiagnostics.readthedocs.io/en/latest/) -[![Readthedocs](https://readthedocs.org/projects/pip/badge/?version=stable)](http://earthdiagnostics.readthedocs.io/en/stable/) + This tool is a set of diagnostics used at BSC-ES department for NEMO and EC-EARTH models postprocessing. They are based on CDO, NCO and CDFTOOLS 3.0. For CDFTOOLS, a custom build is required. FULL DOCUMENTATION AND HOW TOs ============================== -Check the Earth Diagnostics documentation in PDF format in EarthDiagnostics.pdf available also in this folder. +Latest version: + +[![Readthedocs](https://readthedocs.org/projects/pip/badge/)](http://earthdiagnostics.readthedocs.io/en/latest/) + +Latest release: + +[![Readthedocs](https://readthedocs.org/projects/pip/badge/?version=stable)](http://earthdiagnostics.readthedocs.io/en/stable/) CONTACT ======= -- GitLab From 507e5667b1bd4c9a902488ff1f767c670360b2b7 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 12:35:43 +0100 Subject: [PATCH 059/168] Updated readme and fixed error in config --- README.md | 2 +- diags.conf | 5 +++-- earthdiagnostics/config.py | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0f5541cc..f4956d66 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Codacy Badge](https://api.codacy.com/project/badge/Grade/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Grade) - +[![Codacy Badge](https://api.codacy.com/project/badge/Coverage/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Coverage) This tool is a set of diagnostics used at BSC-ES department for NEMO and EC-EARTH models postprocessing. They are based on CDO, NCO and CDFTOOLS 3.0. For CDFTOOLS, a custom build is required. diff --git a/diags.conf b/diags.conf index e211e9e6..f5a579a1 100644 --- a/diags.conf +++ b/diags.conf @@ -27,7 +27,7 @@ CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir RESTORE_MESHES = False # Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available)z -MAX_CORES = 2 +MAX_CORES = 1 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False @@ -88,11 +88,12 @@ OCEAN_TIMESTEP = 6 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = testing_erainterim -STARTDATES = {19960104,20151231,W} {19960107,20151231,W} +STARTDATES = {19960104,20151231,D} MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 CHUNKS = 1 +CALENDAR = 366 # CHUNKS = 1 diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 3ec7d1a5..d3915998 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -299,6 +299,9 @@ class ExperimentConfig(object): import exrex self.startdates = [] for startdate_pattern in startdates: + startdate_pattern = startdate_pattern.strip() + if not startdate_pattern: + continue if startdate_pattern[0] == '{' and startdate_pattern[-1] == '}': self._read_startdates(startdate_pattern[1:-1]) else: -- GitLab From 59ca15600ddc92cf7a7e9459fa3ea4efbf5ad476 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 12:37:10 +0100 Subject: [PATCH 060/168] Updated readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f4956d66..db2b71e8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Codacy Badge](https://api.codacy.com/project/badge/Grade/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Grade) [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/206d0f75e8c64742a4fb4a0bd3015565)](https://www.codacy.com/app/BSC-Earth/earthdiagnostics?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/earthdiagnostics&utm_campaign=Badge_Coverage) + This tool is a set of diagnostics used at BSC-ES department for NEMO and EC-EARTH models postprocessing. They are based on CDO, NCO and CDFTOOLS 3.0. For CDFTOOLS, a custom build is required. -- GitLab From bb792e3adb4f78e07a1bfcbd3e76bdf83f5952a3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 15:44:26 +0100 Subject: [PATCH 061/168] Cleaned code --- diags.conf | 4 ++-- earthdiagnostics/general/rewrite.py | 1 - earthdiagnostics/general/yearlymean.py | 2 +- earthdiagnostics/ocean/averagesection.py | 2 +- earthdiagnostics/ocean/interpolatecdo.py | 2 +- earthdiagnostics/publisher.py | 10 +++++----- earthdiagnostics/variable.py | 2 +- earthdiagnostics/work_manager.py | 8 ++++---- 8 files changed, 15 insertions(+), 16 deletions(-) diff --git a/diags.conf b/diags.conf index f5a579a1..8786f6d9 100644 --- a/diags.conf +++ b/diags.conf @@ -17,7 +17,7 @@ CON_FILES = /esnas/autosubmit/con_files/ # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty #DIAGS = discretize,atmos,sfcWind,,0,40 -DIAGS = interpcdo,ocean,tas,r240x121,bilinear,False,ecmwf,False +DIAGS = interpcdo,ocean,tas,r240x121,bilinear,False,,False # DIAGS = monmean,ocean,uovmean0.0-30.0m,day monmean,ocean,vovmean0.0-30.0m,day # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. @@ -88,7 +88,7 @@ OCEAN_TIMESTEP = 6 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = testing_erainterim -STARTDATES = {19960104,20151231,D} +STARTDATES = {20050101,20161231,D} MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index 2aa937ba..7d38c0ae 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -1,6 +1,5 @@ # coding=utf-8 from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption -from earthdiagnostics.modelingrealm import ModelingRealm class Rewrite(Diagnostic): diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py index 148f0ca2..c10221d3 100644 --- a/earthdiagnostics/general/yearlymean.py +++ b/earthdiagnostics/general/yearlymean.py @@ -1,11 +1,11 @@ # coding=utf-8 import os + from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm class YearlyMean(Diagnostic): diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 8ca8abb0..8457b1d4 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -1,9 +1,9 @@ # coding=utf-8 import os + from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm class AverageSection(Diagnostic): diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 80721a5b..06bed028 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -4,7 +4,7 @@ import os import numpy as np from earthdiagnostics.diagnostic import * -from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms +from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py index 4b318b2c..0442ca28 100644 --- a/earthdiagnostics/publisher.py +++ b/earthdiagnostics/publisher.py @@ -9,10 +9,10 @@ class Publisher(object): def subscribe(self, who, callback=None): """ Add a suscriber to the current publisher - + :param who: subscriber to add :type who: object - :param callback: method to execute when publisher updates + :param callback: method to execute when publisher updates :type callback: callable | NoneType """ if callback is None: @@ -22,7 +22,7 @@ class Publisher(object): def unsubscribe(self, who): """ Removes a suscriber from the current publisher - + :param who: suscriber to remove :type who: object """ @@ -31,10 +31,10 @@ class Publisher(object): def dispatch(self, *args): """ Notify update to all the suscribers - + :param args: arguments to pass """ - for subscriber, callback in self._subscribers.items(): + for callback in self._subscribers.values(): # noinspection PyCallingNonCallable callback(*args) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 588d18e2..75526be2 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -375,7 +375,7 @@ class Variable(object): self.grid = var_line[9].strip() def get_table(self, frequency, data_convention): - for table, priority in self.tables: + for table, _ in self.tables: if table.frequency == frequency: return table if self.domain: diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index b9bc1c09..d78c8c27 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -191,20 +191,20 @@ class WorkManager(object): @staticmethod def _run_job(job): - time = datetime.datetime.now() + start_time = datetime.datetime.now() try: Log.info('Starting {0}', job) job.status = DiagnosticStatus.RUNNING job.compute() except Exception as ex: - job.consumed_time = datetime.datetime.now() - time + job.consumed_time = datetime.datetime.now() - start_time exc_type, exc_value, exc_traceback = sys.exc_info() job.message = '{0}\n{1}'.format(ex, ''.join(traceback.format_tb(exc_traceback))) - Log.error('Job {0} failed: {1}', job, job.message ) + Log.error('Job {0} failed ({2}): {1}', job, job.message, exc_type) job.status = DiagnosticStatus.FAILED return False - job.consumed_time = datetime.datetime.now() - time + job.consumed_time = datetime.datetime.now() - start_time Log.result('Finished {0}', job) job.status = DiagnosticStatus.COMPLETED return True -- GitLab From 54f551492db94e3986925aeaf6e0ea157414824e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 15:52:25 +0100 Subject: [PATCH 062/168] Fix some codacy issues --- earthdiagnostics/diagnostic.py | 15 ++++++++------- earthdiagnostics/earthdiags.py | 3 +-- earthdiagnostics/variable.py | 2 +- launch_diags.sh | 10 ++++------ 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index f038547a..dd40f171 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -271,6 +271,7 @@ class Diagnostic(Publisher): return len([request.storage_status != StorageStatus.READY or request.local_status != LocalStatus.READY for request in self._requests]) + class DiagnosticOption(object): def __init__(self, name, default_value=None): @@ -439,18 +440,18 @@ class DiagnosticChoiceOption(DiagnosticOption): if default_value is not None: self.parse(default_value) - def parse(self, value): - value = self.check_default(value) + def parse(self, option_value): + option_value = self.check_default(option_value) if self.ignore_case: - value = value.lower() + option_value = option_value.lower() for choice in self.choices: - if value == choice.lower(): + if option_value == choice.lower(): return choice else: - if value in self.choices: - return value + if option_value in self.choices: + return option_value raise DiagnosticOptionError('Value {1} in option {0} is not a valid choice. ' - 'Options are {2}'.format(self.name, value, self.choices)) + 'Options are {2}'.format(self.name, option_value, self.choices)) class DiagnosticOptionError(Exception): diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 49508aa8..b8aa3ef9 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -376,10 +376,9 @@ class EarthDiags(object): try: os.remove(destiny) except OSError as ex: - if ex.errno == 13: #Permission denied + if ex.errno == 13: # Permission denied Log.info('Link already created') return - pass os.symlink(source, destiny) Log.info('File {0} ready', destiny) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 75526be2..fadc86e4 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -324,7 +324,7 @@ class Variable(object): if 'out_name' in json_var: self.short_name = json_var['out_name'].strip() else: - raise VariableJsonException('Variable has no out name defined'.format(key)) + raise VariableJsonException('Variable {0} has no out name defined'.format(key)) self.standard_name = json_var['standard_name'].strip() self.long_name = json_var['long_name'].strip() diff --git a/launch_diags.sh b/launch_diags.sh index bcfa6b07..1c87fec2 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -12,15 +12,13 @@ PATH_TO_DIAGNOSTICS=~jvegas/earthdiagnostics PATH_TO_CONDAENV=diags module purge -module load NCO/4.5.4-foss-2015a -module load CDO/1.7.2-foss-2015a module load CDFTOOLS/3.0a8-foss-2015a module load Miniconda2 set -xv -source activate diags +source activate ${PATH_TO_CONDAENV} - export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} - cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ - ./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} +export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} +cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ +./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} -- GitLab From fa2c0d9cd4c00441d85ed4fc60587fd6fae66490 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 16:17:40 +0100 Subject: [PATCH 063/168] Fix some codacy issues --- earthdiagnostics/general/__init__.py | 12 ------- earthdiagnostics/general/module.py | 5 +-- earthdiagnostics/ocean/__init__.py | 27 +-------------- earthdiagnostics/ocean/averagesection.py | 3 +- earthdiagnostics/ocean/interpolate.py | 2 +- earthdiagnostics/ocean/interpolatecdo.py | 8 ++--- earthdiagnostics/statistics/__init__.py | 4 --- earthdiagnostics/statistics/discretize.py | 2 +- earthdiagnostics/work_manager.py | 42 +++++++++++++++++++++-- 9 files changed, 51 insertions(+), 54 deletions(-) diff --git a/earthdiagnostics/general/__init__.py b/earthdiagnostics/general/__init__.py index 34820058..9bad5790 100644 --- a/earthdiagnostics/general/__init__.py +++ b/earthdiagnostics/general/__init__.py @@ -1,13 +1 @@ # coding=utf-8 -from earthdiagnostics.general.monthlymean import MonthlyMean -from earthdiagnostics.general.dailymean import DailyMean -from earthdiagnostics.general.yearlymean import YearlyMean -from earthdiagnostics.general.rewrite import Rewrite -from earthdiagnostics.general.relink import Relink -from earthdiagnostics.general.scale import Scale -from earthdiagnostics.general.attribute import Attribute -from earthdiagnostics.general.relinkall import RelinkAll -from earthdiagnostics.general.simplify_dimensions import SimplifyDimensions -from earthdiagnostics.general.select_levels import SelectLevels -from earthdiagnostics.general.module import Module -from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index e8cf6d85..e6081cd8 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -1,8 +1,9 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import * -from earthdiagnostics.utils import Utils, TempFile import numpy as np +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, DiagnosticDomainOption, DiagnosticOption +from earthdiagnostics.utils import Utils, TempFile + class Module(Diagnostic): """ diff --git a/earthdiagnostics/ocean/__init__.py b/earthdiagnostics/ocean/__init__.py index c867bdc5..d21438cd 100644 --- a/earthdiagnostics/ocean/__init__.py +++ b/earthdiagnostics/ocean/__init__.py @@ -1,27 +1,2 @@ # coding=utf-8 -""" -Module containing the diagnostics related to the ocean output -""" -from earthdiagnostics.ocean.areamoc import AreaMoc -from earthdiagnostics.ocean.averagesection import AverageSection -from earthdiagnostics.ocean.convectionsites import ConvectionSites -from earthdiagnostics.ocean.cutsection import CutSection -from earthdiagnostics.ocean.gyres import Gyres -from earthdiagnostics.ocean.heatcontent import HeatContent -from earthdiagnostics.ocean.heatcontentlayer import HeatContentLayer -from earthdiagnostics.ocean.interpolate import Interpolate -from earthdiagnostics.ocean.interpolatecdo import InterpolateCDO -from earthdiagnostics.ocean.mask_land import MaskLand -from earthdiagnostics.ocean.maxmoc import MaxMoc -from earthdiagnostics.ocean.mixedlayerheatcontent import MixedLayerHeatContent -from earthdiagnostics.ocean.mixedlayersaltcontent import MixedLayerSaltContent -from earthdiagnostics.ocean.moc import Moc -from earthdiagnostics.ocean.mxl import Mxl -from earthdiagnostics.ocean.psi import Psi -from earthdiagnostics.ocean.regionmean import RegionMean -from earthdiagnostics.ocean.regionsum import RegionSum -from earthdiagnostics.ocean.rotation import Rotation -from earthdiagnostics.ocean.siasiesiv import Siasiesiv -from earthdiagnostics.ocean.verticalgradient import VerticalGradient -from earthdiagnostics.ocean.verticalmean import VerticalMean -from earthdiagnostics.ocean.verticalmeanmeters import VerticalMeanMeters + diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 8457b1d4..85f98ffa 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -2,7 +2,8 @@ import os from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import * +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticDomainOption, DiagnosticVariableOption, \ + DiagnosticIntOption, DiagnosticOption from earthdiagnostics.utils import Utils, TempFile diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index dab72b36..c4ac2e6d 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -177,7 +177,7 @@ class Interpolate(Diagnostic): self.grid, lev + 1) if not os.path.isfile(weights_file): raise Exception('Level {0} weights file does not exist for model {1} ' - 'and grid {2}'.format(lev+1, self.model_version, self.grid)) + 'and grid {2}'.format(lev + 1, self.model_version, self.grid)) namelist_file = TempFile.get(suffix='') scrip_use_in = open(namelist_file, 'w') scrip_use_in.writelines("&remap_inputs\n") diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 06bed028..17f2f2b5 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -3,7 +3,9 @@ import os import numpy as np -from earthdiagnostics.diagnostic import * +from earthdiagnostics.constants import Basins +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticDomainOption, DiagnosticVariableListOption, \ + DiagnosticChoiceOption, DiagnosticBoolOption, DiagnosticOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile @@ -105,7 +107,7 @@ class InterpolateCDO(Diagnostic): else: startdate, member, chunk = diags.config.experiment.get_chunk_list()[0] weights_job = ComputeWeights(diags.data_manager, startdate, member, chunk, options['domain'], - options['variables'][0],target_grid, options['original_grid'], weights, + options['variables'][0], target_grid, options['original_grid'], weights, options['method']) for var in options['variables']: @@ -294,5 +296,3 @@ class ComputeWeights(Diagnostic): def declare_data_generated(self): pass - - diff --git a/earthdiagnostics/statistics/__init__.py b/earthdiagnostics/statistics/__init__.py index 2424b993..9bad5790 100644 --- a/earthdiagnostics/statistics/__init__.py +++ b/earthdiagnostics/statistics/__init__.py @@ -1,5 +1 @@ # coding=utf-8 -from monthlypercentile import MonthlyPercentile -from climatologicalpercentile import ClimatologicalPercentile -from daysoverpercentile import DaysOverPercentile -from discretize import Discretize diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index c997e5ec..0f4478ff 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -178,7 +178,7 @@ class Discretize(Diagnostic): bins_bounds = np.zeros((self.num_bins, 2)) for x in range(self.num_bins): - bins[x] = (self.bins[x+1] - self.bins[x]) / 2 + self.bins[x] + bins[x] = (self.bins[x + 1] - self.bins[x]) / 2 + self.bins[x] bins_bounds[x, 0] = self.bins[x] bins_bounds[x, 1] = self.bins[x+1] diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index d78c8c27..51bd8b65 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -12,9 +12,6 @@ from concurrent.futures import ThreadPoolExecutor from earthdiagnostics.datafile import StorageStatus, LocalStatus from earthdiagnostics.diagnostic import DiagnosticStatus, Diagnostic, DiagnosticOptionError -from earthdiagnostics.general import * -from earthdiagnostics.ocean import * -from earthdiagnostics.statistics import * from earthdiagnostics.utils import Utils, TempFile @@ -219,6 +216,11 @@ class WorkManager(object): @staticmethod def _register_stats_diagnostics(): + from earthdiagnostics.statistics.monthlypercentile import MonthlyPercentile + from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile + from earthdiagnostics.statistics.daysoverpercentile import DaysOverPercentile + from earthdiagnostics.statistics.discretize import Discretize + Diagnostic.register(MonthlyPercentile) Diagnostic.register(ClimatologicalPercentile) Diagnostic.register(DaysOverPercentile) @@ -226,6 +228,18 @@ class WorkManager(object): @staticmethod def _register_general_diagnostics(): + from earthdiagnostics.general.attribute import Attribute + from earthdiagnostics.general.dailymean import DailyMean + from earthdiagnostics.general.module import Module + from earthdiagnostics.general.monthlymean import MonthlyMean + from earthdiagnostics.general.yearlymean import YearlyMean + from earthdiagnostics.general.rewrite import Rewrite + from earthdiagnostics.general.relink import Relink + from earthdiagnostics.general.relinkall import RelinkAll + from earthdiagnostics.general.scale import Scale + from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris + from earthdiagnostics.general.simplify_dimensions import SimplifyDimensions + Diagnostic.register(DailyMean) Diagnostic.register(MonthlyMean) Diagnostic.register(YearlyMean) @@ -240,6 +254,28 @@ class WorkManager(object): @staticmethod def _register_ocean_diagnostics(): + from earthdiagnostics.ocean.mixedlayerheatcontent import MixedLayerHeatContent + from earthdiagnostics.ocean.mixedlayersaltcontent import MixedLayerSaltContent + from earthdiagnostics.ocean.siasiesiv import Siasiesiv + from earthdiagnostics.ocean.verticalmean import VerticalMean + from earthdiagnostics.ocean.verticalmeanmeters import VerticalMeanMeters + from earthdiagnostics.ocean.verticalgradient import VerticalGradient + from earthdiagnostics.ocean.interpolate import Interpolate + from earthdiagnostics.ocean.interpolatecdo import InterpolateCDO + from earthdiagnostics.ocean.moc import Moc + from earthdiagnostics.ocean.areamoc import AreaMoc + from earthdiagnostics.ocean.maxmoc import MaxMoc + from earthdiagnostics.ocean.psi import Psi + from earthdiagnostics.ocean.gyres import Gyres + from earthdiagnostics.ocean.convectionsites import ConvectionSites + from earthdiagnostics.ocean.cutsection import CutSection + from earthdiagnostics.ocean.averagesection import AverageSection + from earthdiagnostics.ocean.heatcontentlayer import HeatContentLayer + from earthdiagnostics.ocean.heatcontent import HeatContent + from earthdiagnostics.ocean.regionmean import RegionMean + from earthdiagnostics.ocean.regionsum import RegionSum + from earthdiagnostics.ocean.rotation import Rotation + Diagnostic.register(MixedLayerSaltContent) Diagnostic.register(Siasiesiv) Diagnostic.register(VerticalMean) -- GitLab From 2f2af0add5722065edb3738639a1150148736613 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 16:42:40 +0100 Subject: [PATCH 064/168] Fix some codacy issues --- earthdiagnostics/cmormanager.py | 4 +-- earthdiagnostics/config.py | 1 - earthdiagnostics/datafile.py | 2 +- earthdiagnostics/general/scale.py | 4 ++- .../general/verticalmeanmetersiris.py | 4 +-- earthdiagnostics/ocean/mask_land.py | 5 ++- earthdiagnostics/ocean/verticalgradient.py | 6 ++-- earthdiagnostics/statistics/discretize.py | 4 +-- earthdiagnostics/threddsmanager.py | 27 ++++++++-------- fix.py | 32 ------------------- patch.bash | 18 ----------- 11 files changed, 28 insertions(+), 79 deletions(-) delete mode 100644 fix.py delete mode 100755 patch.bash diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index a0237d94..1c4b5d9b 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -336,7 +336,7 @@ class CMORManager(DataManager): time_bound = "{0:04}{1:02}".format(chunk_start.year, chunk_start.month) else: time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, - chunk_end.month, separator) + chunk_end.month, separator) return time_bound def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, @@ -715,6 +715,6 @@ class MergeYear(Diagnostic): 'Variable: {0.domain}:{0.var} Grid: {0.grid} Box: {0.box}'.format(self) def __eq__(self, other): - return self.startdate == other.startdate and self.member == other.member and self.year == other.year and\ + return self.startdate == other.startdate and self.member == other.member and self.year == other.year and \ self.domain == other.domain and self.var == other.var and self.grid == other.grid and \ self.box == other.box diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index d3915998..c07dccff 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -343,7 +343,6 @@ class ExperimentConfig(object): else: raise ConfigException('Interval {0} not supported in STARTDATES definition: {1}', interval, pattern) - def get_chunk_list(self): """ Return a list with all the chunks diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index b28f319a..ceab3aad 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -75,7 +75,7 @@ class DataFile(Publisher): def clean_local(self): if self.local_status != LocalStatus.READY or len(self.suscribers) > 0 or self.upload_required() or \ - self.storage_status == StorageStatus.UPLOADING: + self.storage_status == StorageStatus.UPLOADING: return Log.debug('File {0} no longer needed. Deleting from scratch...'.format(self.remote_file)) os.remove(self.local_file) diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 099d6774..0e650486 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -3,7 +3,9 @@ import math import numpy as np -from earthdiagnostics.diagnostic import * +from earthdiagnostics.constants import Basins +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticDomainOption, DiagnosticVariableOption, \ + DiagnosticFloatOption, DiagnosticBoolOption, DiagnosticListFrequenciesOption, DiagnosticOption from earthdiagnostics.utils import Utils diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py index 23dce4d8..98db4d86 100644 --- a/earthdiagnostics/general/verticalmeanmetersiris.py +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -119,8 +119,8 @@ class VerticalMeanMetersIris(Diagnostic): lev_max = coord.points[-1] else: lev_max = self.box.max_depth - var_cube = var_cube.extract(iris.Constraint(coord_values= - {coord.var_name: lambda cell: lev_min <= cell <= lev_max})) + lev_constraint = iris.Constraint(coord_values={coord.var_name: lambda cell: lev_min <= cell <= lev_max}) + var_cube = var_cube.extract(lev_constraint) var_cube = var_cube.collapsed(coord, iris.analysis.MEAN) temp = TempFile.get() iris.save(var_cube, temp, zlib=True) diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index 24370378..1f6c7dd0 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -1,8 +1,9 @@ # coding=utf-8 +import numpy as np + from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableListOption, \ DiagnosticDomainOption, DiagnosticChoiceOption, DiagnosticOption from earthdiagnostics.utils import Utils, TempFile -import numpy as np class MaskLand(Diagnostic): @@ -84,8 +85,6 @@ class MaskLand(Diagnostic): mask_file.close() return mask - - def request_data(self): self.var_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 9cd9ab61..7c8e1fcc 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -1,8 +1,8 @@ # coding=utf-8 from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticVariableOption -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class VerticalGradient(Diagnostic): @@ -96,8 +96,8 @@ class VerticalGradient(Diagnostic): if 'lev' not in handler.dimensions: raise Exception('Variable {0} does not have a level dimension') var_handler = handler.variables[self.variable] - upper_level = var_handler[:, self.box.min_depth-1, ...] - lower_level = var_handler[:, self.box.max_depth-1, ...] + upper_level = var_handler[:, self.box.min_depth - 1, ...] + lower_level = var_handler[:, self.box.max_depth - 1, ...] gradient = upper_level - lower_level temp = TempFile.get() diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index 0f4478ff..38469cc5 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -200,8 +200,8 @@ class Discretize(Diagnostic): units='months')) lead_date = add_months(date, leadtime - 1, self.data_manager.config.experiment.calendar) leadtime_cube.add_aux_coord(iris.coords.AuxCoord(cf_units.date2num(lead_date, - unit='days since 1950-01-01', - calendar="standard"), + unit='days since 1950-01-01', + calendar="standard"), var_name='time', units='days since 1950-01-01')) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 8637121a..2ba4f6f8 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -1,5 +1,6 @@ # coding=utf-8 import os +from datetime import datetime from time import strptime import iris @@ -7,14 +8,12 @@ import netCDF4 import numpy as np from bscearth.utils.date import parse_date, add_months, chunk_start_date, chunk_end_date from bscearth.utils.log import Log -from iris.coords import DimCoord from cf_units import Unit +from iris.coords import DimCoord from datafile import DataFile, StorageStatus, LocalStatus from earthdiagnostics.datamanager import DataManager from earthdiagnostics.utils import TempFile, Utils -from datetime import datetime - from earthdiagnostics.variable_type import VariableType @@ -205,7 +204,7 @@ class THREDDSManager(DataManager): """ Creates the link of a given file from the CMOR repository. - :param cmor_var: + :param cmor_var: :param move_old: :param date_str: :param year: if frequency is yearly, this parameter is used to give the corresponding year @@ -239,7 +238,7 @@ class THREDDSManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param vartype: + :param vartype: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -277,8 +276,8 @@ class THREDDSManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param diagnostic: - :param region: + :param diagnostic: + :param region: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -326,13 +325,13 @@ class THREDDSError(Exception): class THREDDSSubset(DataFile): def __init__(self, thredds_path, file_path, var, start_time, end_time): """ - - :param thredds_path: - :param file_path: - :param var: + + :param thredds_path: + :param file_path: + :param var: :type var: str - :param start_time: - :param end_time: + :param start_time: + :param end_time: """ super(THREDDSSubset, self).__init__() self.thredds_path = thredds_path @@ -379,7 +378,7 @@ class THREDDSSubset(DataFile): if time.units.origin.startswith('month'): ref = strptime(time.units.origin[time.units.origin.index(' since ') + 7:], '%Y-%m-%d %H:%M:%S') helper = np.vectorize(lambda x: datetime(year=ref.tm_year + int(x) / 12, - month=int(x-1) % 12 + 1, + month=int(x - 1) % 12 + 1, day=ref.tm_mday)) times = np.round(time.points + ref.tm_mon) dates = helper(times) diff --git a/fix.py b/fix.py deleted file mode 100644 index b5444a53..00000000 --- a/fix.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import shutil - -base_path = '/esarchive/exp/PREFACE/ctrl/cmorfiles/Cerfacs' -for experiment in os.listdir(base_path): - exp_path = os.path.join(base_path, experiment) - if not os.path.isdir(exp_path): - continue - for startdate in os.listdir(exp_path): - if not os.path.isdir(os.path.join(exp_path, startdate)): - continue - for freq in os.listdir(os.path.join(exp_path, startdate)): - for domain in os.listdir(os.path.join(exp_path, startdate, freq)): - for var in os.listdir(os.path.join(exp_path, startdate, freq, domain)): - for member in os.listdir(os.path.join(exp_path, startdate, freq, domain, var)): - for version in os.listdir(os.path.join(exp_path, startdate, freq, domain, var, member)): - for filename in os.listdir(os.path.join(exp_path, startdate, freq, domain, var, member, version)): - print(os.path.join(exp_path, startdate, freq, domain, var, member, version, filename)) - print(os.path.join(exp_path, startdate, freq, domain, var, member, version, - filename.replace('_CNRM-CM-HR_', '_CNRM-CM-HR_{0}_'.format(experiment)))) - print('') - shutil.move(os.path.join(exp_path, startdate, freq, domain, var, member, version, filename), - os.path.join(exp_path, startdate, freq, domain, var, member, version, - filename.replace('_CNRM-CM-HR_', '_CNRM-CM-HR_{0}_'.format(experiment)))) - - - # original_tos_path = os.path.join(exp_path, startdate, 'mon/atmos/tos') - # if os.path.isdir(original_tos_path): - # new_tos_path = os.path.join(exp_path, startdate, 'mon/ocean') - # if not os.path.isdir(new_tos_path): - # os.makedirs(new_tos_path) - # shutil.move(original_tos_path, new_tos_path) \ No newline at end of file diff --git a/patch.bash b/patch.bash deleted file mode 100755 index 60e7b8f8..00000000 --- a/patch.bash +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -o nounset -set -o errexit - -exp=a0fj -sd=19580101 -root=esarchive -mod=nemo - -if [ -d /${root}/exp/${mod}/${exp}/original_files ] -then - echo "folder already exists" - exit -fi - -mkdir -p /${root}/exp/${mod}/${exp}/original_files -mv /${root}/exp/${mod}/${exp}/${sd} /${root}/exp/${mod}/${exp}/original_files -ln -sf /${root}/exp/${mod}/${exp}/original_files/${sd} /${root}/exp/${mod}/${exp}/${sd} -- GitLab From 531b86e6a6ee380b1e7c43e514bb4032ab83978b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 16:56:19 +0100 Subject: [PATCH 065/168] Fix some codacy issues --- earthdiagnostics/box.py | 10 ++-- earthdiagnostics/cdftools.py | 32 ++++++------- earthdiagnostics/cmorizer.py | 24 ++++------ earthdiagnostics/diagnostic.py | 3 +- earthdiagnostics/obsreconmanager.py | 8 ++-- earthdiagnostics/ocean/heatcontent.py | 2 +- .../ocean/mixedlayerheatcontent.py | 6 +-- .../ocean/mixedlayersaltcontent.py | 5 +- earthdiagnostics/ocean/moc.py | 4 +- earthdiagnostics/ocean/mxl.py | 2 +- earthdiagnostics/ocean/psi.py | 4 +- earthdiagnostics/ocean/regionmean.py | 2 +- earthdiagnostics/ocean/regionsum.py | 2 +- earthdiagnostics/ocean/verticalmean.py | 4 +- earthdiagnostics/ocean/verticalmeanmeters.py | 4 +- .../statistics/daysoverpercentile.py | 3 +- test/unit/test_cdftools.py | 48 +++++++++---------- 17 files changed, 78 insertions(+), 85 deletions(-) diff --git a/earthdiagnostics/box.py b/earthdiagnostics/box.py index ea7ccc15..51cefe96 100644 --- a/earthdiagnostics/box.py +++ b/earthdiagnostics/box.py @@ -88,7 +88,7 @@ class Box(object): if value >= 360 or value <= -360: raise ValueError('{0} is not a valid longitude. Must be between -360 and 360'.format(value)) self._min_lon = value - + def get_lat_str(self): """ Gets a string representation of the latitude in the format XX{N/S}. @@ -102,16 +102,16 @@ class Box(object): direction = 'S' else: direction = 'N' - + string = str(abs(self.min_lat)) + direction - + if self.max_lat != self.min_lat: if self.max_lat < 0: direction = 'S' else: direction = 'N' string += str(abs(self.max_lat)) + direction - + return string def get_lon_str(self): @@ -158,5 +158,3 @@ class Box(object): else: string = '{0:d}{1}'.format(int(abs(self.max_depth)), suffix) return string - - diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index ad0355c3..9537dc86 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -20,15 +20,15 @@ class CDFTools(object): self.data_convention = '' # noinspection PyShadowingBuiltins - def run(self, command, input, output=None, options=None, log_level=Log.INFO, input_option=None): + def run(self, command, input_file, output_file=None, options=None, log_level=Log.INFO, input_option=None): """ Runs one of the CDFTools :param command: executable to run :type command: str | iterable - :param input: input file - :type input: str - :param output: output file. Not all tools support this parameter + :param input_file: input file + :type input_file: str + :param output_file: output file. Not all tools support this parameter :type options: str :param options: options for the tool. :type options: str | [str] | Tuple[str] | NoneType @@ -43,22 +43,22 @@ class CDFTools(object): self._check_command_existence(line[0]) if input_option: line.append(input_option) - self._check_input(command, input, line) + self._check_input(command, input_file, line) if options: if isinstance(options, six.string_types): options = options.split() for option in options: line.append(str(option)) - if output: - if input == output: + if output_file: + if input_file == output_file: raise ValueError('Input and output file can not be the same on CDFTools') line.append('-o') - line.append(output) + line.append(output_file) Log.debug('Executing {0}', ' '.join(line)) shell_output = Utils.execute_shell_command(line, log_level) - self._check_output_was_created(line, output) + self._check_output_was_created(line, output_file) return shell_output @staticmethod @@ -69,14 +69,14 @@ class CDFTools(object): # noinspection PyShadowingBuiltins @staticmethod - def _check_input(command, input, line): - if input: - if isinstance(input, six.string_types): - line.append(input) - if not os.path.isfile(input): - raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, input) + def _check_input(command, input_file, line): + if input_file: + if isinstance(input_file, six.string_types): + line.append(input_file) + if not os.path.isfile(input_file): + raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, input_file) else: - for element in input: + for element in input_file: line.append(element) if not os.path.isfile(element): raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 2e684132..7ec4d346 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -368,11 +368,12 @@ class Cmorizer(object): Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) Utils.rename_variables(filename, self.alt_coord_names, False, True) - self._remove_valid_limits(filename) - self._add_common_attributes(filename, frequency) - self._update_time_variables(filename) - handler = Utils.openCdf(filename) + self._remove_valid_limits(handler) + self._add_common_attributes(handler, frequency) + self._update_time_variables(handler) + handler.sync() + Log.info('Splitting file {0}', filename) for variable in handler.variables.keys(): if variable in Cmorizer.NON_DATA_VARIABLES: @@ -385,7 +386,6 @@ class Cmorizer(object): handler.close() os.remove(filename) - # noinspection PyMethodMayBeStatic def _remove_valid_limits(self, filename): handler = Utils.openCdf(filename) for variable in handler.variables.keys(): @@ -627,8 +627,7 @@ class Cmorizer(object): self._cmorize_nc_file(merged_file) - def _update_time_variables(self, filename): - handler = Utils.openCdf(filename) + def _update_time_variables(self, handler): time_var = handler.variables['time'] if hasattr(time_var, 'calendar'): calendar = time_var.calendar @@ -639,11 +638,9 @@ class Cmorizer(object): handler.variables['time_bnds'].units = time_var.units Utils.convert_units(handler.variables['time_bnds'], 'days since 1850-01-01 00:00:00', calendar, calendar) Utils.convert_units(time_var, 'days since 1850-1-1 00:00:00', calendar) - handler.close() - self._set_leadtime_var(filename) + self._set_leadtime_var(handler) - def _set_leadtime_var(self, filename): - handler = Utils.openCdf(filename) + def _set_leadtime_var(self, handler): if 'leadtime' in handler.variables: var = handler.variables['leadtime'] else: @@ -657,12 +654,10 @@ class Cmorizer(object): for time in leadtime] for lt in range(0, len(leadtime)): var[lt] = leadtime[lt].days - handler.close() - def _add_common_attributes(self, filename, frequency): + def _add_common_attributes(self, handler, frequency): cmor = self.config.cmor experiment = self.config.experiment - handler = Utils.openCdf(filename) handler.associated_experiment = cmor.associated_experiment handler.batch = '{0}{1}'.format(experiment.institute, datetime.now().strftime('%Y-%m-%d(T%H:%M:%SZ)')) handler.contact = 'Pierre-Antoine Bretonniere, pierre-antoine.bretonniere@bsc.es , ' \ @@ -687,7 +682,6 @@ class Cmorizer(object): handler.tracking_id = str(uuid.uuid1()) handler.title = "{0} model output prepared for {2} {1}".format(experiment.model, experiment.experiment_name, self.config.data_convention.upper()) - handler.close() def gribfiles_available(self): grb_path = os.path.join(self.original_files_path, '*.grb') diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index dd40f171..35fe4fa3 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -198,8 +198,7 @@ class Diagnostic(Publisher): if len(options) > len(options_available): raise DiagnosticOptionError('You have specified more options than available for diagnostic ' '{0}'.format(cls.alias)) - for x in range(len(options_available)): - option_definition = options_available[x] + for x, option_definition in enumerate(options_available): if len(options) <= x: option_value = '' else: diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index 356eac77..fa99ad5e 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -164,7 +164,7 @@ class ObsReconManager(DataManager): """ Creates the link of a given file from the CMOR repository. - :param cmor_var: + :param cmor_var: :param move_old: :param date_str: :param year: if frequency is yearly, this parameter is used to give the corresponding year @@ -196,7 +196,7 @@ class ObsReconManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param vartype: + :param vartype: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -227,8 +227,8 @@ class ObsReconManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param diagnostic: - :param region: + :param diagnostic: + :param region: :param domain: CMOR domain :type domain: Domain :param var: variable name diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 81ef23cc..a5848325 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -200,7 +200,7 @@ class HeatContent(Diagnostic): temp2 = TempFile.get() - cdftools.run('cdfheatc', options=para, input=temperature_file, output=temp2, input_option='-f') + cdftools.run('cdfheatc', options=para, input_file=temperature_file, output_file=temp2, input_option='-f') results = Utils.openCdf(temp2) heatcsum_temp = TempFile.get() diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index e81624d2..8767298e 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -1,10 +1,10 @@ # coding=utf-8 import os -from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics import cdftools -from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class MixedLayerHeatContent(Diagnostic): @@ -79,7 +79,7 @@ class MixedLayerHeatContent(Diagnostic): Utils.nco.ncks(input=self.mlotst.local_file, output=temperature_file, options=('-A -v mlotst',)) temp = TempFile.get() - cdftools.run('cdfmxlheatc', input=temperature_file, output=temp) + cdftools.run('cdfmxlheatc', input_file=temperature_file, output_file=temp) os.remove(temperature_file) diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index ad350961..0ffe00a5 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -1,9 +1,10 @@ # coding=utf-8 import os + from earthdiagnostics import cdftools from earthdiagnostics.diagnostic import Diagnostic -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class MixedLayerSaltContent(Diagnostic): @@ -77,7 +78,7 @@ class MixedLayerSaltContent(Diagnostic): Utils.nco.ncks(input=self.mlotst.local_file, output=salinity_file, options=('-A -v mlotst',)) temp = TempFile.get() - cdftools.run('cdfmxlsaltc', input=salinity_file, output=temp) + cdftools.run('cdfmxlsaltc', input_file=salinity_file, output_file=temp) os.remove(salinity_file) Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlsaltc': 'scvsummlotst'}, False, True) diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index 7686e84b..4a9cd6a6 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -5,8 +5,8 @@ from bscearth.utils.log import Log from earthdiagnostics import cdftools from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import Diagnostic -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class Moc(Diagnostic): @@ -79,7 +79,7 @@ class Moc(Diagnostic): temp = TempFile.get() Log.debug('Computing MOC') - cdftools.run('cdfmoc', input=self.variable_file.local_file, output=temp) + cdftools.run('cdfmoc', input_file=self.variable_file.local_file, output_file=temp) Utils.nco.ncks(input=self.variable_file.local_file, output=temp, options=('-A -v lev',)) Utils.convert2netcdf4(temp) diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py index b5e7c980..ce001cf2 100644 --- a/earthdiagnostics/ocean/mxl.py +++ b/earthdiagnostics/ocean/mxl.py @@ -66,7 +66,7 @@ class Mxl(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - cdftools.run('cdfmxl', input=[self.thetao_file, self.so_file], output=temp, options='-nc4') + cdftools.run('cdfmxl', input_file=[self.thetao_file, self.so_file], output_file=temp, options='-nc4') temp2 = TempFile.get() source = Utils.openCdf(temp) destiny = Utils.openCdf(temp2, 'w') diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index fd1ee553..bb5a5b16 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -1,8 +1,8 @@ # coding=utf-8 from earthdiagnostics import cdftools from earthdiagnostics.diagnostic import Diagnostic -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class Psi(Diagnostic): @@ -72,7 +72,7 @@ class Psi(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - cdftools.run('cdfpsi', input=[self.uo.local_file, self.vo.local_file], output=temp, options='-mean -mask') + cdftools.run('cdfpsi', input_file=[self.uo.local_file, self.vo.local_file], output_file=temp, options='-mean -mask') Utils.rename_variable(temp, 'sobarstf', Psi.vsftbarot) Utils.setminmax(temp, Psi.vsftbarot) self.psi.set_local_file(temp) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 8bfce0de..397f1711 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -140,7 +140,7 @@ class RegionMean(Diagnostic): cdfmean_options.append('mask_regions.3d.nc') cdfmean_options.append(self.basin.name) - cdftools.run('cdfmean', input=variable_file, output=mean_file, options=cdfmean_options) + cdftools.run('cdfmean', input_file=variable_file, output_file=mean_file, options=cdfmean_options) Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) self.send_var('mean', False, mean_file) diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index 0f4c59a9..5cdc39c8 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -134,7 +134,7 @@ class RegionSum(Diagnostic): cdfmean_options.append('mask_regions.3d.nc') cdfmean_options.append(self.basin.name) - cdftools.run('cdfsum', input=variable_file, input_option='-f', output=mean_file, options=cdfmean_options) + cdftools.run('cdfsum', input_file=variable_file, input_option='-f', output_file=mean_file, options=cdfmean_options) Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) self.send_var(False, mean_file) diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 21bd4a88..d47f538f 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -2,8 +2,8 @@ from earthdiagnostics import cdftools from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticVariableOption -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class VerticalMean(Diagnostic): @@ -106,7 +106,7 @@ class VerticalMean(Diagnostic): lev_max = self.box.max_depth handler.close() - cdftools.run('cdfvertmean', input=self.variable_file.local_file, output=temp, + cdftools.run('cdfvertmean', input_file=self.variable_file.local_file, output_file=temp, options=[self.variable, 'T', lev_min, lev_max, '-debug']) Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) self.results.set_local_file(temp, rename_var='{0}_vert_mean'.format(self.variable)) diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 8951f493..50c5e9ad 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -3,8 +3,8 @@ from earthdiagnostics import cdftools from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticFloatOption, DiagnosticDomainOption, \ DiagnosticVariableOption, DiagnosticChoiceOption -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class VerticalMeanMeters(Diagnostic): @@ -108,7 +108,7 @@ class VerticalMeanMeters(Diagnostic): lev_max = self.box.max_depth handler.close() - cdftools.run('cdfvertmean', input=self.variable_file.local_file, output=temp, + cdftools.run('cdfvertmean', input_file=self.variable_file.local_file, output_file=temp, options=[self.variable, self.grid_point, lev_min, lev_max, '-debug']) Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) self.results.set_local_file(temp, rename_var='{0}_vert_mean'.format(self.variable)) diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index 663618af..91d46958 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -190,7 +190,8 @@ class DaysOverPercentile(Diagnostic): del self.lat_coord del self.lon_coord - def save_to_file(self, perc, results_over, var_daysover): + @staticmethod + def save_to_file(perc, results_over, var_daysover): temp = TempFile.get() iris.save(results_over[perc].merge_cube(), temp, zlib=True, unlimited_dimensions=['time']) Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, diff --git a/test/unit/test_cdftools.py b/test/unit/test_cdftools.py index 28de4ac3..bc7ed38e 100644 --- a/test/unit/test_cdftools.py +++ b/test/unit/test_cdftools.py @@ -1,10 +1,10 @@ # coding=utf-8 +import os from unittest import TestCase -import os +import mock from earthdiagnostics.cdftools import CDFTools -import mock # noinspection PyUnusedLocal @@ -22,22 +22,22 @@ class TestCDFTools(TestCase): self.cdftools = CDFTools('/test/path') execute_mock.return_value = ['Command output'] with self.assertRaises(ValueError): - self.cdftools.run('badcommand', input='input_file', output='output_file') + self.cdftools.run('badcommand', input_file='input_file', output_file='output_file') with self.assertRaises(ValueError): - self.cdftools.run('command', input='badinput_file', output='output_file') + self.cdftools.run('command', input_file='badinput_file', output_file='output_file') with self.assertRaises(ValueError): - self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') + self.cdftools.run('command', input_file=['input_file', 'badinput_file'], output_file='output_file') with self.assertRaises(ValueError): - self.cdftools.run('command', input='input_file', output='input_file') + self.cdftools.run('command', input_file='input_file', output_file='input_file') with self.assertRaises(Exception): - self.cdftools.run('command', input='input_file', output='badoutput_file') + self.cdftools.run('command', input_file='input_file', output_file='badoutput_file') - self.cdftools.run('command', input='input_file', output='output_file') - self.cdftools.run('command', input='input_file') - self.cdftools.run('command', input=None) - self.cdftools.run('command', input=['input_file', 'input_file2']) - self.cdftools.run('command', input='input_file', options='-o -p') - self.cdftools.run('command', input='input_file', options=('-o', '-p')) + self.cdftools.run('command', input_file='input_file', output_file='output_file') + self.cdftools.run('command', input_file='input_file') + self.cdftools.run('command', input_file=None) + self.cdftools.run('command', input_file=['input_file', 'input_file2']) + self.cdftools.run('command', input_file='input_file', options='-o -p') + self.cdftools.run('command', input_file='input_file', options=('-o', '-p')) # noinspection PyUnusedLocal @mock.patch('os.path.isfile', side_effect=bad_file) @@ -47,19 +47,19 @@ class TestCDFTools(TestCase): self.cdftools = CDFTools('') execute_mock.return_value = ['Command output'] with self.assertRaises(ValueError): - self.cdftools.run('badcommand', input='input_file', output='output_file') + self.cdftools.run('badcommand', input_file='input_file', output_file='output_file') with self.assertRaises(ValueError): - self.cdftools.run('command', input='badinput_file', output='output_file') + self.cdftools.run('command', input_file='badinput_file', output_file='output_file') with self.assertRaises(ValueError): - self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') + self.cdftools.run('command', input_file=['input_file', 'badinput_file'], output_file='output_file') with self.assertRaises(ValueError): - self.cdftools.run('command', input='input_file', output='input_file') + self.cdftools.run('command', input_file='input_file', output_file='input_file') with self.assertRaises(Exception): - self.cdftools.run('command', input='input_file', output='badoutput_file') + self.cdftools.run('command', input_file='input_file', output_file='badoutput_file') - self.cdftools.run('command', input='input_file', output='output_file') - self.cdftools.run('command', input='input_file') - self.cdftools.run('command', input=None) - self.cdftools.run('command', input=['input_file', 'input_file2']) - self.cdftools.run('command', input='input_file', options='-o -p') - self.cdftools.run('command', input='input_file', options=('-o', '-p')) + self.cdftools.run('command', input_file='input_file', output_file='output_file') + self.cdftools.run('command', input_file='input_file') + self.cdftools.run('command', input_file=None) + self.cdftools.run('command', input_file=['input_file', 'input_file2']) + self.cdftools.run('command', input_file='input_file', options='-o -p') + self.cdftools.run('command', input_file='input_file', options=('-o', '-p')) -- GitLab From 67f43ec5c439ac251e369cb5a68b7801734d3789 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 17:55:55 +0100 Subject: [PATCH 066/168] Fix some codacy issues --- earthdiagnostics/datafile.py | 2 +- earthdiagnostics/ocean/moc.py | 2 +- earthdiagnostics/ocean/psi.py | 3 +- earthdiagnostics/ocean/regionsum.py | 4 +- earthdiagnostics/utils.py | 323 ++++++++++++++++++++-------- 5 files changed, 235 insertions(+), 99 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index ceab3aad..8c20aa30 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -404,7 +404,7 @@ class DataFile(Publisher): history_line = history_line + handler.history except AttributeError: history_line = history_line - handler.history = Utils.convert_to_ASCII_if_possible(history_line) + handler.history = Utils.convert_to_ascii_if_possible(history_line) handler.close() diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index 4a9cd6a6..f66243f7 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -100,7 +100,7 @@ class Moc(Diagnostic): ('time', 'lev', 'i', 'j', 'basin'), fill_value=example._FillValue) - moc.units = Utils.convert_to_ASCII_if_possible(example.units) + moc.units = Utils.convert_to_ascii_if_possible(example.units) moc.add_offset = example.add_offset moc.scale_factor = example.scale_factor diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index bb5a5b16..e67a25bc 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -72,7 +72,8 @@ class Psi(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - cdftools.run('cdfpsi', input_file=[self.uo.local_file, self.vo.local_file], output_file=temp, options='-mean -mask') + cdftools.run('cdfpsi', input_file=[self.uo.local_file, self.vo.local_file], output_file=temp, + options='-mean -mask') Utils.rename_variable(temp, 'sobarstf', Psi.vsftbarot) Utils.setminmax(temp, Psi.vsftbarot) self.psi.set_local_file(temp) diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index 5cdc39c8..b2068d1c 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -134,7 +134,8 @@ class RegionSum(Diagnostic): cdfmean_options.append('mask_regions.3d.nc') cdfmean_options.append(self.basin.name) - cdftools.run('cdfsum', input_file=variable_file, input_option='-f', output_file=mean_file, options=cdfmean_options) + cdftools.run('cdfsum', input_file=variable_file, input_option='-f', output_file=mean_file, + options=cdfmean_options) Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) self.send_var(False, mean_file) @@ -177,4 +178,3 @@ class RegionSum(Diagnostic): self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, box=box_save, region=self.basin, grid=self.grid) - diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index c70960eb..683d9659 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -1,4 +1,7 @@ # coding=utf-8 +""" +Common utilities for multiple topics that are not big enough to have their own module +""" import datetime import os import re @@ -26,6 +29,9 @@ from earthdiagnostics.constants import Basins @contextmanager def suppress_stdout(): + """ + Redirects the standard output to devnull + """ with open(os.devnull, "w") as devnull: old_stdout = sys.stdout sys.stdout = devnull @@ -35,10 +41,6 @@ def suppress_stdout(): sys.stdout = old_stdout -class File(object): - pass - - class Utils(object): """ Container class for miscellaneous utility methods @@ -52,14 +54,21 @@ class Utils(object): @staticmethod def get_mask(basin): """ - Returns a numpy array containing the mask for the given basin + Return the mask for the given basin - :param basin: basin to retrieve - :type basin: Basin - :return: mask - :rtype: numpy.array - """ + Parameters + ---------- + basin: Basin + Returns + ------- + numpy.array + + Raises + ------ + + Exception: If mask.regions.nc is not available + """ basin = Basins().parse(basin) if basin != Basins().Global: try: @@ -76,6 +85,16 @@ class Utils(object): @staticmethod def setminmax(filename, variable_list): + """ + Set the valid_max and valid_min values to the current max and min values on the file + + Parameters + ---------- + filename: str + variable_list: str or iterable of str + + """ + """ Sets the valid_max and valid_min values to the current max and min values on the file :param filename: path to file @@ -102,17 +121,22 @@ class Utils(object): @staticmethod def rename_variable(filepath, old_name, new_name, must_exist=True, rename_dimension=False): """ - Rename multiple variables from a NetCDF file - :param filepath: path to file - :type filepath: str - :param old_name: variable's name to change - :type old_name: str - :param new_name: new name - :type new_name: str - :param must_exist: if True, the function will raise an exception if the variable name does not exist - :type must_exist: bool - :param rename_dimension: if True, also rename dimensions with the same name - :type rename_dimension: bool + Rename variable from a NetCDF file + + This function is just a wrapper around Utils.rename_variables + + Parameters + ---------- + filepath: str + old_name: str + new_name: str + must_exist: bool, optional + rename_dimension: bool, optional + + See Also + -------- + Utils.rename_variables + """ Utils.rename_variables(filepath, {old_name: new_name}, must_exist, rename_dimension) @@ -120,14 +144,21 @@ class Utils(object): def rename_variables(filepath, dic_names, must_exist=True, rename_dimension=False): """ Rename multiple variables from a NetCDF file - :param filepath: path to file - :type filepath: str - :param dic_names: dictionary containing old names as keys and new names as values - :type dic_names: dict - :param must_exist: if True, the function will raise an exception if the variable name does not exist - :type must_exist: bool - :param rename_dimension: if True, also rename dimensions with the same name - :type rename_dimension: bool + + Parameters + ---------- + filepath: str + dic_names: dict of str: str + Gives the renaming to do in the form old_name: new_name + must_exist: bool, optional + rename_dimension: bool, optional + + Raises + ------- + ValueError + If any original name is the same as the new + Exception + If any requested variable does not exist and must_exist is True """ for old, new in six.iteritems(dic_names): if old == new: @@ -167,6 +198,20 @@ class Utils(object): @staticmethod def check_netcdf_file(filepath): + """ + Check if a NetCDF file is well stored + + This functions is used to check if a NetCDF file is corrupted. It prefers to raise a false postive than + to have false negatives. + + Parameters + ---------- + filepath + + Returns + ------- + bool + """ with suppress_stdout(): try: handler = Utils.openCdf(filepath) @@ -195,6 +240,18 @@ class Utils(object): @staticmethod def get_file_variables(filename): + """ + Get all the variables in a file + + Parameters + ---------- + filename + + Returns + ------- + iterable of str + + """ handler = Utils.openCdf(filename) variables = handler.variables.keys() handler.close() @@ -206,7 +263,7 @@ class Utils(object): new_handler = Utils.openCdf(temp, 'w') for attribute in original_handler.ncattrs(): original = getattr(original_handler, attribute) - setattr(new_handler, attribute, Utils.convert_to_ASCII_if_possible(original)) + setattr(new_handler, attribute, Utils.convert_to_ascii_if_possible(original)) for dimension in original_handler.dimensions.keys(): Utils.copy_dimension(original_handler, new_handler, dimension, new_names=dic_names) for variable in original_handler.variables.keys(): @@ -216,15 +273,31 @@ class Utils(object): # noinspection PyPep8Naming @staticmethod - def convert_to_ASCII_if_possible(string, encoding='ascii'): + def convert_to_ascii_if_possible(string, encoding='ascii'): + """ + Convert an Unicode string to ASCII if all characters can be translated. + + If a string can not be translated it is unchanged. It also automatically + replaces Bretonnière with Bretonniere + + Parameters + ---------- + string: unicode + encoding: str, optional + + Returns + ------- + str + + """ # noinspection PyTypeChecker if isinstance(string, six.string_types): try: return string.encode(encoding) except UnicodeEncodeError: if u'Bretonnière' in string: - string = string.replace(u'Bretonnière', 'Bretonniere') - return Utils.convert_to_ASCII_if_possible(string, encoding) + string = string.replace(u'Bretonnière', 'Bretonnière') + return Utils.convert_to_ascii_if_possible(string, encoding) return string @staticmethod @@ -243,22 +316,30 @@ class Utils(object): if hasattr(var, 'coordinates') and " {0} ".format(old_name) in var.coordinates: new_coordinates = var.coordinates.replace(" {0} ".format(old_name), " {0} ".format(new_name)) - var.coordinates = Utils.convert_to_ASCII_if_possible(new_coordinates) + var.coordinates = Utils.convert_to_ascii_if_possible(new_coordinates) elif must_exist: raise Exception("Variable {0} does not exist in file {1}".format(old_name, filepath)) handler.sync() @staticmethod - def copy_file(source, destiny, save_hash=False, use_stored_hash=True): + def copy_file(source, destiny, save_hash=False, use_stored_hash=True, retrials=3): """ - Copies a file from source to destiny, creating dirs if necessary + Copy a file and compute a hash to check if the copy is equal to the source - :param save_hash: if True, stores hash value in a file - :type save_hash: bool - :param source: path to source - :type source: str - :param destiny: path to destiny - :type destiny: str + Parameters + ---------- + source: str + destiny: str + save_hash: bool, optional + If True, stores a copy of the hash + use_stored_hash: bool, optional + If True, try to use the stored value of the source hash instead of computing it + retrials: int, optional + Minimum value is 1 + + See Also + -------- + move_file """ dirname_path = os.path.dirname(destiny) if dirname_path and not os.path.exists(dirname_path): @@ -273,7 +354,9 @@ class Utils(object): Log.debug('Hashing original file... {0}', datetime.datetime.now()) hash_original = Utils.get_file_hash(source, use_stored=use_stored_hash) - retrials = 3 + if retrials < 1: + retrials = 1 + while hash_original != hash_destiny: if retrials == 0: raise Exception('Can not copy {0} to {1}'.format(source, destiny)) @@ -285,33 +368,56 @@ class Utils(object): Log.debug('Finished {0}', datetime.datetime.now()) @staticmethod - def move_file(source, destiny, save_hash=False): + def move_file(source, destiny, save_hash=False, retrials=3): """ - Moves a file from source to destiny, creating dirs if necessary + Moves a file and compute a hash to check if the copy is equal to the source + + It is just a call to Utils.copy_file followed bu + + Parameters + ---------- + source: str + destiny: str + save_hash: bool, optional + If True, stores a copy of the hash + retrials: int, optional + Minimum value is 1 + + See Also + -------- + copy_file - :param source: path to source - :type source: str - :param destiny: path to destiny - :type destiny: str - :param save_hash: if True, stores hash value in a file - :type save_hash: bool """ - Utils.copy_file(source, destiny, save_hash) + Utils.copy_file(source, destiny, save_hash, retrials) os.remove(source) @staticmethod def remove_file(path): """ - Removes a file, checking before if its exists + Delete a file only if it previously exists + + Parameters + ---------- + path: str - :param path: path to file - :type path: str """ if os.path.isfile(path): os.remove(path) @staticmethod def copy_tree(source, destiny): + """ + Copies a full tree to a new location + + Parameters + ---------- + source: str + destiny: str + + See Also + -------- + move_tree + """ if not os.path.exists(destiny): os.makedirs(destiny) shutil.copystat(source, destiny) @@ -326,22 +432,36 @@ class Utils(object): @staticmethod def move_tree(source, destiny): + """ + Moves a tree to a new location + + Parameters + ---------- + source: str + destiny: str + + See Also + ------- + copy_tree + """ Utils.copy_tree(source, destiny) shutil.rmtree(source) @staticmethod def get_file_hash(filepath, use_stored=False, save=False): """ - Returns the xxHash hash for the given filepath - :param filepath: path to the file to compute hash on - :type filepath:str - :param use_stored: if True, try to read the hash value from file - :type use_stored: bool - :param save: if True, stores hash value in a file - :type save: bool - :return: file's xxHash hash - :rtype: str + Get the xxHash hash for a given file + + + Parameters + ---------- + filepath: str + use_stored: bool, optional + If True, tries to use the stored hash before computing it + save: bool, optional + If True, saves the hash to a file """ + if use_stored: hash_file = Utils._get_hash_filename(filepath) if os.path.isfile(hash_file): @@ -373,14 +493,25 @@ class Utils(object): @staticmethod def execute_shell_command(command, log_level=Log.DEBUG): """ - Executes a sheel commandsi - :param command: command to execute + Execute shell command + + Writes the output to the log with the specified level + + Parameters + ---------- + command: str or iterable of str + log_level: int, optional + + Returns + ------- + iterable of str + Standard output of the command + + Raises + ------ + Utils.ExecutionError + If the command return value is non zero - Log.info('Detailed time for diagnostic class') - :param log_level: log level to use for command output - :type log_level: int - :return: command output - :rtype: list """ # noinspection PyTypeChecker if isinstance(command, six.string_types): @@ -403,9 +534,7 @@ class Utils(object): @staticmethod def available_cpu_count(): - """ - Number of available virtual or physical CPUs on this systemx - """ + """Number of available virtual or physical CPUs on this system""" if Utils._cpu_count is None: try: m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', @@ -426,12 +555,14 @@ class Utils(object): @staticmethod def convert2netcdf4(filetoconvert): """ - Checks if a file is in netCDF4 format and converts to netCDF4 if not + Convert a file to NetCDF4 - :param filetoconvert: file to convert - :type filetoconvert: str - """ + Conversion only performed if required. Deflation level set to 4 and shuffle activated. + Parameters + ---------- + filetoconvert: str + """ if Utils._is_compressed_netcdf4(filetoconvert): return @@ -465,28 +596,32 @@ class Utils(object): @staticmethod def openCdf(filepath, mode='a'): """ - Opens a netCDF file and returns a handler to it + Open a NetCDF file - :param filepath: path to the file - :type filepath: str - :param mode: mode to open the file. By default, a (append) - :type mode: str - :return: handler to the file - :rtype: netCDF4.Dataset + Parameters + ---------- + filepath: str + mode: str, optional + + Returns + ------- + netCDF4.Dataset """ return netCDF4.Dataset(filepath, mode) @staticmethod def get_datetime_from_netcdf(handler, time_variable='time'): """ - Gets a datetime array from a netCDF file + Get time from NetCDF files + + Parameters + ---------- + handler: netCDF4.Dataset + time_variable: str, optional - :param handler: file to read - :type handler: netCDF4.Dataset - :param time_variable: variable to read, by default 'time' - :type time_variable: str - :return: Datetime numpy array created from the values stored at the netCDF file - :rtype: np.array + Returns + ------- + numpy.array of Datetime """ var_time = handler.variables[time_variable] nctime = var_time[:] # get values @@ -546,7 +681,7 @@ class Utils(object): Utils.copy_attributes(new_var, original_var) if hasattr(new_var, 'coordinates'): coords = [new_names[coord] if coord in new_names else coord for coord in new_var.coordinates.split(' ')] - new_var.coordinates = Utils.convert_to_ASCII_if_possible(' '.join(coords)) + new_var.coordinates = Utils.convert_to_ascii_if_possible(' '.join(coords)) new_var[:] = original_var[:] @@ -554,7 +689,7 @@ class Utils(object): def copy_attributes(new_var, original_var, omitted_attributtes=None): if omitted_attributtes is None: omitted_attributtes = [] - new_var.setncatts({k: Utils.convert_to_ASCII_if_possible(original_var.getncattr(k)) + new_var.setncatts({k: Utils.convert_to_ascii_if_possible(original_var.getncattr(k)) for k in original_var.ncattrs() if k not in omitted_attributtes}) @staticmethod -- GitLab From 91e3dbd13352ed5fa114f6e86986c91ee039a556 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 7 Dec 2017 18:11:33 +0100 Subject: [PATCH 067/168] Fix some codacy issues --- earthdiagnostics/utils.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 683d9659..deaa3977 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -29,9 +29,7 @@ from earthdiagnostics.constants import Basins @contextmanager def suppress_stdout(): - """ - Redirects the standard output to devnull - """ + """Redirects the standard output to devnull""" with open(os.devnull, "w") as devnull: old_stdout = sys.stdout sys.stdout = devnull @@ -42,9 +40,7 @@ def suppress_stdout(): class Utils(object): - """ - Container class for miscellaneous utility methods - """ + """Container class for miscellaneous utility methods""" nco = Nco() """An instance of Nco class ready to be used""" @@ -274,7 +270,7 @@ class Utils(object): # noinspection PyPep8Naming @staticmethod def convert_to_ascii_if_possible(string, encoding='ascii'): - """ + u""" Convert an Unicode string to ASCII if all characters can be translated. If a string can not be translated it is unchanged. It also automatically @@ -370,7 +366,7 @@ class Utils(object): @staticmethod def move_file(source, destiny, save_hash=False, retrials=3): """ - Moves a file and compute a hash to check if the copy is equal to the source + Move a file and compute a hash to check if the copy is equal to the source It is just a call to Utils.copy_file followed bu @@ -433,7 +429,7 @@ class Utils(object): @staticmethod def move_tree(source, destiny): """ - Moves a tree to a new location + Move a tree to a new location Parameters ---------- -- GitLab From 0c5db1619e999f36dd9acd0211fde365919f2537 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 11 Dec 2017 12:58:04 +0100 Subject: [PATCH 068/168] Improved doc --- earthdiagnostics/diagnostic.py | 301 ++++++++++++++++++++++++++------- earthdiagnostics/utils.py | 63 +++---- 2 files changed, 275 insertions(+), 89 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 35fe4fa3..9ae7038b 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -12,6 +12,7 @@ from earthdiagnostics.variable_type import VariableType class DiagnosticStatus(object): + """Enumeration of diagnostic status""" WAITING = 0 READY = 1 RUNNING = 2 @@ -21,20 +22,27 @@ class DiagnosticStatus(object): class Diagnostic(Publisher): """ - Base class for the diagnostics. Provides a common interface for them and also - has a mechanism that allows diagnostic retrieval by name. + Base class for the diagnostics. + + Provides a common interface for them and also has a mechanism that allows diagnostic retrieval by name. :param data_manager: data manager that will be used to store and retrieve the necessary data :type data_manager: DataManager """ alias = None - """ - Alias to call the diagnostic. Must be overridden at the derived clases - """ + """ Alias to call the diagnostic. Must be overridden at the derived clases""" _diag_list = dict() def __init__(self, data_manager): + """ + Diagnostic's constructor + + + Parameters + ---------- + data_manager: DataManager + """ super(Diagnostic, self).__init__() self._generated_files = [] self.data_manager = data_manager @@ -44,21 +52,49 @@ class Diagnostic(Publisher): self.subjobs = [] def __ne__(self, other): + """ + Check if a diagnostic is different than other + + Implementation is just the negation of the equal, that should be implemented by the derived classes + + Parameters + ---------- + other: Diagnostic or NoneType + Diagnostic to be compared + + Returns + ------- + bool + + """ return not self == other def can_skip_run(self): + """ + Check if a diagnostic calculation can be skipped + + Looks if the data to be generated is already there and is not going to be modified + + Returns + ------- + bool + + """ for file_generated in self._generated_files: if file_generated.storage_status != StorageStatus.READY: return False if file_generated.has_modifiers(): Log.warning('Can not skip diagnostics run when data is going to be modified: {0}'.format(self)) return False + return True def __repr__(self): + """ Diagnostic's string represenatation. Defaults to str""" return str(self) @property def status(self): + """ Diagnostic's current status""" return self._status @status.setter @@ -76,9 +112,13 @@ class Diagnostic(Publisher): @staticmethod def register(cls): """ - Register a new diagnostic using the given alias. It must be call using the derived class. - :param cls: diagnostic class to register - :type cls: Type[Diagnostic] + Register a new diagnostic using the given alias. + + It must be called using the derived class. + + Parameters + ---------- + cls: Type[Diagnostic] """ if not issubclass(cls, Diagnostic): raise ValueError('Class {0} must be derived from Diagnostic'.format(cls)) @@ -92,10 +132,14 @@ class Diagnostic(Publisher): """ Return the class for a diagnostic given its name - :param name: diagnostic alias - :type name: str - :return: the selected Diagnostic class, None if name can not be found - :rtype: Diagnostic + Parameters + ---------- + name: str + + Returns + ------- + Type[Diagnostic] or None + """ if name in Diagnostic._diag_list.keys(): return Diagnostic._diag_list[name] @@ -128,22 +172,24 @@ class Diagnostic(Publisher): def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, vartype=VariableType.MEAN): """ - - :param domain: - :type domain: ModelingRealm - :param var: - :param startdate: - :param member: - :param chunk: - :param grid: - :param region: - :param box: - :param frequency: - :type frequency: Frequency - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: datafile object - :rtype: earthdiagnostics.datafile.DataFile + Declare a chunk that is going to be generated by the diagnostic + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str + region: Basin + box: Box + frequency: Frequency + vartype: VariableType + + Returns + ------- + DataFile """ if isinstance(region, Basin): region = region.name @@ -157,19 +203,23 @@ class Diagnostic(Publisher): def declare_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN): """ + Declare a year that is going to be generated by the diagnostic + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: 1 + year: 1 + grid: str + box: Box + vartype: VariableType + + Returns + ------- + DataFile - :param domain: - :type domain: ModelingRealm - :param var: - :param startdate: - :param member: - :param grid: - :param box: - :param year: - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: datafile object - :rtype: DataFile """ generated_year = self.data_manager.declare_year(domain, var, startdate, member, year, grid, box, diagnostic=self, vartype=vartype) @@ -183,6 +233,21 @@ class Diagnostic(Publisher): Must be implemented by derived classes. + Parameters + ---------- + diags: Diags + options: list of str + + Returns + ------- + list of Diagnostic + + """ + """ + Generate the instances of the diagnostics that will be run by the manager + + Must be implemented by derived classes. + :param diags: diagnostics manager :type diags: Diags :param options: list of strings containing the options passed to the diagnostic @@ -193,6 +258,24 @@ class Diagnostic(Publisher): @classmethod def process_options(cls, options, options_available): + """ + Process the configuration of a diagnostic + + Parameters + ---------- + options: iterable of str + options_available: iterable of DiagnosticOptiion + + Returns + ------- + dict of str: str + Dictionary of names and values for the options + + Raises + ------ + DiagnosticOptionError: + If there are more options that admitted for the diagnostic + """ processed = dict() options = options[1:] if len(options) > len(options_available): @@ -208,17 +291,21 @@ class Diagnostic(Publisher): def __str__(self): """ + String represenation of the diagnostic + Must be implemented by derived classes - :return: """ return 'Developer must override base class __str__ method' def add_subjob(self, subjob): """ - Adds a subjob - :param subjob: - :type subjob: Diagnostic - :return: + Add a subjob + + Add a diagnostic that must be run before the current one + + Parameters + ---------- + subjob: Diagnostic """ self.subjobs.append(subjob) subjob.subscribe(self, self._subjob_status_changed) @@ -228,6 +315,34 @@ class Diagnostic(Publisher): def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, to_modify=False, vartype=VariableType.MEAN): + """ + Request one chunk of data required by the diagnostic + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str + box: Box + frequency: Frequency + to_modify: bool + Flag that must be active if the diagnostic is going to generate a modified version of this data. In this + case this data must not be declared as an output of the diagnostic + vartype: VariableType + + Returns + ------- + DataFile + + See Also + -------- + request_year + declare_chunk + declare_year + """ request = self.data_manager.request_chunk(domain, var, startdate, member, chunk, grid, box, frequency, vartype) if to_modify: request.add_modifier(self) @@ -236,6 +351,32 @@ class Diagnostic(Publisher): return request def request_year(self, domain, var, startdate, member, year, grid=None, box=None, frequency=None, to_modify=False): + """ + Requests one year of data that is required for the diagnostic + + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + year: int + grid: str + box: Box + frequency: Frequency + to_modify: str + + Returns + ------- + DataFile + + See Also + -------- + request_chunk + declare_chunk + declare_year + """ request = self.data_manager.request_year(self, domain, var, startdate, member, year, grid, box, frequency) if to_modify: request.add_modifier(self) @@ -255,6 +396,7 @@ class Diagnostic(Publisher): self.check_is_ready() def check_is_ready(self): + """ Check if a diagnostic is ready to run and change its status accordingly """ if all([request.ready_to_run(self) for request in self._requests]) and\ all([subjob.status == DiagnosticStatus.COMPLETED for subjob in self.subjobs]): self.status = DiagnosticStatus.READY @@ -264,24 +406,63 @@ class Diagnostic(Publisher): request.unsubscribe(self) def all_requests_in_storage(self): + """ + Check if all the data requested is in the local scratch + + Returns + ------- + bool + """ return self.pending_requests() == 0 def pending_requests(self): + """ + Get the number of data request pending to be fulfilled + + Returns + ------- + int + """ return len([request.storage_status != StorageStatus.READY or request.local_status != LocalStatus.READY for request in self._requests]) class DiagnosticOption(object): - + """Class to manage string options for the diagnostic""" def __init__(self, name, default_value=None): + """ + + Parameters + ---------- + name: str + default_value: object, optional + """ self.name = name self.default_value = default_value def parse(self, option_value): - option_value = self.check_default(option_value) + """ + Get the final value for the option + + If option_value is empty, return default_value + + Parameters + ---------- + option_value: str + + Returns + ------- + str + + Raises + ------ + DiagnosticOptionError: + If the option is empty and default_value is False + """ + option_value = self._check_default(option_value) return option_value - def check_default(self, option_value): + def _check_default(self, option_value): if option_value == '': if self.default_value is None: raise DiagnosticOptionError('Option {0} is not optional'.format(self.name)) @@ -292,7 +473,7 @@ class DiagnosticOption(object): class DiagnosticFloatOption(DiagnosticOption): def parse(self, option_value): - return float(self.check_default(option_value)) + return float(self._check_default(option_value)) class DiagnosticIntOption(DiagnosticOption): @@ -303,7 +484,7 @@ class DiagnosticIntOption(DiagnosticOption): self.max_limit = max_limit def parse(self, option_value): - value = int(self.check_default(option_value)) + value = int(self._check_default(option_value)) if self.min_limit is not None and value < self.min_limit: raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) if self.max_limit is not None and value > self.max_limit: @@ -331,7 +512,7 @@ class DiagnosticListIntOption(DiagnosticOption): """ Upper limit """ def parse(self, option_value): - option_value = self.check_default(option_value) + option_value = self._check_default(option_value) if isinstance(option_value, tuple) or isinstance(option_value, list): return option_value values = [int(i) for i in option_value.split('-')] @@ -352,7 +533,7 @@ class DiagnosticListFrequenciesOption(DiagnosticOption): super(DiagnosticListFrequenciesOption, self).__init__(name, default_value) def parse(self, option_value): - option_value = self.check_default(option_value) + option_value = self._check_default(option_value) if isinstance(option_value, tuple) or isinstance(option_value, list): return option_value values = [Frequency(i) for i in option_value.split('-')] @@ -365,7 +546,7 @@ class DiagnosticVariableOption(DiagnosticOption): self.var_manager = var_manager def parse(self, option_value): - option_value = self.check_default(option_value) + option_value = self._check_default(option_value) real_name = self.var_manager.get_variable(option_value, False) if real_name is None: return option_value @@ -379,7 +560,7 @@ class DiagnosticVariableListOption(DiagnosticOption): self.var_manager = var_manager def parse(self, option_value): - option_value = self.check_default(option_value) + option_value = self._check_default(option_value) var_names = [] for value in option_value.split(':'): real_name = self.var_manager.get_variable(value, False) @@ -395,7 +576,7 @@ class DiagnosticDomainOption(DiagnosticOption): super(DiagnosticDomainOption, self).__init__(name, default_value) def parse(self, option_value): - return ModelingRealms.parse(self.check_default(option_value)) + return ModelingRealms.parse(self._check_default(option_value)) class DiagnosticFrequencyOption(DiagnosticOption): @@ -403,12 +584,12 @@ class DiagnosticFrequencyOption(DiagnosticOption): super(DiagnosticFrequencyOption, self).__init__(name, default_value) def parse(self, option_value): - return Frequency.parse(self.check_default(option_value)) + return Frequency.parse(self._check_default(option_value)) class DiagnosticBasinOption(DiagnosticOption): def parse(self, option_value): - value = self.check_default(option_value) + value = self._check_default(option_value) basin = Basins().parse(value) if basin is None: raise DiagnosticOptionError('Basin {0} not recognized'.format(value)) @@ -417,12 +598,12 @@ class DiagnosticBasinOption(DiagnosticOption): class DiagnosticComplexStrOption(DiagnosticOption): def parse(self, option_value): - return self.check_default(option_value).replace('&;', ',').replace('&.', ' ') + return self._check_default(option_value).replace('&;', ',').replace('&.', ' ') class DiagnosticBoolOption(DiagnosticOption): def parse(self, option_value): - option_value = self.check_default(option_value) + option_value = self._check_default(option_value) if isinstance(option_value, bool): return option_value else: @@ -440,7 +621,7 @@ class DiagnosticChoiceOption(DiagnosticOption): self.parse(default_value) def parse(self, option_value): - option_value = self.check_default(option_value) + option_value = self._check_default(option_value) if self.ignore_case: option_value = option_value.lower() for choice in self.choices: diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index deaa3977..d1f74b19 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -632,22 +632,22 @@ class Utils(object): @staticmethod def copy_variable(source, destiny, variable, must_exist=True, add_dimensions=False, new_names=None): """ - Copies the given variable from source to destiny - - :param add_dimensions: if it's true, dimensions required by the variable will be automatically added to the - file. It will also add the dimension variable - :type add_dimensions: bool - :param source: origin file - :type source: netCDF4.Dataset - :param destiny: destiny file - :type destiny: netCDF4.Dataset - :param variable: variable to copy - :type variable: str - :param must_exist: if false, does not raise an error uf variable does not exist - :type must_exist: bool - :param new_names: dictionary containing variables to rename and new name as key-value pairs - :type new_names: dict - :return: + Copy the given variable from source to destiny + + Parameters + ---------- + source: netCDF4.Dataset + destiny: netCDF4.Dataset + variable: str + must_exist: bool, optional + add_dimensions: bool, optional + new_names: dict of str: str + + Raises + ------ + Exception + If dimensions are not correct in the destiny file and add_dimensions is False + """ if not must_exist and variable not in source.variables.keys(): return @@ -683,6 +683,16 @@ class Utils(object): @staticmethod def copy_attributes(new_var, original_var, omitted_attributtes=None): + """ + Copy attributtes from one variable to another + + Parameters + ---------- + new_var: netCDF4.Variable + original_var: netCDF4.Variable + omitted_attributtes: iterable of str + Collection of attributtes that should not be copied + """ if omitted_attributtes is None: omitted_attributtes = [] new_var.setncatts({k: Utils.convert_to_ascii_if_possible(original_var.getncattr(k)) @@ -691,20 +701,15 @@ class Utils(object): @staticmethod def copy_dimension(source, destiny, dimension, must_exist=True, new_names=None): """ - Copies the given dimension from source to destiny, including dimension variables if present + Copy the given dimension from source to destiny, including dimension variables if present - :param new_names: dictionary containing variables to rename and new name as key-value pairs - :type new_names: dict - :param source: origin file - :type source: netCDF4.Dataset - :param destiny: destiny file - :type destiny: netCDF4.Dataset - :param dimension: variable to copy - :type dimension: str - :param must_exist: if false, does not raise an error uf variable does not exist - :type must_exist: bool - - :return: + Parameters + ---------- + source: netCDF4.Dataset + destiny: netCDF4.Dataset + dimension: str + must_exist: bool, optional + new_names: dict of str: str or NoneType, optional """ if not must_exist and dimension not in source.dimensions.keys(): return -- GitLab From 1132d8ce0d58c3d6f9078c46486110b058246b90 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 11 Dec 2017 16:50:34 +0100 Subject: [PATCH 069/168] Improved doc --- earthdiagnostics/cmorizer.py | 10 +-- earthdiagnostics/cmormanager.py | 4 +- earthdiagnostics/datafile.py | 12 +-- earthdiagnostics/diagnostic.py | 15 +--- earthdiagnostics/earthdiags.py | 2 +- earthdiagnostics/general/attribute.py | 2 +- earthdiagnostics/general/dailymean.py | 5 +- earthdiagnostics/general/module.py | 4 +- earthdiagnostics/general/monthlymean.py | 5 +- earthdiagnostics/general/scale.py | 2 +- .../general/simplify_dimensions.py | 4 +- earthdiagnostics/general/yearlymean.py | 4 +- earthdiagnostics/ocean/areamoc.py | 8 +- earthdiagnostics/ocean/convectionsites.py | 9 +- earthdiagnostics/ocean/cutsection.py | 12 +-- earthdiagnostics/ocean/gyres.py | 6 +- earthdiagnostics/ocean/heatcontent.py | 10 +-- earthdiagnostics/ocean/heatcontentlayer.py | 10 +-- earthdiagnostics/ocean/interpolate.py | 4 +- earthdiagnostics/ocean/interpolatecdo.py | 8 +- earthdiagnostics/ocean/mask_land.py | 4 +- earthdiagnostics/ocean/maxmoc.py | 4 +- earthdiagnostics/ocean/moc.py | 2 +- earthdiagnostics/ocean/mxl.py | 4 +- earthdiagnostics/ocean/regionmean.py | 4 +- earthdiagnostics/ocean/regionsum.py | 4 +- earthdiagnostics/ocean/rotation.py | 13 +-- earthdiagnostics/ocean/siasiesiv.py | 8 +- earthdiagnostics/ocean/verticalgradient.py | 4 +- earthdiagnostics/ocean/verticalmean.py | 2 +- earthdiagnostics/ocean/verticalmeanmeters.py | 2 +- .../statistics/daysoverpercentile.py | 6 +- earthdiagnostics/statistics/discretize.py | 2 +- .../statistics/monthlypercentile.py | 11 +-- earthdiagnostics/utils.py | 82 +++++++++---------- 35 files changed, 139 insertions(+), 149 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 7ec4d346..b66f705a 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -133,7 +133,7 @@ class Cmorizer(object): for cmor_var in ("hfss", 'hfls')] total_seconds = (self.experiment.atmos_timestep * 3600) for filename in glob.glob(os.path.join(self.cmor_scratch, '*.nc')): - handler = Utils.openCdf(filename) + handler = Utils.open_cdf(filename) for varname in handler.variables.keys(): cmor_var = self.data_manager.variable_list.get_variable(varname, True) @@ -368,7 +368,7 @@ class Cmorizer(object): Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) Utils.rename_variables(filename, self.alt_coord_names, False, True) - handler = Utils.openCdf(filename) + handler = Utils.open_cdf(filename) self._remove_valid_limits(handler) self._add_common_attributes(handler, frequency) self._update_time_variables(handler) @@ -387,7 +387,7 @@ class Cmorizer(object): os.remove(filename) def _remove_valid_limits(self, filename): - handler = Utils.openCdf(filename) + handler = Utils.open_cdf(filename) for variable in handler.variables.keys(): var = handler.variables[variable] if 'valid_min' in var.ncattrs(): @@ -505,7 +505,7 @@ class Cmorizer(object): return chunk def _add_coordinate_variables(self, handler, temp): - handler_cmor = Utils.openCdf(temp) + handler_cmor = Utils.open_cdf(temp) Utils.copy_variable(handler, handler_cmor, self.lon_name, False) Utils.copy_variable(handler, handler_cmor, self.lat_name, False) if 'time' in handler_cmor.dimensions.keys(): @@ -594,7 +594,7 @@ class Cmorizer(object): gribfile, var_code, frequency)) h_var_file = '{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency) - handler = Utils.openCdf(h_var_file) + handler = Utils.open_cdf(h_var_file) if new_units: for var in handler.variables.values(): if 'code' in var.ncattrs() and var.code == var_code: diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 1c4b5d9b..4b75224c 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -594,7 +594,7 @@ class CMORManager(DataManager): for name in os.listdir(os.path.join(path, member, table, var, grid)): filepath = os.path.join(path, member, table, var, grid, name) if os.path.isfile(filepath): - original_handler = Utils.openCdf(filepath) + original_handler = Utils.open_cdf(filepath) if original_handler.dimensions['i'].size < original_handler.dimensions['j'].size: original_handler.close() Utils.rename_variables(filepath, {'i': 'j', 'j': 'i'}, False, True) @@ -676,7 +676,7 @@ class MergeYear(Diagnostic): def _select_data_of_given_year(self, data_file): temp2 = TempFile.get() - handler = Utils.openCdf(data_file) + handler = Utils.open_cdf(data_file) times = Utils.get_datetime_from_netcdf(handler) x = 0 first_index = None diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 8c20aa30..e308ad3e 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -210,7 +210,7 @@ class DataFile(Publisher): pass def _correct_metadata(self): - handler = Utils.openCdf(self.local_file) + handler = Utils.open_cdf(self.local_file) var_handler = handler.variables[self.final_name] coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name}, set(handler.variables.keys())) var_handler.coordinates = ' '.join(coords) @@ -301,7 +301,7 @@ class DataFile(Publisher): self._update_var_with_region_data() self._correct_metadata() Utils.nco.ncks(input=self.local_file, output=self.local_file, options=['--fix_rec_dmn region']) - handler = Utils.openCdf(self.local_file) + handler = Utils.open_cdf(self.local_file) regions = handler.variables['region'][...].tolist() if len(regions) > 1: ordered_regions = sorted(regions) @@ -322,14 +322,14 @@ class DataFile(Publisher): def _update_var_with_region_data(self): temp = TempFile.get() shutil.copyfile(self.remote_file, temp) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) var_handler = handler.variables[self.final_name] var_type = var_handler.dtype handler.close() self._fix_values_metadata(var_type, temp) Utils.nco.ncks(input=temp, output=temp, options=['--mk_rec_dmn region']) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) var_handler = handler.variables[self.final_name] if hasattr(var_handler, 'valid_min'): del var_handler.valid_min @@ -356,7 +356,7 @@ class DataFile(Publisher): Utils.move_file(temp, self.local_file) def _add_region_dimension_to_var(self): - handler = Utils.openCdf(self.local_file) + handler = Utils.open_cdf(self.local_file) handler.createDimension('region') var_region = handler.createVariable('region', str, 'region') var_region[0] = self.region @@ -399,7 +399,7 @@ class DataFile(Publisher): utc_datetime = 'UTC ' + datetime.utcnow().isoformat() history_line = '{0}: {1};'.format(utc_datetime, history_line) - handler = Utils.openCdf(self.local_file) + handler = Utils.open_cdf(self.local_file) try: history_line = history_line + handler.history except AttributeError: diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 9ae7038b..fb254e96 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -50,6 +50,7 @@ class Diagnostic(Publisher): self._requests = [] self.consumed_time = datetime.timedelta() self.subjobs = [] + self.message = None def __ne__(self, other): """ @@ -242,17 +243,6 @@ class Diagnostic(Publisher): ------- list of Diagnostic - """ - """ - Generate the instances of the diagnostics that will be run by the manager - - Must be implemented by derived classes. - - :param diags: diagnostics manager - :type diags: Diags - :param options: list of strings containing the options passed to the diagnostic - :type options: list[str] - :return: """ raise NotImplementedError("Class must override generate_jobs class method") @@ -472,12 +462,13 @@ class DiagnosticOption(object): class DiagnosticFloatOption(DiagnosticOption): + """ Class for parsing float options """ def parse(self, option_value): return float(self._check_default(option_value)) class DiagnosticIntOption(DiagnosticOption): - + """ Class for parsing integer options """ def __init__(self, name, default_value=None, min_limit=None, max_limit=None): super(DiagnosticIntOption, self).__init__(name, default_value) self.min_limit = min_limit diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index b8aa3ef9..db732f05 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -174,7 +174,7 @@ class EarthDiags(object): def _read_basins_from_file(filename): if not os.path.isfile(filename): return - handler = Utils.openCdf(filename) + handler = Utils.open_cdf(filename) Basins().get_available_basins(handler) handler.close() diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 4259d0a3..1d460ac2 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -90,7 +90,7 @@ class Attribute(Diagnostic): Runs the diagnostic """ variable_file = self.variable_file.local_file - handler = Utils.openCdf(variable_file) + handler = Utils.open_cdf(variable_file) handler.setncattr(self.attributte_name, self.attributte_value) handler.close() if not Utils.check_netcdf_file(variable_file): diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py index 1411b0ed..d2737b5c 100644 --- a/earthdiagnostics/general/dailymean.py +++ b/earthdiagnostics/general/dailymean.py @@ -1,6 +1,7 @@ # coding=utf-8 import os + from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies @@ -93,13 +94,13 @@ class DailyMean(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - handler = Utils.openCdf(self.variable_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) if 'region' in handler.variables: noregion = TempFile.get() Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) Utils.cdo.daymean(input=noregion, output=temp) os.remove(noregion) - monmean_handler = Utils.openCdf(temp) + monmean_handler = Utils.open_cdf(temp) Utils.copy_variable(handler, monmean_handler, 'region') monmean_handler.close() else: diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index e6081cd8..17b01d67 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -94,8 +94,8 @@ class Module(Diagnostic): """ temp = TempFile.get() Utils.copy_file(self.component_u_file.local_file, temp) - component_u = Utils.openCdf(temp) - component_v = Utils.openCdf(self.component_v_file.local_file) + component_u = Utils.open_cdf(temp) + component_v = Utils.open_cdf(self.component_v_file.local_file) variable_u = component_u.variables[self.componentu] variable_v = component_v.variables[self.componentv] diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 6062d075..55caf8c8 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -1,6 +1,7 @@ # coding=utf-8 import os + from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies @@ -91,14 +92,14 @@ class MonthlyMean(Diagnostic): """ Runs the diagnostic """ - handler = Utils.openCdf(self.variable_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) temp = TempFile.get() if 'region' in handler.variables: noregion = TempFile.get() Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) Utils.cdo.monmean(input=noregion, output=temp) os.remove(noregion) - monmean_handler = Utils.openCdf(temp) + monmean_handler = Utils.open_cdf(temp) Utils.copy_variable(handler, monmean_handler, 'region') monmean_handler.close() else: diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 0e650486..1d1d918a 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -108,7 +108,7 @@ class Scale(Diagnostic): """ variable_file = self.variable_file.local_file - handler = Utils.openCdf(variable_file) + handler = Utils.open_cdf(variable_file) var = handler.variables[self.variable] self.original_values = var[:] if self.apply_mask: diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index c9c39e21..4c83e376 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -93,7 +93,7 @@ class SimplifyDimensions(Diagnostic): """ Runs the diagnostic """ - handler = Utils.openCdf(self.variable_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) if 'i' not in handler.dimensions: raise Exception('Variable {0.domain}:{0.variable} does not have i,j dimensions'.format(self)) lat = handler.variables[self.lat_name] @@ -111,7 +111,7 @@ class SimplifyDimensions(Diagnostic): '{0.domain}:{0.variable}'.format(self)) temp = TempFile.get() - new_file = Utils.openCdf(temp, 'w') + new_file = Utils.open_cdf(temp, 'w') for dim in handler.dimensions.keys(): if dim in (self.lon_name, self.lat_name, 'i', 'j', 'vertices'): continue diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py index c10221d3..2f3c96d4 100644 --- a/earthdiagnostics/general/yearlymean.py +++ b/earthdiagnostics/general/yearlymean.py @@ -95,12 +95,12 @@ class YearlyMean(Diagnostic): """ temp = TempFile.get() - handler = Utils.openCdf(self.variable_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) if 'region' in handler.variables: noregion = TempFile.get() Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) Utils.cdo.yearmean(input=noregion, output=temp) - monmean_handler = Utils.openCdf(temp) + monmean_handler = Utils.open_cdf(temp) Utils.copy_variable(handler, monmean_handler, 'region') monmean_handler.close() else: diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index c73895fd..c96def76 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -110,11 +110,11 @@ class AreaMoc(Diagnostic): Utils.copy_file(self.variable_file.local_file, temp) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) if 'i' in handler.dimensions: handler.close() nco.ncwa(input=temp, output=temp, options=('-O -a i',)) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) basin_index = np.where(handler.variables['basin'][:] == self.basin.name) if 'lat' in handler.variables: @@ -135,8 +135,8 @@ class AreaMoc(Diagnostic): # To select basin and remove dimension nco.ncwa(input=temp, output=temp, options=('-O -d basin,{0} -a basin'.format(basin_index),)) - source = Utils.openCdf(temp) - destiny = Utils.openCdf(temp2, 'w') + source = Utils.open_cdf(temp) + destiny = Utils.open_cdf(temp2, 'w') Utils.copy_dimension(source, destiny, 'time') Utils.copy_dimension(source, destiny, 'lev') diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index 74271682..9d9eff99 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -1,10 +1,11 @@ # coding=utf-8 import numpy as np from bscearth.utils.log import Log -from earthdiagnostics.diagnostic import Diagnostic -from earthdiagnostics.utils import Utils, TempFile + from earthdiagnostics.constants import Models +from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class ConvectionSites(Diagnostic): @@ -93,8 +94,8 @@ class ConvectionSites(Diagnostic): mlotst_file = self.mixed_layer.local_file output = TempFile.get() - self.mlotst_handler = Utils.openCdf(mlotst_file) - handler = Utils.openCdf(output, 'w') + self.mlotst_handler = Utils.open_cdf(mlotst_file) + handler = Utils.open_cdf(output, 'w') handler.createDimension('time', self.mlotst_handler.variables['time'].shape[0]) handler.createDimension('region', 4) Utils.copy_variable(self.mlotst_handler, handler, 'time') diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index d4e5f89c..d5e03355 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -2,11 +2,11 @@ import numpy as np from bscearth.utils.log import Log +from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBoolOption, DiagnosticIntOption, \ DiagnosticDomainOption, DiagnosticVariableOption -from earthdiagnostics.box import Box -from earthdiagnostics.utils import Utils from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils class CutSection(Diagnostic): @@ -104,7 +104,7 @@ class CutSection(Diagnostic): """ nco = Utils.nco - handler = Utils.openCdf('mesh_hgr.nc') + handler = Utils.open_cdf('mesh_hgr.nc') dimi = handler.dimensions['i'].size dimj = handler.dimensions['j'].size dimlev = handler.dimensions['lev'].size @@ -113,7 +113,7 @@ class CutSection(Diagnostic): lat = handler.variables['lat'][:] handler.close() - handler = Utils.openCdf('mask.nc') + handler = Utils.open_cdf('mask.nc') mask_lev = handler.variables['tmask'][:] mask_lev = mask_lev.astype(float) # noinspection PyTypeChecker @@ -152,7 +152,7 @@ class CutSection(Diagnostic): temp = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) dimtime = handler.dimensions['time'].size var_array = handler.variables[self.variable][:] handler.close() @@ -171,7 +171,7 @@ class CutSection(Diagnostic): nco.ncks(input=temp, output=temp, options=('-O -v lev,time',)) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) if not self.zonal: handler.createDimension('lat', size) coord_var = handler.createVariable('lat', float, 'lat') diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index b0197148..abca9e55 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -4,8 +4,8 @@ from bscearth.utils.log import Log from earthdiagnostics.constants import Models from earthdiagnostics.diagnostic import Diagnostic -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class Gyres(Diagnostic): @@ -101,9 +101,9 @@ class Gyres(Diagnostic): output = TempFile.get() vsftbarot_file = self.vsftbarot.local_file - handler_original = Utils.openCdf(vsftbarot_file) + handler_original = Utils.open_cdf(vsftbarot_file) self.var_vsftbarot = handler_original.variables['vsftbarot'] - handler = Utils.openCdf(output, 'w') + handler = Utils.open_cdf(output, 'w') handler.createDimension('time', handler_original.variables['time'].shape[0]) handler.createDimension('region', 8) Utils.copy_variable(handler_original, handler, 'time') diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index a5848325..ca572a3f 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -133,7 +133,7 @@ class HeatContent(Diagnostic): @classmethod def _read_level_values(cls): - handler = Utils.openCdf('mesh_zgr.nc') + handler = Utils.open_cdf('mesh_zgr.nc') if 'gdepw_1d' in handler.variables: depth_w = handler.variables['gdepw_1d'][0, :] elif 'gdepw_0' in handler.variables: @@ -189,7 +189,7 @@ class HeatContent(Diagnostic): para.append('-mxloption') para.append(str(self.mxloption)) if self.basin != Basins().Global: - handler = Utils.openCdf('mask_regions.3d.nc') + handler = Utils.open_cdf('mask_regions.3d.nc') if self.basin.name not in handler.variables: raise Exception('Basin {0} is not defined on mask_regions.nc'.format(self.basin.name)) @@ -202,13 +202,13 @@ class HeatContent(Diagnostic): cdftools.run('cdfheatc', options=para, input_file=temperature_file, output_file=temp2, input_option='-f') - results = Utils.openCdf(temp2) + results = Utils.open_cdf(temp2) heatcsum_temp = TempFile.get() heatcvmean_temp = TempFile.get() nco.ncks(input=temperature_file, output=heatcsum_temp, options=('-O -v time',)) shutil.copy(heatcsum_temp, heatcvmean_temp) - heatcsum_handler = Utils.openCdf(heatcsum_temp) + heatcsum_handler = Utils.open_cdf(heatcsum_temp) thc = heatcsum_handler.createVariable('heatcsum', float, 'time') thc.standard_name = "integral_of_sea_water_potential_temperature_expressed_as_heat_content" thc.long_name = "Total heat content" @@ -216,7 +216,7 @@ class HeatContent(Diagnostic): thc[:] = results.variables['heatc3d'][:, 0, 0] heatcsum_handler.close() - heatcvmean_handler = Utils.openCdf(heatcvmean_temp) + heatcvmean_handler = Utils.open_cdf(heatcvmean_temp) uhc = heatcvmean_handler.createVariable('heatcvmean', float, 'time') uhc.standard_name = "integral_of_sea_water_potential_temperature_expressed_as_heat_content" uhc.long_name = "Heat content per unit volume" diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index c64b1902..4662bd13 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -1,11 +1,11 @@ # coding=utf-8 import numpy as np -from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box +from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class HeatContentLayer(Diagnostic): @@ -71,7 +71,7 @@ class HeatContentLayer(Diagnostic): box.max_depth = options['max_depth'] job_list = list() - handler = Utils.openCdf('mesh_zgr.nc') + handler = Utils.open_cdf('mesh_zgr.nc') # mask = Utils.get_mask(options['basin']) mask = handler.variables['tmask'][:] @@ -152,7 +152,7 @@ class HeatContentLayer(Diagnostic): Utils.copy_file(self.thetao.local_file, thetao_file) - handler = Utils.openCdf(thetao_file) + handler = Utils.open_cdf(thetao_file) Utils.convert_units(handler.variables['thetao'], 'K') heatc_sl = np.sum(handler.variables['thetao'][:, self.min_level:self.max_level, :] * self.weight, 1) handler.sync() @@ -161,7 +161,7 @@ class HeatContentLayer(Diagnostic): nco.ncks(input=thetao_file, output=results, options=('-O -v lon,lat,time',)) Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False, True) - handler_results = Utils.openCdf(results) + handler_results = Utils.open_cdf(results) handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) handler_results.sync() handler_results.variables['heatc'][:] = heatc_sl diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index c4ac2e6d..d9b6275f 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -116,7 +116,7 @@ class Interpolate(Diagnostic): Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False, rename_dimension=True) cdo = Utils.cdo nco = Utils.nco - handler = Utils.openCdf(variable_file) + handler = Utils.open_cdf(variable_file) if 'lev' in handler.dimensions: num_levels = handler.dimensions['lev'].size has_levels = True @@ -137,7 +137,7 @@ class Interpolate(Diagnostic): else: Utils.move_file(self._get_level_file(0), temp) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) if 'record' in handler.dimensions: handler.renameDimension('record', 'lev') handler.close() diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 17f2f2b5..1772d7de 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -135,7 +135,7 @@ class InterpolateCDO(Diagnostic): def get_sample_grid_file(cls): temp = TempFile.get() - handler = Utils.openCdf('mask.nc') + handler = Utils.open_cdf('mask.nc') lat_name, lon_name = cls._get_lat_lon_alias(handler) lon_bnds_name = '{0}_bnds'.format(lon_name) @@ -143,7 +143,7 @@ class InterpolateCDO(Diagnostic): Utils.nco.ncks(input='mask.nc', output=temp, options=('-O -v tmask,{0},{1},gphif,glamf'.format(lat_name, lon_name),)) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) lon = handler.variables[lon_name] lon.units = "degrees_east" lon.long_name = "Longitude" @@ -229,7 +229,7 @@ class InterpolateCDO(Diagnostic): 'SSTK_ens0': 'tos', 'SSTK_ens1': 'tos', 'SSTK_ens2': 'tos', 'nav_lat': 'lat', 'nav_lon': 'lon'}, must_exist=False, rename_dimension=True) - handler = Utils.openCdf(variable_file) + handler = Utils.open_cdf(variable_file) lat_name, lon_name = self._get_lat_lon_alias(handler) var = handler.variables[self.variable] units = var.units @@ -252,7 +252,7 @@ class InterpolateCDO(Diagnostic): temp = TempFile.get() Utils.cdo.remap(','.join((self.grid.split('_')[0], self.weights)), input=variable_file, output=temp) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) handler.variables[self.variable].units = units handler.close() diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index 1f6c7dd0..20b14369 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -79,7 +79,7 @@ class MaskLand(Diagnostic): @classmethod def _get_mask(cls, cell_point): - mask_file = Utils.openCdf('mask.nc') + mask_file = Utils.open_cdf('mask.nc') mask = mask_file.variables['{0}mask'.format(cell_point)][:].astype(float) mask[mask == 0] = np.nan mask_file.close() @@ -100,7 +100,7 @@ class MaskLand(Diagnostic): temp = TempFile.get() Utils.copy_file(self.var_file.local_file, temp) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) if 'lev' not in handler.dimensions: mask = self.mask[:, 0, ...] else: diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index fb86ecb5..bbbf1a22 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -122,13 +122,13 @@ class MaxMoc(Diagnostic): temp = TempFile.get() Utils.copy_file(self.variable_file.local_file, temp) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) if 'i' in handler.dimensions: handler.close() nco.ncwa(input=temp, output=temp, options=('-O -a i',)) else: handler.close() - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) basin_index = np.where(handler.variables['basin'][:] == self.basin.name) if len(basin_index) == 0: raise Exception("Basin {1} is not defined in {0}", temp, self.basin.name) diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index f66243f7..6171c975 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -84,7 +84,7 @@ class Moc(Diagnostic): Utils.convert2netcdf4(temp) Log.debug('Reformatting variables') - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) basins_list = [Basins().Global.name] if 'zomsfatl' in handler.variables: diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py index ce001cf2..ff6a85e2 100644 --- a/earthdiagnostics/ocean/mxl.py +++ b/earthdiagnostics/ocean/mxl.py @@ -68,8 +68,8 @@ class Mxl(Diagnostic): temp = TempFile.get() cdftools.run('cdfmxl', input_file=[self.thetao_file, self.so_file], output_file=temp, options='-nc4') temp2 = TempFile.get() - source = Utils.openCdf(temp) - destiny = Utils.openCdf(temp2, 'w') + source = Utils.open_cdf(temp) + destiny = Utils.open_cdf(temp2, 'w') Utils.copy_variable(source, destiny, 'somxl010', must_exist=True, add_dimensions=True) Utils.copy_variable(source, destiny, 'lat', must_exist=False) Utils.copy_variable(source, destiny, 'latitude', must_exist=False) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 397f1711..dbff3fc5 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -124,7 +124,7 @@ class RegionMean(Diagnostic): variable_file = self.variable_file.local_file - handler = Utils.openCdf(variable_file) + handler = Utils.open_cdf(variable_file) self.save3d &= 'lev' in handler.dimensions if "latitude" in handler.variables: self.lat_name = 'latitude' @@ -167,7 +167,7 @@ class RegionMean(Diagnostic): temp2 = TempFile.get() Utils.nco.ncks(input=mean_file, output=temp2, options=('-v {0},{2.lat_name},{2.lon_name}{1}'.format(original_name, levels, self),)) - handler = Utils.openCdf(temp2) + handler = Utils.open_cdf(temp2) var_handler = handler.variables[original_name] if hasattr(var_handler, 'valid_min'): del var_handler.valid_min diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index b2068d1c..21a731d0 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -118,7 +118,7 @@ class RegionSum(Diagnostic): variable_file = self.variable_file.local_file - handler = Utils.openCdf(variable_file) + handler = Utils.open_cdf(variable_file) self.save3d &= 'lev' in handler.dimensions if "latitude" in handler.variables: self.lat_name = 'latitude' @@ -159,7 +159,7 @@ class RegionSum(Diagnostic): temp2 = TempFile.get() Utils.nco.ncks(input=mean_file, output=temp2, options=('-v {0},{2.lat_name},{2.lon_name}{1}'.format(original_name, levels, self),)) - handler = Utils.openCdf(temp2) + handler = Utils.open_cdf(temp2) var_handler = handler.variables[original_name] if hasattr(var_handler, 'valid_min'): del var_handler.valid_min diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index bdd6132c..2ac29073 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -1,10 +1,11 @@ # coding=utf-8 import shutil + from bscearth.utils.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class Rotation(Diagnostic): @@ -92,7 +93,7 @@ class Rotation(Diagnostic): Runs the diagnostic """ - handler = Utils.openCdf(self.ufile.local_file) + handler = Utils.open_cdf(self.ufile.local_file) if 'lev' in handler.dimensions: self.num_levels = handler.dimensions['lev'].size self.has_levels = True @@ -107,12 +108,12 @@ class Rotation(Diagnostic): urotated = self._merge_levels(self.variableu, 'u') vrotated = self._merge_levels(self.variablev, 'v') - ufile_handler = Utils.openCdf(self.ufile.local_file) + ufile_handler = Utils.open_cdf(self.ufile.local_file) self._add_metadata_and_vars(ufile_handler, urotated, self.variableu) ufile_handler.close() self.urotated_file.set_local_file(urotated) - vfile_handler = Utils.openCdf(self.vfile.local_file) + vfile_handler = Utils.open_cdf(self.vfile.local_file) self._add_metadata_and_vars(vfile_handler, vrotated, self.variablev) vfile_handler.close() self.vrotated_file.set_local_file(urotated) @@ -122,7 +123,7 @@ class Rotation(Diagnostic): if self.has_levels: Utils.nco.ncecat(input=self._get_level_file(0, direction), output=temp, options=("-n {0},2,1 -v '{1}'".format(self.num_levels, var),)) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) if 'record' in handler.dimensions: handler.renameDimension('record', 'lev') handler.close() @@ -164,7 +165,7 @@ class Rotation(Diagnostic): return namelist_file def _add_metadata_and_vars(self, reference_file_handler, rotaded_file, var_name): - rotated_handler = Utils.openCdf(rotaded_file) + rotated_handler = Utils.open_cdf(rotaded_file) self._copy_extra_variables(reference_file_handler, rotated_handler) Utils.copy_attributes(rotated_handler.variables[var_name], reference_file_handler.variables[var_name], ('_FillValue',)) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 4a33aa98..29de93d5 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -89,7 +89,7 @@ class Siasiesiv(Diagnostic): for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, options['basin'], mask, diags.config.var_manager, options['omit_volume'])) - mesh_handler = Utils.openCdf('mesh_hgr.nc') + mesh_handler = Utils.open_cdf('mesh_hgr.nc') Siasiesiv.e1t = np.asfortranarray(mesh_handler.variables['e1t'][0, :]) Siasiesiv.e2t = np.asfortranarray(mesh_handler.variables['e2t'][0, :]) Siasiesiv.gphit = np.asfortranarray(mesh_handler.variables['gphit'][0, :]) @@ -125,7 +125,7 @@ class Siasiesiv(Diagnostic): """ import earthdiagnostics.cdftoolspython as cdftoolspython - sic_handler = Utils.openCdf(self.sic.local_file) + sic_handler = Utils.open_cdf(self.sic.local_file) Utils.convert_units(sic_handler.variables[self.sic_varname], '1.0') sic = np.asfortranarray(sic_handler.variables[self.sic_varname][:]) timesteps = sic_handler.dimensions['time'].size @@ -134,7 +134,7 @@ class Siasiesiv(Diagnostic): if self.omit_volume: sit = sic else: - sit_handler = Utils.openCdf(self.sit.local_file) + sit_handler = Utils.open_cdf(self.sit.local_file) sit = np.asfortranarray(sit_handler.variables[self.sit_varname][:]) sit_handler.close() @@ -155,7 +155,7 @@ class Siasiesiv(Diagnostic): def _extract_variable_and_rename(self, values, cmor_name, units): temp = TempFile.get() - reference_handler = Utils.openCdf(self.sic.local_file) + reference_handler = Utils.open_cdf(self.sic.local_file) os.remove(temp) handler = netCDF4.Dataset(temp, 'w') diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 7c8e1fcc..515aeed3 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -92,7 +92,7 @@ class VerticalGradient(Diagnostic): """ Runs the diagnostic """ - handler = Utils.openCdf(self.variable_file) + handler = Utils.open_cdf(self.variable_file) if 'lev' not in handler.dimensions: raise Exception('Variable {0} does not have a level dimension') var_handler = handler.variables[self.variable] @@ -101,7 +101,7 @@ class VerticalGradient(Diagnostic): gradient = upper_level - lower_level temp = TempFile.get() - new_file = Utils.openCdf(temp, 'w') + new_file = Utils.open_cdf(temp, 'w') for var in handler.variables.keys(): if var in (self.variable, 'lev', 'lev_bnds'): continue diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index d47f538f..040fa5b3 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -94,7 +94,7 @@ class VerticalMean(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - handler = Utils.openCdf(self.variable_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) if self.box.min_depth is None: lev_min = handler.variables['lev'][0] else: diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 50c5e9ad..4eb240ea 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -96,7 +96,7 @@ class VerticalMeanMeters(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - handler = Utils.openCdf(self.variable_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) if self.box.min_depth is None: lev_min = handler.variables['lev'][0] else: diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index 91d46958..2ed57601 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -106,7 +106,7 @@ class DaysOverPercentile(Diagnostic): iris.FUTURE.netcdf_promote = True percentiles = iris.load_cube(self.percentiles_file.local_file) - handler = Utils.openCdf(self.variable_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) if 'realization' in handler.variables: handler.variables[self.variable].coordinates = 'realization' handler.close() @@ -196,10 +196,10 @@ class DaysOverPercentile(Diagnostic): iris.save(results_over[perc].merge_cube(), temp, zlib=True, unlimited_dimensions=['time']) Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, must_exist=False, rename_dimension=True) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) if 'time' not in handler.dimensions: new_file = TempFile.get() - new_handler = Utils.openCdf(new_file, 'w') + new_handler = Utils.open_cdf(new_file, 'w') new_handler.createDimension('time', 1) for dimension in handler.dimensions: diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index 38469cc5..7e078db3 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -145,7 +145,7 @@ class Discretize(Diagnostic): def _load_cube(self): - handler = Utils.openCdf(self.original_data.local_file) + handler = Utils.open_cdf(self.original_data.local_file) if 'realization' in handler.variables: handler.variables[self.variable].coordinates = 'realization' handler.close() diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index 1ecb5c44..aa361f6d 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -1,11 +1,12 @@ # coding=utf-8 +from calendar import monthrange + from bscearth.utils.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable_type import VariableType -from calendar import monthrange class MonthlyPercentile(Diagnostic): @@ -102,7 +103,7 @@ class MonthlyPercentile(Diagnostic): Runs the diagnostic """ temp = TempFile.get() - handler = Utils.openCdf(self.variable_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) datetimes = Utils.get_datetime_from_netcdf(handler) handler.close() @@ -140,19 +141,19 @@ class MonthlyPercentile(Diagnostic): Log.debug('Computing percentile {0}', percentile) Utils.cdo.monpctl(str(percentile), input=[temp, monmin_file, monmax_file], output=temp) Utils.rename_variable(temp, 'lev', 'ensemble', False, True) - handler = Utils.openCdf(monmax_file) + handler = Utils.open_cdf(monmax_file) handler.variables[self.variable].long_name += ' {0} Percentile'.format(percentile) handler.close() self.percentiles[percentile].set_local_file(temp, rename_var=self.variable) Utils.rename_variable(monmax_file, 'lev', 'ensemble', False, True) - handler = Utils.openCdf(monmax_file) + handler = Utils.open_cdf(monmax_file) handler.variables[self.variable].long_name += ' Monthly Maximum' handler.close() self.max_file.set_local_file(monmax_file, rename_var=self.variable) Utils.rename_variable(monmin_file, 'lev', 'ensemble', False, True) - handler = Utils.openCdf(monmin_file) + handler = Utils.open_cdf(monmin_file) handler.variables[self.variable].long_name += ' Monthly Minimum' handler.close() self.min_file.set_local_file(monmin_file, rename_var=self.variable) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index d1f74b19..8bc0fb87 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -68,13 +68,13 @@ class Utils(object): basin = Basins().parse(basin) if basin != Basins().Global: try: - mask_handler = Utils.openCdf('mask_regions.nc') + mask_handler = Utils.open_cdf('mask_regions.nc') mask = mask_handler.variables[basin.name][:, 0, :] mask_handler.close() except IOError: raise Exception('File mask.regions.nc is required for basin {0}'.format(basin)) else: - mask_handler = Utils.openCdf('mask.nc') + mask_handler = Utils.open_cdf('mask.nc') mask = mask_handler.variables['tmask'][0, 0, :] mask_handler.close() return mask @@ -89,14 +89,6 @@ class Utils(object): filename: str variable_list: str or iterable of str - """ - - """ - Sets the valid_max and valid_min values to the current max and min values on the file - :param filename: path to file - :type filename: str - :param variable_list: list of variables in which valid_min and valid_max will be set - :type variable_list: str | list """ # noinspection PyTypeChecker if isinstance(variable_list, six.string_types): @@ -104,7 +96,7 @@ class Utils(object): Log.info('Getting max and min values for {0}', ' '.join(variable_list)) - handler = Utils.openCdf(filename) + handler = Utils.open_cdf(filename) for variable in variable_list: var = handler.variables[variable] values = [np.max(var), np.min(var)] @@ -159,7 +151,7 @@ class Utils(object): for old, new in six.iteritems(dic_names): if old == new: raise ValueError('{0} original name is the same as the new') - handler = Utils.openCdf(filepath) + handler = Utils.open_cdf(filepath) original_names = set(handler.variables.keys()).union(handler.dimensions.keys()) if not any((True for x in dic_names.keys() if x in original_names)): @@ -172,7 +164,7 @@ class Utils(object): temp = TempFile.get() shutil.copyfile(filepath, temp) - handler = Utils.openCdf(temp) + handler = Utils.open_cdf(temp) error = False try: @@ -187,7 +179,7 @@ class Utils(object): if error: Log.debug('First attemp to rename failed. Using secondary rename method for netCDF') - Utils._rename_vars_by_creating_new_file(dic_names, filepath, temp) + Utils._rename_by_new_file(dic_names, filepath, temp) Log.debug('Rename done') Utils.move_file(temp, filepath) @@ -210,7 +202,7 @@ class Utils(object): """ with suppress_stdout(): try: - handler = Utils.openCdf(filepath) + handler = Utils.open_cdf(filepath) if 'time' in handler.variables: if handler.variables['time'].dimensions != ('time', ): handler.close() @@ -221,15 +213,7 @@ class Utils(object): cubes = iris.load(filepath) if len(cubes) == 0: return False - except iris.exceptions.IrisError as ex: - Log.debug('netCDF checks failed: {0}', ex) - return False - except RuntimeError as ex: - # HDF error, usually - Log.debug('netCDF checks failed: {0}', ex) - return False - except Exception as ex: - # HDF error, usually + except (iris.exceptions.IrisError, RuntimeError) as ex: Log.debug('netCDF checks failed: {0}', ex) return False return True @@ -248,15 +232,15 @@ class Utils(object): iterable of str """ - handler = Utils.openCdf(filename) + handler = Utils.open_cdf(filename) variables = handler.variables.keys() handler.close() return variables @staticmethod - def _rename_vars_by_creating_new_file(dic_names, filepath, temp): - original_handler = Utils.openCdf(filepath) - new_handler = Utils.openCdf(temp, 'w') + def _rename_by_new_file(dic_names, filepath, temp): + original_handler = Utils.open_cdf(filepath) + new_handler = Utils.open_cdf(temp, 'w') for attribute in original_handler.ncattrs(): original = getattr(original_handler, attribute) setattr(new_handler, attribute, Utils.convert_to_ascii_if_possible(original)) @@ -533,10 +517,9 @@ class Utils(object): """Number of available virtual or physical CPUs on this system""" if Utils._cpu_count is None: try: - m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', - open('/proc/self/status').read()) - if m: - res = bin(int(m.group(1).replace(',', ''), 16)).count('1') + match = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', open('/proc/self/status').read()) + if match: + res = bin(int(match.group(1).replace(',', ''), 16)).count('1') if res > 0: Utils._cpu_count = res except IOError: @@ -570,7 +553,7 @@ class Utils(object): @classmethod def _is_compressed_netcdf4(cls, filetoconvert): is_compressed = True - handler = Utils.openCdf(filetoconvert) + handler = Utils.open_cdf(filetoconvert) if not handler.file_format == 'NETCDF4': is_compressed = False handler.close() @@ -590,7 +573,7 @@ class Utils(object): # noinspection PyPep8Naming @staticmethod - def openCdf(filepath, mode='a'): + def open_cdf(filepath, mode='a'): """ Open a NetCDF file @@ -739,8 +722,8 @@ class Utils(object): :type remove_source: bool """ if os.path.exists(destiny): - handler_total = Utils.openCdf(destiny) - handler_variable = Utils.openCdf(source) + handler_total = Utils.open_cdf(destiny) + handler_variable = Utils.open_cdf(source) concatenated = dict() for var in handler_variable.variables: if var not in handler_total.variables: @@ -792,20 +775,32 @@ class Utils(object): # noinspection PyBroadException try: os.makedirs(path) - except Exception: - # Here we can have a race condition. Let's check again for existence and rethrow if still not exists + except OSError: + # This could happen if two threads are tying to create the folder. + # Let's check again for existence and rethrow if still not exists if not os.path.isdir(path): raise @staticmethod def give_group_write_permissions(path): - st = os.stat(path) - if st.st_mode & stat.S_IWGRP: + """ Give write permissions to the group""" + stats = os.stat(path) + if stats.st_mode & stat.S_IWGRP: return - os.chmod(path, st.st_mode | stat.S_IWGRP) + os.chmod(path, stats.st_mode | stat.S_IWGRP) @staticmethod def convert_units(var_handler, new_units, calendar=None, old_calendar=None): + """ + Convert units + + Parameters + ---------- + var_handler: Dataset + new_units: str + calendar: str + old_calendar: str + """ if new_units == var_handler.units: return @@ -916,9 +911,9 @@ class TempFile(object): if filename: path = os.path.join(TempFile.scratch_folder, filename) else: - fd, path = tempfile.mkstemp(dir=TempFile.scratch_folder, prefix=TempFile.prefix, suffix=suffix) + file_descriptor, path = tempfile.mkstemp(dir=TempFile.scratch_folder, prefix=TempFile.prefix, suffix=suffix) path = str(path) - os.close(fd) + os.close(file_descriptor) if clean: TempFile.files.append(path) @@ -934,4 +929,3 @@ class TempFile(object): if os.path.exists(temp_file): os.remove(temp_file) TempFile.files = list() - -- GitLab From a369b207b7daca519a194e0481125c14aab698b9 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 10:22:29 +0100 Subject: [PATCH 070/168] Fixed codacy warnings --- earthdiagnostics/diagnostic.py | 26 +++++++++++++++----------- earthdiagnostics/utils.py | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index fb254e96..5fc2dfe8 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -13,6 +13,7 @@ from earthdiagnostics.variable_type import VariableType class DiagnosticStatus(object): """Enumeration of diagnostic status""" + WAITING = 0 READY = 1 RUNNING = 2 @@ -36,8 +37,7 @@ class Diagnostic(Publisher): def __init__(self, data_manager): """ - Diagnostic's constructor - + Diagnostic constructor Parameters ---------- @@ -87,15 +87,15 @@ class Diagnostic(Publisher): if file_generated.has_modifiers(): Log.warning('Can not skip diagnostics run when data is going to be modified: {0}'.format(self)) return False - return True + return False def __repr__(self): - """ Diagnostic's string represenatation. Defaults to str""" + """Full string representation. Defaults to str""" return str(self) @property def status(self): - """ Diagnostic's current status""" + """Current execution status""" return self._status @status.setter @@ -283,7 +283,7 @@ class Diagnostic(Publisher): """ String represenation of the diagnostic - Must be implemented by derived classes + Must be implemented by derived classesgit """ return 'Developer must override base class __str__ method' @@ -342,8 +342,7 @@ class Diagnostic(Publisher): def request_year(self, domain, var, startdate, member, year, grid=None, box=None, frequency=None, to_modify=False): """ - Requests one year of data that is required for the diagnostic - + Request one year of data that is required for the diagnostic Parameters ---------- @@ -386,7 +385,7 @@ class Diagnostic(Publisher): self.check_is_ready() def check_is_ready(self): - """ Check if a diagnostic is ready to run and change its status accordingly """ + """Check if a diagnostic is ready to run and change its status accordingly""" if all([request.ready_to_run(self) for request in self._requests]) and\ all([subjob.status == DiagnosticStatus.COMPLETED for subjob in self.subjobs]): self.status = DiagnosticStatus.READY @@ -421,11 +420,13 @@ class DiagnosticOption(object): """Class to manage string options for the diagnostic""" def __init__(self, name, default_value=None): """ + Option constructor Parameters ---------- name: str default_value: object, optional + If None, the option is required and an exception will be thrown at parse time if the value is empty """ self.name = name self.default_value = default_value @@ -462,13 +463,15 @@ class DiagnosticOption(object): class DiagnosticFloatOption(DiagnosticOption): - """ Class for parsing float options """ + """Class for parsing float options""" + def parse(self, option_value): return float(self._check_default(option_value)) class DiagnosticIntOption(DiagnosticOption): - """ Class for parsing integer options """ + """Class for parsing integer options""" + def __init__(self, name, default_value=None, min_limit=None, max_limit=None): super(DiagnosticIntOption, self).__init__(name, default_value) self.min_limit = min_limit @@ -626,4 +629,5 @@ class DiagnosticChoiceOption(DiagnosticOption): class DiagnosticOptionError(Exception): + """ Exception class for errors related to bad options for the diagnostics""" pass diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 8bc0fb87..f58c1d65 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -783,7 +783,7 @@ class Utils(object): @staticmethod def give_group_write_permissions(path): - """ Give write permissions to the group""" + """Give write permissions to the group""" stats = os.stat(path) if stats.st_mode & stat.S_IWGRP: return -- GitLab From e46ea40c2d4b619e28c8d5714762daa94217276c Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 10:45:49 +0100 Subject: [PATCH 071/168] Updated diagnostic doc --- earthdiagnostics/diagnostic.py | 98 ++++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 5fc2dfe8..425478e4 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -470,7 +470,18 @@ class DiagnosticFloatOption(DiagnosticOption): class DiagnosticIntOption(DiagnosticOption): - """Class for parsing integer options""" + """ + Class for parsing integer options + + Parameters + ---------- + name: str + default_value: list, optional + min_limit: int, optional + If setted, any value below this will not be accepted + max_limit: int, optional + If setted, any value over this will not be accepted + """ def __init__(self, name, default_value=None, min_limit=None, max_limit=None): super(DiagnosticIntOption, self).__init__(name, default_value) @@ -488,14 +499,16 @@ class DiagnosticIntOption(DiagnosticOption): class DiagnosticListIntOption(DiagnosticOption): """ - :param name: - :type name: str - :param default_value: - :type default_value: int|NoneType - :param min_limit: - :type min_limit: int|NoneType - :param max_limit: - :type max_limit: int|NoneType + Class for parsing integer list options + + Parameters + ---------- + name: str + default_value: list, optional + min_limit: int, optional + If setted, any value below this will not be accepted + max_limit: int, optional + If setted, any value over this will not be accepted """ def __init__(self, name, default_value=None, min_limit=None, max_limit=None): @@ -522,6 +535,14 @@ class DiagnosticListIntOption(DiagnosticOption): class DiagnosticListFrequenciesOption(DiagnosticOption): + """ + Class for parsing an option which is a list of frequencies + + Parameters + ---------- + name: str + default_value: list, optional + """ def __init__(self, name, default_value=None): super(DiagnosticListFrequenciesOption, self).__init__(name, default_value) @@ -535,6 +556,15 @@ class DiagnosticListFrequenciesOption(DiagnosticOption): class DiagnosticVariableOption(DiagnosticOption): + """ + Class to parse variable options + + Parameters + ---------- + var_manager: VariableManager + name: str, optional + default_value: str, optional + """ def __init__(self, var_manager, name='variable', default_value=None): super(DiagnosticVariableOption, self).__init__(name, default_value) self.var_manager = var_manager @@ -548,7 +578,15 @@ class DiagnosticVariableOption(DiagnosticOption): class DiagnosticVariableListOption(DiagnosticOption): + """ + Class to parse variable list options + Parameters + ---------- + var_manager: VariableManager + name: str, optional + default_value: str, optional + """ def __init__(self, var_manager, name, default_value=None): super(DiagnosticVariableListOption, self).__init__(name, default_value) self.var_manager = var_manager @@ -566,6 +604,14 @@ class DiagnosticVariableListOption(DiagnosticOption): class DiagnosticDomainOption(DiagnosticOption): + """ + Class to parse domain options + + Parameters + ---------- + name: str, optional + default_value: str, optional + """ def __init__(self, name='domain', default_value=None): super(DiagnosticDomainOption, self).__init__(name, default_value) @@ -574,6 +620,14 @@ class DiagnosticDomainOption(DiagnosticOption): class DiagnosticFrequencyOption(DiagnosticOption): + """ + Class to parse frequency options + + Parameters + ---------- + name: str, optional + default_value: Frequency,optional + """ def __init__(self, name='frequency', default_value=None): super(DiagnosticFrequencyOption, self).__init__(name, default_value) @@ -582,6 +636,9 @@ class DiagnosticFrequencyOption(DiagnosticOption): class DiagnosticBasinOption(DiagnosticOption): + """ + Class to parse basin options + """ def parse(self, option_value): value = self._check_default(option_value) basin = Basins().parse(value) @@ -591,11 +648,19 @@ class DiagnosticBasinOption(DiagnosticOption): class DiagnosticComplexStrOption(DiagnosticOption): + """ + Class to parse complex string options + + It replaces '&;' with ',' and '&.' with ' ' + """ def parse(self, option_value): return self._check_default(option_value).replace('&;', ',').replace('&.', ' ') class DiagnosticBoolOption(DiagnosticOption): + """ + Class to parse boolean options + """ def parse(self, option_value): option_value = self._check_default(option_value) if isinstance(option_value, bool): @@ -605,6 +670,19 @@ class DiagnosticBoolOption(DiagnosticOption): class DiagnosticChoiceOption(DiagnosticOption): + """ + Class to parse choice option + + Parameters + ---------- + name: str + choices: list of str + Valid options for the option + default_value: str, optional + If not None, it should ve a valid choice + ignore_case: bool, optional + If false, value must match case of the valid choice + """ def __init__(self, name, choices, default_value=None, ignore_case=True): super(DiagnosticChoiceOption, self).__init__(name, default_value) self.choices = choices @@ -629,5 +707,5 @@ class DiagnosticChoiceOption(DiagnosticOption): class DiagnosticOptionError(Exception): - """ Exception class for errors related to bad options for the diagnostics""" + """Exception class for errors related to bad options for the diagnostics""" pass -- GitLab From a08d406bb92740ffc7297d39ded4e1adc6031e80 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 10:51:48 +0100 Subject: [PATCH 072/168] Fixed codacy issues --- earthdiagnostics/diagnostic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 425478e4..08d26108 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -565,6 +565,7 @@ class DiagnosticVariableOption(DiagnosticOption): name: str, optional default_value: str, optional """ + def __init__(self, var_manager, name='variable', default_value=None): super(DiagnosticVariableOption, self).__init__(name, default_value) self.var_manager = var_manager @@ -587,6 +588,7 @@ class DiagnosticVariableListOption(DiagnosticOption): name: str, optional default_value: str, optional """ + def __init__(self, var_manager, name, default_value=None): super(DiagnosticVariableListOption, self).__init__(name, default_value) self.var_manager = var_manager @@ -612,6 +614,7 @@ class DiagnosticDomainOption(DiagnosticOption): name: str, optional default_value: str, optional """ + def __init__(self, name='domain', default_value=None): super(DiagnosticDomainOption, self).__init__(name, default_value) @@ -628,6 +631,7 @@ class DiagnosticFrequencyOption(DiagnosticOption): name: str, optional default_value: Frequency,optional """ + def __init__(self, name='frequency', default_value=None): super(DiagnosticFrequencyOption, self).__init__(name, default_value) @@ -636,9 +640,7 @@ class DiagnosticFrequencyOption(DiagnosticOption): class DiagnosticBasinOption(DiagnosticOption): - """ - Class to parse basin options - """ + """Class to parse basin options""" def parse(self, option_value): value = self._check_default(option_value) basin = Basins().parse(value) @@ -653,6 +655,7 @@ class DiagnosticComplexStrOption(DiagnosticOption): It replaces '&;' with ',' and '&.' with ' ' """ + def parse(self, option_value): return self._check_default(option_value).replace('&;', ',').replace('&.', ' ') @@ -661,6 +664,7 @@ class DiagnosticBoolOption(DiagnosticOption): """ Class to parse boolean options """ + def parse(self, option_value): option_value = self._check_default(option_value) if isinstance(option_value, bool): @@ -683,6 +687,7 @@ class DiagnosticChoiceOption(DiagnosticOption): ignore_case: bool, optional If false, value must match case of the valid choice """ + def __init__(self, name, choices, default_value=None, ignore_case=True): super(DiagnosticChoiceOption, self).__init__(name, default_value) self.choices = choices @@ -708,4 +713,5 @@ class DiagnosticChoiceOption(DiagnosticOption): class DiagnosticOptionError(Exception): """Exception class for errors related to bad options for the diagnostics""" + pass -- GitLab From c05b8370154c342a8b11e5f063a60bf8e55abbfd Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 12:43:59 +0100 Subject: [PATCH 073/168] Fixed tests for Python 3 --- earthdiagnostics/config.py | 2 +- earthdiagnostics/datamanager.py | 2 +- earthdiagnostics/diagnostic.py | 59 ++++++++++++++++-------- earthdiagnostics/modelingrealm.py | 3 ++ earthdiagnostics/ocean/interpolatecdo.py | 3 ++ test/unit/test_cmormanager.py | 27 +++++++++++ 6 files changed, 76 insertions(+), 20 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index c07dccff..03d7941d 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -8,8 +8,8 @@ from bscearth.utils.log import Log from earthdiagnostics import cdftools from earthdiagnostics.frequency import Frequency, Frequencies +from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.variable import VariableManager -from modelingrealm import ModelingRealm class ConfigException(Exception): diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index b4022bbd..0c9667c3 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -208,7 +208,7 @@ class UnitConversion(object): Load conversions from the configuration file """ cls._dict_conversions = dict() - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'rb') as csvfile: + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'r') as csvfile: reader = csv.reader(csvfile, dialect='excel') for line in reader: if line[0] == 'original': diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 08d26108..8909a7e8 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""This module contains the Diagnostic base class and all the classes for parsing the options passed to them""" import datetime from bscearth.utils.log import Log @@ -42,6 +43,7 @@ class Diagnostic(Publisher): Parameters ---------- data_manager: DataManager + """ super(Diagnostic, self).__init__() self._generated_files = [] @@ -70,6 +72,9 @@ class Diagnostic(Publisher): """ return not self == other + def __hash__(self): + return hash(str(self)) + def can_skip_run(self): """ Check if a diagnostic calculation can be skipped @@ -95,7 +100,7 @@ class Diagnostic(Publisher): @property def status(self): - """Current execution status""" + """Execution status""" return self._status @status.setter @@ -111,7 +116,7 @@ class Diagnostic(Publisher): self.dispatch(self) @staticmethod - def register(cls): + def register(diagnostic_class): """ Register a new diagnostic using the given alias. @@ -119,13 +124,14 @@ class Diagnostic(Publisher): Parameters ---------- - cls: Type[Diagnostic] + diagnostic_class: Type[Diagnostic] + """ - if not issubclass(cls, Diagnostic): - raise ValueError('Class {0} must be derived from Diagnostic'.format(cls)) - if cls.alias is None: - raise ValueError('Diagnostic class {0} must have defined an alias'.format(cls)) - Diagnostic._diag_list[cls.alias] = cls + if not issubclass(diagnostic_class, Diagnostic): + raise ValueError('Class {0} must be derived from Diagnostic'.format(diagnostic_class)) + if diagnostic_class.alias is None: + raise ValueError('Diagnostic class {0} must have defined an alias'.format(diagnostic_class)) + Diagnostic._diag_list[diagnostic_class.alias] = diagnostic_class # noinspection PyProtectedMember @staticmethod @@ -148,7 +154,7 @@ class Diagnostic(Publisher): def compute(self): """ - Calculates the diagnostic and stores the output + Calculate the diagnostic and stores the output Must be implemented by derived classes """ @@ -156,7 +162,7 @@ class Diagnostic(Publisher): def request_data(self): """ - Calculates the diagnostic and stores the output + Request the data required by the diagnostic Must be implemented by derived classes """ @@ -164,7 +170,7 @@ class Diagnostic(Publisher): def declare_data_generated(self): """ - Calculates the diagnostic and stores the output + Declare the data to be generated by the diagnostic Must be implemented by derived classes """ @@ -191,6 +197,7 @@ class Diagnostic(Publisher): Returns ------- DataFile + """ if isinstance(region, Basin): region = region.name @@ -265,6 +272,7 @@ class Diagnostic(Publisher): ------ DiagnosticOptionError: If there are more options that admitted for the diagnostic + """ processed = dict() options = options[1:] @@ -281,7 +289,7 @@ class Diagnostic(Publisher): def __str__(self): """ - String represenation of the diagnostic + Represenation of the diagnostic as a string Must be implemented by derived classesgit """ @@ -296,6 +304,7 @@ class Diagnostic(Publisher): Parameters ---------- subjob: Diagnostic + """ self.subjobs.append(subjob) subjob.subscribe(self, self._subjob_status_changed) @@ -332,6 +341,7 @@ class Diagnostic(Publisher): request_year declare_chunk declare_year + """ request = self.data_manager.request_chunk(domain, var, startdate, member, chunk, grid, box, frequency, vartype) if to_modify: @@ -365,6 +375,7 @@ class Diagnostic(Publisher): request_chunk declare_chunk declare_year + """ request = self.data_manager.request_year(self, domain, var, startdate, member, year, grid, box, frequency) if to_modify: @@ -401,6 +412,7 @@ class Diagnostic(Publisher): Returns ------- bool + """ return self.pending_requests() == 0 @@ -411,6 +423,7 @@ class Diagnostic(Publisher): Returns ------- int + """ return len([request.storage_status != StorageStatus.READY or request.local_status != LocalStatus.READY for request in self._requests]) @@ -418,6 +431,7 @@ class Diagnostic(Publisher): class DiagnosticOption(object): """Class to manage string options for the diagnostic""" + def __init__(self, name, default_value=None): """ Option constructor @@ -427,6 +441,7 @@ class DiagnosticOption(object): name: str default_value: object, optional If None, the option is required and an exception will be thrown at parse time if the value is empty + """ self.name = name self.default_value = default_value @@ -449,6 +464,7 @@ class DiagnosticOption(object): ------ DiagnosticOptionError: If the option is empty and default_value is False + """ option_value = self._check_default(option_value) return option_value @@ -481,6 +497,7 @@ class DiagnosticIntOption(DiagnosticOption): If setted, any value below this will not be accepted max_limit: int, optional If setted, any value over this will not be accepted + """ def __init__(self, name, default_value=None, min_limit=None, max_limit=None): @@ -509,6 +526,7 @@ class DiagnosticListIntOption(DiagnosticOption): If setted, any value below this will not be accepted max_limit: int, optional If setted, any value over this will not be accepted + """ def __init__(self, name, default_value=None, min_limit=None, max_limit=None): @@ -542,6 +560,7 @@ class DiagnosticListFrequenciesOption(DiagnosticOption): ---------- name: str default_value: list, optional + """ def __init__(self, name, default_value=None): @@ -549,7 +568,7 @@ class DiagnosticListFrequenciesOption(DiagnosticOption): def parse(self, option_value): option_value = self._check_default(option_value) - if isinstance(option_value, tuple) or isinstance(option_value, list): + if isinstance(option_value, (tuple, list)): return option_value values = [Frequency(i) for i in option_value.split('-')] return values @@ -564,6 +583,7 @@ class DiagnosticVariableOption(DiagnosticOption): var_manager: VariableManager name: str, optional default_value: str, optional + """ def __init__(self, var_manager, name='variable', default_value=None): @@ -587,6 +607,7 @@ class DiagnosticVariableListOption(DiagnosticOption): var_manager: VariableManager name: str, optional default_value: str, optional + """ def __init__(self, var_manager, name, default_value=None): @@ -613,6 +634,7 @@ class DiagnosticDomainOption(DiagnosticOption): ---------- name: str, optional default_value: str, optional + """ def __init__(self, name='domain', default_value=None): @@ -630,6 +652,7 @@ class DiagnosticFrequencyOption(DiagnosticOption): ---------- name: str, optional default_value: Frequency,optional + """ def __init__(self, name='frequency', default_value=None): @@ -641,6 +664,7 @@ class DiagnosticFrequencyOption(DiagnosticOption): class DiagnosticBasinOption(DiagnosticOption): """Class to parse basin options""" + def parse(self, option_value): value = self._check_default(option_value) basin = Basins().parse(value) @@ -654,6 +678,7 @@ class DiagnosticComplexStrOption(DiagnosticOption): Class to parse complex string options It replaces '&;' with ',' and '&.' with ' ' + """ def parse(self, option_value): @@ -661,16 +686,13 @@ class DiagnosticComplexStrOption(DiagnosticOption): class DiagnosticBoolOption(DiagnosticOption): - """ - Class to parse boolean options - """ + """Class to parse boolean options""" def parse(self, option_value): option_value = self._check_default(option_value) if isinstance(option_value, bool): return option_value - else: - return option_value.lower() in ('true', 't', 'yes') + return option_value.lower() in ('true', 't', 'yes') class DiagnosticChoiceOption(DiagnosticOption): @@ -686,6 +708,7 @@ class DiagnosticChoiceOption(DiagnosticOption): If not None, it should ve a valid choice ignore_case: bool, optional If false, value must match case of the valid choice + """ def __init__(self, name, choices, default_value=None, ignore_case=True): diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index 703d5b47..11d655f6 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -22,6 +22,9 @@ class ModelingRealm(object): def __eq__(self, other): return other.__class__ == ModelingRealm and self.name == other.name + def __hash__(self): + return hash(self.name) + def __ne__(self, other): return not (self == other) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 1772d7de..ff2ab184 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -68,6 +68,9 @@ class InterpolateCDO(Diagnostic): self.variable == other.variable and self.mask_oceans == other.mask_oceans and self.grid == other.grid and \ self.original_grid == other.original_grid + def __hash__(self): + return hash(str(self)) + def __str__(self): return 'Interpolate with CDO Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ 'Variable: {0.domain}:{0.variable} Target grid: {0.grid} Original grid: {0.original_grid} ' \ diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index f536dd64..c10226c2 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -164,6 +164,8 @@ class TestCMORManager(TestCase): None, frequency, year='1998', date_str='1998') self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 1, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + None, frequency) def test_get_file_path_specs_date_str(self): cmor_manager = CMORManager(self.config) @@ -317,3 +319,28 @@ class TestCMORManager(TestCase): frequency) self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_01.nc')) + + def test_create_link(self): + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) + filename = os.path.basename(path) + self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + + def test_create_link_with_grid(self): + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) + filename = os.path.basename(path) + var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') + var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') + self.assertTrue(os.path.islink(var_mainfolder)) + self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) + self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) + -- GitLab From aca2beaf5aff6ae969456f1997ab29ed8edaee39 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 14:29:20 +0100 Subject: [PATCH 074/168] Clean code --- earthdiagnostics/cmormanager.py | 2 +- earthdiagnostics/config.py | 2 +- earthdiagnostics/diagnostic.py | 32 ++++++++++++++++---------------- earthdiagnostics/work_manager.py | 6 +++--- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 4b75224c..d15ca56c 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -521,7 +521,7 @@ class CMORManager(DataManager): bad_path = os.path.join(self.cmor_path, self.experiment.institute, self.experiment.model, self.experiment.model) Log.debug('Correcting double model appearance') - for (dirpath, dirnames, filenames) in os.walk(bad_path, False): + for (dirpath, _, filenames) in os.walk(bad_path, False): for filename in filenames: if '_S{0}_'.format(startdate) in filename: continue diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 03d7941d..74d0c9f1 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -429,7 +429,7 @@ class ExperimentConfig(object): first_january += 1 years = list() - for chunk in range(first_january, chunks_per_year, self.num_chunks): + for _ in range(first_january, chunks_per_year, self.num_chunks): years.append(first_year) first_year += 1 return years diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 8909a7e8..e56b23bc 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -186,12 +186,12 @@ class Diagnostic(Publisher): domain: ModelingRealm var: str startdate: str - member: int - chunk: int - grid: str - region: Basin - box: Box - frequency: Frequency + member: int or NoneType + chunk: int or NoneType + grid: str or NoneType + region: Basin or NoneType + box: Box or NoneType + frequency: Frequency or NoneType vartype: VariableType Returns @@ -218,10 +218,10 @@ class Diagnostic(Publisher): domain: ModelingRealm var: str startdate: str - member: 1 - year: 1 - grid: str - box: Box + member: int + year: int + grid: str or NoneType + box: Box or NoneType vartype: VariableType Returns @@ -321,12 +321,12 @@ class Diagnostic(Publisher): ---------- domain: ModelingRealm var: str - startdate: str - member: int - chunk: int - grid: str - box: Box - frequency: Frequency + startdate: str or NoneType + member: int or NoneType + chunk: int or NoneType + grid: str or NoneType + box: Box or NoneType + frequency: Frequency or NoneType to_modify: bool Flag that must be active if the diagnostic is going to generate a modified version of this data. In this case this data must not be declared as an output of the diagnostic diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 51bd8b65..e7219dfa 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -5,11 +5,11 @@ import sys import threading import time import traceback - -from bscearth.utils.log import Log # noinspection PyCompatibility from concurrent.futures import ThreadPoolExecutor +from bscearth.utils.log import Log + from earthdiagnostics.datafile import StorageStatus, LocalStatus from earthdiagnostics.diagnostic import DiagnosticStatus, Diagnostic, DiagnosticOptionError from earthdiagnostics.utils import Utils, TempFile @@ -195,7 +195,7 @@ class WorkManager(object): job.compute() except Exception as ex: job.consumed_time = datetime.datetime.now() - start_time - exc_type, exc_value, exc_traceback = sys.exc_info() + exc_type, _, exc_traceback = sys.exc_info() job.message = '{0}\n{1}'.format(ex, ''.join(traceback.format_tb(exc_traceback))) Log.error('Job {0} failed ({2}): {1}', job, job.message, exc_type) job.status = DiagnosticStatus.FAILED -- GitLab From 6cada7cc7006ae105c13cc32489a5fbc46ba62b8 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 14:51:11 +0100 Subject: [PATCH 075/168] Adapated to python3 --- diags.conf | 3 ++- earthdiagnostics/publisher.py | 2 +- earthdiagnostics/utils.py | 2 ++ earthdiagnostics/variable.py | 4 ++-- earthdiagnostics/work_manager.py | 3 ++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/diags.conf b/diags.conf index 8786f6d9..a3176d96 100644 --- a/diags.conf +++ b/diags.conf @@ -88,7 +88,8 @@ OCEAN_TIMESTEP = 6 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = testing_erainterim -STARTDATES = {20050101,20161231,D} +STARTDATES = 20050101 +#{20050101,20161231,D} MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py index 0442ca28..4c60c8f0 100644 --- a/earthdiagnostics/publisher.py +++ b/earthdiagnostics/publisher.py @@ -34,7 +34,7 @@ class Publisher(object): :param args: arguments to pass """ - for callback in self._subscribers.values(): + for callback in tuple(self._subscribers.values()): # noinspection PyCallingNonCallable callback(*args) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index f58c1d65..8d686867 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -502,6 +502,8 @@ class Utils(object): for line in comunicate: if not line: continue + if six.PY3: + line = str(line, encoding='UTF-8') if log_level != Log.NO_LOG: Log.log.log(log_level, line) output.append(line) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index fadc86e4..e1ba8b78 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -107,7 +107,7 @@ class VariableManager(object): return json_folder def _load_file(self, csv_table_path, default=False): - with open(self._get_csv_path(csv_table_path), 'rb') as csvfile: + with open(self._get_csv_path(csv_table_path), 'r') as csvfile: reader = csv.reader(csvfile, dialect='excel') for line in reader: if line[0] == 'Variable': @@ -162,7 +162,7 @@ class VariableManager(object): if not os.path.isfile(file_path): return - with open(file_path, 'rb') as csvfile: + with open(file_path, 'r') as csvfile: reader = csv.reader(csvfile, dialect='excel') for line in reader: if line[0] == 'Aliases': diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index e7219dfa..18304bff 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -7,6 +7,7 @@ import time import traceback # noinspection PyCompatibility from concurrent.futures import ThreadPoolExecutor +from functools import cmp_to_key from bscearth.utils.log import Log @@ -354,7 +355,7 @@ class Downloader(object): return time.sleep(0.01) continue - self._downloads.sort(prioritize) + self._downloads.sort(key=cmp_to_key(prioritize)) datafile = self._downloads[0] self._downloads.remove(datafile) datafile.download() -- GitLab From 46afdfcc9433f9944bf68977c440b13710e122e5 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 17:06:59 +0100 Subject: [PATCH 076/168] Cleaned code --- earthdiagnostics/datafile.py | 90 ++++++++++--------- earthdiagnostics/publisher.py | 4 +- .../statistics/daysoverpercentile.py | 26 +++--- earthdiagnostics/work_manager.py | 2 +- 4 files changed, 63 insertions(+), 59 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index e308ad3e..05d551d0 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -15,6 +15,7 @@ from variable_type import VariableType class LocalStatus(object): + """Local file status enumeration""" PENDING = 0 DOWNLOADING = 1 READY = 2 @@ -24,6 +25,7 @@ class LocalStatus(object): class StorageStatus(object): + """Remote file status enumeration""" PENDING = 0 UPLOADING = 1 READY = 2 @@ -32,6 +34,11 @@ class StorageStatus(object): class DataFile(Publisher): + """ + Represent a data file + + Must be derived for each concrete data file format + """ def __init__(self): super(DataFile, self).__init__() @@ -53,15 +60,15 @@ class DataFile(Publisher): self.job_added = False self._modifiers = [] self._size = None + self.lon_name = None + self.lat_name = None def __str__(self): return 'Data file for {0}'.format(self.remote_file) - def unsubscribe(self, who): - super(DataFile, self).unsubscribe(who) - @property def size(self): + """File size""" if self._size is None: self._get_size() return self._size @@ -70,7 +77,7 @@ class DataFile(Publisher): try: if self.local_status == LocalStatus.READY: self._size = os.path.getsize(self.local_file) - except Exception: + except OSError: self._size = None def clean_local(self): @@ -105,7 +112,7 @@ class DataFile(Publisher): self._modifiers.append(diagnostic) def has_modifiers(self): - return len(self._modifiers) > 0 + return self._modifiers def ready_to_run(self, diagnostic): if not self.local_status == LocalStatus.READY: @@ -183,16 +190,13 @@ class DataFile(Publisher): self.storage_status = StorageStatus.UPLOADING try: Utils.copy_file(self.local_file, self.remote_file, save_hash=True) - except Exception as ex: + except (OSError, Exception) as ex: Log.error('File {0} can not be uploaded: {1}', self.remote_file, ex) self.storage_status = StorageStatus.FAILED return Log.info('File {0} uploaded!', self.remote_file) - try: - self.create_link() - except Exception as ex: - Log.warning('Link for file {0} can not be created: {1}', self.remote_file, ex) + self.create_link() self.storage_status = StorageStatus.READY def set_local_file(self, local_file, diagnostic=None, rename_var='', region=None): @@ -242,10 +246,12 @@ class DataFile(Publisher): if self.cmor_var is not None: if self.cmor_var.valid_min: - valid_min = '-a valid_min,{0},o,{1},"{2}" '.format(self.final_name, var_type.char, self.cmor_var.valid_min) + valid_min = '-a valid_min,{0},o,{1},"{2}" '.format(self.final_name, var_type.char, + self.cmor_var.valid_min) if self.cmor_var.valid_max: - valid_max = '-a valid_max,{0},o,{1},"{2}" '.format(self.final_name, var_type.char, self.cmor_var.valid_max) + valid_max = '-a valid_max,{0},o,{1},"{2}" '.format(self.final_name, var_type.char, + self.cmor_var.valid_max) Utils.nco.ncatted(input=file_path, output=file_path, options=('-O -a _FillValue,{0},o,{1},"1.e20" ' @@ -409,16 +415,13 @@ class DataFile(Publisher): class UnitConversion(object): - """ - Class to manage unit conversions - """ + """Class to manage unit conversions""" + _dict_conversions = None @classmethod def load_conversions(cls): - """ - Load conversions from the configuration file - """ + """Load conversions from the configuration file""" cls._dict_conversions = dict() with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'rb') as csvfile: reader = csv.reader(csvfile, dialect='excel') @@ -430,10 +433,12 @@ class UnitConversion(object): @classmethod def add_conversion(cls, conversion): """ - Adds a conversion to the dictionary + Add a conversion to the dictionary + + Parameters + ---------- + conversion: UnitConversion - :param conversion: conversion to add - :type conversion: UnitConversion """ cls._dict_conversions[(conversion.source, conversion.destiny)] = conversion @@ -446,15 +451,23 @@ class UnitConversion(object): @classmethod def get_conversion_factor_offset(cls, input_units, output_units): """ - Gets the conversion factor and offset for two units . The conversion has to be done in the following way: + Get the conversion factor and offset for two units. + + The conversion has to be done in the following way: converted = original * factor + offset - :param input_units: original units - :type input_units: str - :param output_units: destiny units - :type output_units: str - :return: factor and offset - :rtype: [float, float] + Parameters + ---------- + input_units: str + output_units = str + + Returns + ------- + float + Factor + float + Offset + """ units = input_units.split() if len(units) == 1: @@ -493,14 +506,13 @@ class UnitConversion(object): # Add only the conversions with a factor greater than 1 if unit == new_unit: return 1, 0 - elif (unit, new_unit) in cls._dict_conversions: + if (unit, new_unit) in cls._dict_conversions: conversion = cls._dict_conversions[(unit, new_unit)] return conversion.factor, conversion.offset - elif (new_unit, unit) in cls._dict_conversions: + if (new_unit, unit) in cls._dict_conversions: conversion = cls._dict_conversions[(new_unit, unit)] return 1 / conversion.factor, -conversion.offset - else: - return None, None + return None, None class NetCDFFile(DataFile): @@ -515,10 +527,10 @@ class NetCDFFile(DataFile): Utils.copy_file(self.remote_file, self.local_file) if self.data_convention == 'meteofrance': Log.debug('Converting variable names from meteofrance convention') - self.alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', - 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', - 'y': 'j'} - Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) + alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', + 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', + 'y': 'j'} + Utils.rename_variables(self.local_file, alt_coord_names, must_exist=False, rename_dimension=True) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY @@ -532,7 +544,7 @@ class NetCDFFile(DataFile): try: self.data_manager.create_link(self.domain, self.remote_file, self.frequency, self.final_name, self.grid, True, self.var_type) - except Exception as ex: + except (ValueError, Exception) as ex: Log.error('Can not create link to {1}: {0}'.format(ex, self.remote_file)) def _get_size(self): @@ -541,7 +553,5 @@ class NetCDFFile(DataFile): self._size = os.path.getsize(self.local_file) if self.storage_status == StorageStatus.READY: self._size = os.path.getsize(self.remote_file) - except Exception: + except OSError: self._size = None - - diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py index 4c60c8f0..17cf8f98 100644 --- a/earthdiagnostics/publisher.py +++ b/earthdiagnostics/publisher.py @@ -1,8 +1,6 @@ # coding=utf-8 class Publisher(object): - """ - Base class to provide functionality to notify updates to other objects - """ + """Base class to provide functionality to notify updates to other objects""" def __init__(self): self._subscribers = dict() diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index 2ed57601..defcbb07 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -116,21 +116,17 @@ class DaysOverPercentile(Diagnostic): leadtimes = {1: PartialDateTime(lead_date.year, lead_date.month, lead_date.day)} def assign_leadtime(coord, x): - # noinspection PyBroadException - try: - leadtime_month = 1 - partial_date = leadtimes[leadtime_month] - while coord.units.num2date(x) >= partial_date: - leadtime_month += 1 - try: - partial_date = leadtimes[leadtime_month] - except KeyError: - new_date = add_months(date, leadtime_month, self.data_manager.config.experiment.calendar) - partial_date = PartialDateTime(new_date.year, new_date.month, new_date.day) - leadtimes[leadtime_month] = partial_date - return leadtime_month - except Exception: - pass + leadtime_month = 1 + partial_date = leadtimes[leadtime_month] + while coord.units.num2date(x) >= partial_date: + leadtime_month += 1 + try: + partial_date = leadtimes[leadtime_month] + except KeyError: + new_date = add_months(date, leadtime_month, self.data_manager.config.experiment.calendar) + partial_date = PartialDateTime(new_date.year, new_date.month, new_date.day) + leadtimes[leadtime_month] = partial_date + return leadtime_month iris.coord_categorisation.add_categorised_coord(var, 'leadtime', 'time', assign_leadtime) iris.coord_categorisation.add_year(var, 'time') iris.coord_categorisation.add_day_of_year(var, 'time') diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 18304bff..a840cd2d 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -156,7 +156,7 @@ class WorkManager(object): return False try: self.lock.release() - except Exception: + except threading.ThreadError: pass return True -- GitLab From ddaece865db8002ef798a40e44b08f7d7cd1cf4d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 17:43:29 +0100 Subject: [PATCH 077/168] Updated code --- earthdiagnostics/datafile.py | 72 ++++++++++++++++++++++++++++++++--- earthdiagnostics/publisher.py | 16 ++++++++ 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 05d551d0..f717db21 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -16,6 +16,7 @@ from variable_type import VariableType class LocalStatus(object): """Local file status enumeration""" + PENDING = 0 DOWNLOADING = 1 READY = 2 @@ -26,6 +27,7 @@ class LocalStatus(object): class StorageStatus(object): """Remote file status enumeration""" + PENDING = 0 UPLOADING = 1 READY = 2 @@ -90,15 +92,26 @@ class DataFile(Publisher): self.local_file = None self.local_status = LocalStatus.PENDING - def only_suscriber(self, who): - if len(self._subscribers) != 1: - return - return who in self._subscribers - def upload_required(self): + """ + Get if an upload is needed for this file + + Returns + ------- + bool + + """ return self.local_status == LocalStatus.READY and self.storage_status == StorageStatus.PENDING def download_required(self): + """ + Get if a download is required for this file + + Returns + ------- + bool + + """ if not self.local_status == LocalStatus.PENDING: return False @@ -109,12 +122,44 @@ class DataFile(Publisher): return True def add_modifier(self, diagnostic): + """ + Register a diagnostic as a modifier of this data + + A modifier diagnostic is a diagnostic that read this data and changes it in any way. + The diagnostic must be a modifier even if it only affects the metadata + + Parameters + ---------- + diagnostic: Diagnostic + + """ self._modifiers.append(diagnostic) def has_modifiers(self): - return self._modifiers + """ + Check if it has registered modifiers + + Returns + ------- + bool + """ + return bool(self._modifiers) def ready_to_run(self, diagnostic): + """ + Check if the data is ready to run for a given diagnostics + + To be ready to run, the datafile should be in the local storage and no modifiers can be pending. + + Parameters + ---------- + diagnostic: Diagnostic + + Returns + ------- + bool + + """ if not self.local_status == LocalStatus.READY: return False if len(self._modifiers) == 0: @@ -123,6 +168,7 @@ class DataFile(Publisher): @property def local_status(self): + """Get local storage status""" return self._local_status @local_status.setter @@ -135,6 +181,7 @@ class DataFile(Publisher): @property def storage_status(self): + """Get remote storage status""" return self._storage_status @storage_status.setter @@ -147,6 +194,7 @@ class DataFile(Publisher): @classmethod def from_storage(cls, filepath, data_convention): + """Create a new datafile to be downloaded from the storage""" file_object = cls() file_object.remote_file = filepath file_object.local_status = LocalStatus.PENDING @@ -155,6 +203,7 @@ class DataFile(Publisher): @classmethod def to_storage(cls, remote_file, data_convention): + """Creates a new datafile object for a file that is going to be generated and stored""" new_object = cls() new_object.remote_file = remote_file new_object.storage_status = StorageStatus.PENDING @@ -162,6 +211,17 @@ class DataFile(Publisher): return new_object def download(self): + """ + Get data from remote storage to the local one + + Must be overriden by the derived classes + + Raises + ------ + NotImplementedError + If the derived classes do not override this + + """ raise NotImplementedError('Class must implement the download method') def prepare_to_upload(self, rename_var): diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py index 17cf8f98..14c3a6f2 100644 --- a/earthdiagnostics/publisher.py +++ b/earthdiagnostics/publisher.py @@ -42,3 +42,19 @@ class Publisher(object): List of suscribers of this publisher """ return self._subscribers.keys() + + def only_suscriber(self, who): + """ + Get if an object is the sole suscriber of this publisher + + Parameters + ---------- + who: object + + Returns + ------- + bool + """ + if len(self._subscribers) != 1: + return + return who in self._subscribers -- GitLab From 1ce4d8b541e640a04e8c8d747cb32f9a40853b7c Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 12 Dec 2017 18:00:49 +0100 Subject: [PATCH 078/168] Added napoleon extension to sphinx --- doc/source/conf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 81114e45..436019f1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -20,12 +20,12 @@ import sys # documentation root, use os.path.abspath to make it absolute, like shown here. # sys.path.insert(0, os.path.abspath('.')) sys.path.insert(0, os.path.abspath('../..')) -print os.path.abspath('../..') +print(os.path.abspath('../..')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -39,6 +39,7 @@ extensions = [ 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', ] # Add any paths that contain templates here, relative to this directory. @@ -62,9 +63,9 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # built documents.source ~/vi # # The short X.Y version. -version = '3.0b' +version = '3.0rc' # The full version, including alpha/beta/rc tags. -release = '3.0.0b57' +release = '3.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -- GitLab From cd7540b90afcf452bf894c06ffb3ebd4a194d22f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 10:26:39 +0100 Subject: [PATCH 079/168] Updated prospector conf --- .prospector.yml | 2 ++ earthdiagnostics/datafile.py | 41 ++++++++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.prospector.yml b/.prospector.yml index 3e01f3ca..142832e4 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -27,4 +27,6 @@ pep257: D212, # Multi-line docstring summary should start at the first line D213, # Multi-line docstring summary should start at the second line D404, # First word of the docstring should not be This + D107, # We are using numpy style and constructor should be documented in class docstring + D105, # Docstring in magic methods should not be required: we all now what they are for ] diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index f717db21..5c19f458 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -1,4 +1,5 @@ # coding: utf-8 +"""Module for classes to manage storage manipulation""" import csv import os import shutil @@ -83,7 +84,8 @@ class DataFile(Publisher): self._size = None def clean_local(self): - if self.local_status != LocalStatus.READY or len(self.suscribers) > 0 or self.upload_required() or \ + """Check if a local file is still needed and remove it if not""" + if self.local_status != LocalStatus.READY or self.suscribers or self.upload_required() or \ self.storage_status == StorageStatus.UPLOADING: return Log.debug('File {0} no longer needed. Deleting from scratch...'.format(self.remote_file)) @@ -142,6 +144,7 @@ class DataFile(Publisher): Returns ------- bool + """ return bool(self._modifiers) @@ -162,7 +165,7 @@ class DataFile(Publisher): """ if not self.local_status == LocalStatus.READY: return False - if len(self._modifiers) == 0: + if not self._modifiers: return True return self._modifiers[0] is diagnostic @@ -203,7 +206,7 @@ class DataFile(Publisher): @classmethod def to_storage(cls, remote_file, data_convention): - """Creates a new datafile object for a file that is going to be generated and stored""" + """Create a new datafile object for a file that is going to be generated and stored""" new_object = cls() new_object.remote_file = remote_file new_object.storage_status = StorageStatus.PENDING @@ -225,6 +228,12 @@ class DataFile(Publisher): raise NotImplementedError('Class must implement the download method') def prepare_to_upload(self, rename_var): + """ + Prepare a local file to be uploaded + + This includes renaming the variable if necessary, updating the metadata and adding the history and + managing the possibility of multiple regions + """ if self.data_convention in ('primavera', 'cmip6'): self.lon_name = 'longitude' self.lat_name = 'latitude' @@ -247,6 +256,7 @@ class DataFile(Publisher): self.upload() def upload(self): + """Send a loal file to the storage""" self.storage_status = StorageStatus.UPLOADING try: Utils.copy_file(self.local_file, self.remote_file, save_hash=True) @@ -260,6 +270,23 @@ class DataFile(Publisher): self.storage_status = StorageStatus.READY def set_local_file(self, local_file, diagnostic=None, rename_var='', region=None): + """ + Set the local file generated by EarthDiagnostics + + This also prepares it for the upload + + Parameters + ---------- + local_file: str + diagnostic: Diagnostic or None + rename_var: str + region: Basin or None + + Returns + ------- + None + + """ if diagnostic in self._modifiers: self._modifiers.remove(diagnostic) if region is not None: @@ -271,6 +298,7 @@ class DataFile(Publisher): self.local_status = LocalStatus.READY def create_link(self): + """Create a link from the original data in the _ folder""" pass def _correct_metadata(self): @@ -449,6 +477,7 @@ class DataFile(Publisher): Utils.rename_variables(self.local_file, variables, False, True) def add_diagnostic_history(self): + """Add the history line corresponding to the diagnostic to the local file""" if not self.diagnostic: return from earthdiagnostics.earthdiags import EarthDiags @@ -457,6 +486,7 @@ class DataFile(Publisher): self._add_history_line(history_line) def add_cmorization_history(self): + """Add the history line corresponding to the cmorization to the local file""" from earthdiagnostics.earthdiags import EarthDiags history_line = 'CMORized with Earthdiagnostics version {0}'.format(EarthDiags.version) self._add_history_line(history_line) @@ -483,7 +513,7 @@ class UnitConversion(object): def load_conversions(cls): """Load conversions from the configuration file""" cls._dict_conversions = dict() - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'rb') as csvfile: + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'r') as csvfile: reader = csv.reader(csvfile, dialect='excel') for line in reader: if line[0] == 'original': @@ -576,8 +606,10 @@ class UnitConversion(object): class NetCDFFile(DataFile): + """Implementation of DataFile for netCDF files""" def download(self): + """Get data from remote storage to the local one""" try: self.local_status = LocalStatus.DOWNLOADING Log.debug('Downloading file {0}...', self.remote_file) @@ -601,6 +633,7 @@ class NetCDFFile(DataFile): self.local_status = LocalStatus.FAILED def create_link(self): + """Create a link from the original data in the _ folder""" try: self.data_manager.create_link(self.domain, self.remote_file, self.frequency, self.final_name, self.grid, True, self.var_type) -- GitLab From 42870b04e5f3a7aecc370bb0dfc59effe1dbf9b5 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 11:16:05 +0100 Subject: [PATCH 080/168] Updated doc --- earthdiagnostics/cmormanager.py | 365 +++++++++++++++++++------------- 1 file changed, 217 insertions(+), 148 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index d15ca56c..829be9c4 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -19,7 +19,13 @@ from earthdiagnostics.variable_type import VariableType class CMORManager(DataManager): """ Data manager class for CMORized experiments + + Parameters + ---------- + config: earthdiagnostics.config.Config + """ + def __init__(self, config): super(CMORManager, self).__init__(config) self._dic_cmorized = dict() @@ -27,6 +33,16 @@ class CMORManager(DataManager): self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid) def find_model_data(self): + """ + Seek the configured data folders for the experiment data + + For each folder, it looks at: + -/ + -// + -/// + + Model has any '-' character removed and is passed to lower + """ data_folders = self.config.data_dir.split(':') experiment_folder = self.experiment.model.lower() if experiment_folder.startswith('ec-earth'): @@ -51,6 +67,27 @@ class CMORManager(DataManager): # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN, possible_versions=None): + """ + Check if a file exists in the storage + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None + box: Box or None + frequency: Frequency or None + vartype: VariableType + possible_versions: iterable od str or None + + Returns + ------- + bool + + """ cmor_var = self.variable_list.get_variable(var) filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) @@ -64,27 +101,24 @@ class CMORManager(DataManager): def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - - :param vartype: - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|NoneType - :return: path to the copy created on the scratch folder - :rtype: str + Request a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None + box: Box or None + frequency: Frequency or None + vartype: VariableType or None + + Returns + ------- + DataFile + """ cmor_var = self.variable_list.get_variable(var) var = self._get_final_var_name(box, var) @@ -94,28 +128,25 @@ class CMORManager(DataManager): def request_year(self, diagnostic, domain, var, startdate, member, year, grid=None, box=None, frequency=None): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - - :param year: - :param diagnostic: - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|NoneType - :return: path to the copy created on the scratch folder - :rtype: str - """ + Request a given year for a variavle from a CMOR repository + + Parameters + ---------- + diagnostic: Diagnostic + domain: ModelingRealm + var: str + startdate: str + member: int + year: int + grid: str or None + box: Box or None + frequency: Frequency or None + + Returns + ------- + DataFile + """ job = MergeYear(self, domain, var, startdate, member, year, grid, box, frequency) job.request_data() job.declare_data_generated() @@ -127,30 +158,26 @@ class CMORManager(DataManager): def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, vartype=VariableType.MEAN, diagnostic=None): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - - :param diagnostic: - :param region: - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|NoneType - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str + Declares a variable chunk to be generated by a diagnostic + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None + region: Basin or None + box: Box or None + frequency: Frequency or None + vartype: VariableType + diagnostic: Diagnostic + + Returns + ------- + DataFile + """ if not frequency: frequency = self.config.frequency @@ -169,26 +196,24 @@ class CMORManager(DataManager): def declare_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN, diagnostic=None): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - - :param diagnostic: - :param year: - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str + Declares a variable year to be generated by a diagnostic + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + year: int + grid: str or None + box: Box or None + vartype: VariableType + diagnostic: Diagnostic + + Returns + ------- + DataFile + """ original_name = var cmor_var = self.variable_list.get_variable(var) @@ -207,31 +232,30 @@ class CMORManager(DataManager): grid=None, year=None, date_str=None): """ Returns the path to a concrete file - :param cmor_var: - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param domain: file's domain - :type domain: Domain - :param var: file's var - :type var: var - :param chunk: file's chunk - :type chunk: int|NoneType - :param frequency: file's frequency - :type frequency: Frequency - :param grid: file's grid - :type grid: str|NoneType - :param year: file's year - :type year: int|str|NoneType - :param date_str: date string to add directly. Overrides year or chunk configurations - :type date_str: str|NoneType - :return: path to the file - :rtype: str|NoneType - :param cmor_var: variable instance describing the selected variable - :type cmor_var: Variable - """ + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + chunk: int or None + frequency: Frequency + grid: str or None + year: int or None + date_str: str or None + + Returns + ------- + str + + Raises + ------ + ValueError + If you provide two or more parameters from chunk, year or date_str or none at all + + """ options = sum(x is not None for x in (chunk, year, date_str)) if options == 0: raise ValueError('You must provide chunk, year or date_str') @@ -345,50 +369,37 @@ class CMORManager(DataManager): """ Creates the link of a given file from the CMOR repository. - :param cmor_var: - :param move_old: - :param date_str: - :param year: if frequency is yearly, this parameter is used to give the corresponding year - :type year: int - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str - :param cmor_var: variable instance describing the selected variable - :type cmor_var: Variable + Parameters + ---------- + domain: ModelingRealm + var: str + cmor_var: + startdate: str + member: int + chunk: int or None, optional + grid: str or None, optional + frequency: Frequency or None, optional + year: int or None, optional + date_str: str or None, optional + move_old: bool, optional + vartype: VariableType, optional """ if not frequency: frequency = self.config.frequency filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, - grid=grid, year=str(year), date_str=date_str) + grid=grid, year=year, date_str=date_str) self.create_link(domain, filepath, frequency, var, grid, move_old, vartype) - # noinspection PyPep8Naming def prepare(self): """ - Prepares the data to be used by the diagnostic. + Prepare the data to be used by the diagnostic. If CMOR data is not created, it show a warning and closes. In the future, an automatic cmorization procedure will be launched If CMOR data is available but packed, the procedure will unpack it. - :return: """ # Check if cmorized and convert if not @@ -400,6 +411,23 @@ class CMORManager(DataManager): self._cmorize_member(startdate, member) def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a chunk domain is cmorized + + A cache is maintained so only the first check is costly + + Parameters + ---------- + startdate: str + member: int + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + """ identifier = (startdate, member, chunk) if identifier not in self._dic_cmorized: self._dic_cmorized[identifier] = {} @@ -549,6 +577,20 @@ class CMORManager(DataManager): Log.debug('Done') def create_links(self, startdate, member=None): + """ + Create links for a gicen startdate or member + + Parameters + ---------- + startdate: str + member: int or None + + Returns + ------- + ValueError: + If the data convention is not supported + + """ if member is not None: member_str = self._get_member_str(member) else: @@ -556,15 +598,15 @@ class CMORManager(DataManager): Log.info('Creating links for CMOR files ({0})', startdate) path = self._get_startdate_path(startdate) if self.config.data_convention.upper() in ('SPECS', 'APPLICATE'): - self._create_links_CMIP5(member_str, path) + self._create_links_cmip5(member_str, path) elif self.config.data_convention.upper() in ('CMIP6', 'PRIMAVERA'): - self._create_links_CMIP6(member_str, path) + self._create_links_cmip6(member_str, path) else: raise ValueError('Dataset convention {0} not supported for massive ' 'link creation'.format(self.config.data_convention)) Log.debug('Links ready') - def _create_links_CMIP5(self, member_str, path): + def _create_links_cmip5(self, member_str, path): for freq in os.listdir(path): frequency = Frequency.parse(freq) for domain in os.listdir(os.path.join(path, freq)): @@ -582,7 +624,7 @@ class CMORManager(DataManager): self.create_link(domain, os.path.join(filepath, filename), frequency, var, "", False, vartype=VariableType.MEAN) - def _create_links_CMIP6(self, member_str, path): + def _create_links_cmip6(self, member_str, path): for member in os.listdir(path): for table in os.listdir(os.path.join(path, member)): frequency = self.variable_list.tables[table].frequency @@ -612,11 +654,15 @@ class CMORManager(DataManager): def _get_startdate_path(self, startdate): """ - Returns the path to the startdate's CMOR folder - :param startdate: target startdate - :type startdate: str - :return: path to the startdate's CMOR º - :rtype: str + Return the path to the startdate's CMOR folder + + Parameters + ---------- + startdate: str + + Returns + ------- + str """ if self.config.data_convention == 'specs': return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.experiment.institute, @@ -642,8 +688,28 @@ class CMORManager(DataManager): class MergeYear(Diagnostic): + """ + Diagnostic to get all the data for a given year and merge it in a file + + Parameters + ---------- + data_manager: DataManager + domain: ModelingRealm + var: str + startdate: str + member: int + year: int + grid: str or None, optional + box: Box or None, optional + frequency: Frequency or None, optional + """ @classmethod def generate_jobs(cls, diags, options): + """ + Method to generate the required diagnostics from a section of the configuration file + + Required by the interface, does nothing as this diagnostic is not meant to be configured in the usal way + """ pass def __init__(self, data_manager, domain, var, startdate, member, year, grid=None, box=None, frequency=None): @@ -660,16 +726,19 @@ class MergeYear(Diagnostic): self.frequency = frequency def request_data(self): + """Request all the data required by the diagnostic""" for chunk in self.experiment.get_year_chunks(self.startdate, self.year): self.chunk_files.append(self.request_chunk(self.domain, self.var, self.startdate, self.member, chunk, grid=self.grid, box=self.box, frequency=self.frequency)) def declare_data_generated(self): + """Declare all the data generated by the diagnostic""" self.year_file = self.declare_year(self.domain, self.var, self.startdate, self.member, self.year, grid=self.grid, box=self.box) self.year_file.storage_status = StorageStatus.NO_STORE def compute(self): + """Create the yearly file for the data""" temp = self._merge_chunk_files() temp2 = self._select_data_of_given_year(temp) self.year_file.set_local_file(temp2) -- GitLab From 1d386ced25f06e713a58993e813b7ec429702cd0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 11:22:32 +0100 Subject: [PATCH 081/168] replaced nonetype with none in doc --- earthdiagnostics/cdftools.py | 2 +- earthdiagnostics/datamanager.py | 4 ++-- earthdiagnostics/diagnostic.py | 30 ++++++++++++++--------------- earthdiagnostics/obsreconmanager.py | 8 ++++---- earthdiagnostics/publisher.py | 2 +- earthdiagnostics/threddsmanager.py | 8 ++++---- earthdiagnostics/utils.py | 2 +- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index 9537dc86..6283753a 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -31,7 +31,7 @@ class CDFTools(object): :param output_file: output file. Not all tools support this parameter :type options: str :param options: options for the tool. - :type options: str | [str] | Tuple[str] | NoneType + :type options: str | [str] | Tuple[str] | None :param log_level: log level at which the output of the cdftool command will be added :type log_level: int :param input_option: option to add before input file diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 0c9667c3..92a74332 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -183,11 +183,11 @@ class DataManager(object): :param chunk: file's chunk :type chunk: int :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType + :type grid: str|None :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|NoneType + :type frequency: Frequency|None :return: path to the copy created on the scratch folder :param vartype: Variable type (mean, statistic) :type vartype: VariableType diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index e56b23bc..11fb3ae0 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -62,7 +62,7 @@ class Diagnostic(Publisher): Parameters ---------- - other: Diagnostic or NoneType + other: Diagnostic or None Diagnostic to be compared Returns @@ -186,12 +186,12 @@ class Diagnostic(Publisher): domain: ModelingRealm var: str startdate: str - member: int or NoneType - chunk: int or NoneType - grid: str or NoneType - region: Basin or NoneType - box: Box or NoneType - frequency: Frequency or NoneType + member: int or None + chunk: int or None + grid: str or None + region: Basin or None + box: Box or None + frequency: Frequency or None vartype: VariableType Returns @@ -220,8 +220,8 @@ class Diagnostic(Publisher): startdate: str member: int year: int - grid: str or NoneType - box: Box or NoneType + grid: str or None + box: Box or None vartype: VariableType Returns @@ -321,12 +321,12 @@ class Diagnostic(Publisher): ---------- domain: ModelingRealm var: str - startdate: str or NoneType - member: int or NoneType - chunk: int or NoneType - grid: str or NoneType - box: Box or NoneType - frequency: Frequency or NoneType + startdate: str or None + member: int or None + chunk: int or None + grid: str or None + box: Box or None + frequency: Frequency or None to_modify: bool Flag that must be active if the diagnostic is going to generate a modified version of this data. In this case this data must not be declared as an output of the diagnostic diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index fa99ad5e..e680fb21 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -208,11 +208,11 @@ class ObsReconManager(DataManager): :param chunk: file's chunk :type chunk: int :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType + :type grid: str|None :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|NoneType + :type frequency: Frequency|None :return: path to the copy created on the scratch folder :rtype: str """ @@ -240,11 +240,11 @@ class ObsReconManager(DataManager): :param chunk: file's chunk :type chunk: int :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType + :type grid: str|None :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|NoneType + :type frequency: Frequency|None :param vartype: Variable type (mean, statistic) :type vartype: VariableType :return: path to the copy created on the scratch folder diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py index 14c3a6f2..395a92b8 100644 --- a/earthdiagnostics/publisher.py +++ b/earthdiagnostics/publisher.py @@ -11,7 +11,7 @@ class Publisher(object): :param who: subscriber to add :type who: object :param callback: method to execute when publisher updates - :type callback: callable | NoneType + :type callback: callable | None """ if callback is None: callback = getattr(who, 'update') diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 2ba4f6f8..8064b8a4 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -250,11 +250,11 @@ class THREDDSManager(DataManager): :param chunk: file's chunk :type chunk: int :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType + :type grid: str|None :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|NoneType + :type frequency: Frequency|None :return: path to the copy created on the scratch folder :rtype: str """ @@ -289,11 +289,11 @@ class THREDDSManager(DataManager): :param chunk: file's chunk :type chunk: int :param grid: file's grid (only needed if it is not the original) - :type grid: str|NoneType + :type grid: str|None :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|NoneType + :type frequency: Frequency|None :param vartype: Variable type (mean, statistic) :type vartype: VariableType :return: path to the copy created on the scratch folder diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 8d686867..cbbf6f11 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -694,7 +694,7 @@ class Utils(object): destiny: netCDF4.Dataset dimension: str must_exist: bool, optional - new_names: dict of str: str or NoneType, optional + new_names: dict of str: str or None, optional """ if not must_exist and dimension not in source.dimensions.keys(): return -- GitLab From 0bb0c4ab278c45a1f28d4ae388ac66f155529aad Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 11:29:47 +0100 Subject: [PATCH 082/168] Fixed some cmormanager issues --- earthdiagnostics/cmormanager.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 829be9c4..069dfab0 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Classes to manage cmorized datasets""" import glob import os from datetime import datetime @@ -158,7 +159,7 @@ class CMORManager(DataManager): def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, vartype=VariableType.MEAN, diagnostic=None): """ - Declares a variable chunk to be generated by a diagnostic + Declare a variable chunk to be generated by a diagnostic Parameters ---------- @@ -196,7 +197,7 @@ class CMORManager(DataManager): def declare_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN, diagnostic=None): """ - Declares a variable year to be generated by a diagnostic + Declare a variable year to be generated by a diagnostic Parameters ---------- @@ -231,7 +232,7 @@ class CMORManager(DataManager): def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency, grid=None, year=None, date_str=None): """ - Returns the path to a concrete file + Return the path to a concrete file Parameters ---------- @@ -365,9 +366,8 @@ class CMORManager(DataManager): def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): - """ - Creates the link of a given file from the CMOR repository. + Create the link of a given file from the CMOR repository. Parameters ---------- @@ -384,7 +384,6 @@ class CMORManager(DataManager): move_old: bool, optional vartype: VariableType, optional """ - if not frequency: frequency = self.config.frequency filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, @@ -703,6 +702,7 @@ class MergeYear(Diagnostic): box: Box or None, optional frequency: Frequency or None, optional """ + @classmethod def generate_jobs(cls, diags, options): """ @@ -785,5 +785,5 @@ class MergeYear(Diagnostic): def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.year == other.year and \ - self.domain == other.domain and self.var == other.var and self.grid == other.grid and \ - self.box == other.box + self.domain == other.domain and self.var == other.var and self.grid == other.grid and \ + self.box == other.box -- GitLab From 3220469d0e6663916b39e32f3cabf2f4c639ccb3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 12:38:55 +0100 Subject: [PATCH 083/168] Improved tests for cmormanager --- test/unit/test_cmormanager.py | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index c10226c2..ab414762 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -25,6 +25,7 @@ class TestCMORManager(TestCase): self.config.experiment.atmos_timestep = 6 self.config.experiment.ocean_timestep = 6 self.config.experiment.chunk_size = 12 + self.config.experiment.num_chunks = 1 self.config.experiment.calendar = 'standard' self.config.cmor.initialization_number = 1 @@ -32,6 +33,8 @@ class TestCMORManager(TestCase): self.config.cmor.default_ocean_grid = 'ocean_grid' self.config.cmor.default_atmos_grid = 'atmos_grid' self.config.cmor.activity = 'activity' + self.config.cmor.force = False + self.config.cmor.force_untar = False self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) @@ -344,3 +347,66 @@ class TestCMORManager(TestCase): self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) + @mock.patch('earthdiagnostics.cmormanager.Cmorizer', autospec=True) + def test_prepare_cmorize(self, mock_cmor): + mock_instance = mock_cmor.return_value + cmor_manager = CMORManager(self.config) + self.config.experiment.get_member_list.return_value = (('20000101', 2),) + cmor_manager.prepare() + mock_instance.cmorize_ocean.assert_called_once() + mock_instance.cmorize_atmos.assert_called_once() + + @mock.patch('earthdiagnostics.cmormanager.Cmorizer', autospec=True) + def test_prepare_cmorize_force(self, mock_cmor): + mock_instance = mock_cmor.return_value + self.config.cmor.force = True + cmor_manager = CMORManager(self.config) + self.config.experiment.get_member_list.return_value = (('20000101', 2),) + cmor_manager.prepare() + mock_instance.cmorize_ocean.assert_called_once() + mock_instance.cmorize_atmos.assert_called_once() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.is_cmorized') + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_links') + @mock.patch('earthdiagnostics.utils.Utils.unzip') + @mock.patch('earthdiagnostics.utils.Utils.untar') + def test_prepare_cmorize_force_untar(self, mock_untar, mock_unzip, mock_create_links, mock_cmorized): + original_cmor_path = os.path.join(self.config.data_dir, self.config.experiment.expid, + 'original_files', 'cmorfiles') + os.makedirs(original_cmor_path) + self.config.experiment.get_member_str.return_value = 'r1i1p1' + self.config.experiment.get_chunk_start_str.return_value = '20000101' + cmor_prefix = 'CMORT_{0}_{1}_{2}_{3}-'.format(self.config.experiment.expid, '20000101', 'r1i1p1', + '20000101') + tempfile.mkstemp('.tar.gz', cmor_prefix, original_cmor_path) + tempfile.mkstemp('.tar', cmor_prefix, original_cmor_path) + mock_cmorized.return_value = True + self.config.cmor.force_untar = True + cmor_manager = CMORManager(self.config) + self.config.experiment.get_member_list.return_value = (('20000101', 2),) + cmor_manager.prepare() + mock_create_links.assert_called_once() + mock_unzip.assert_called_once() + mock_untar.assert_called_once() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.is_cmorized') + def test_prepare_already_cmorized(self, mock_cmorized): + mock_cmorized.return_value = True + self.config.cmor.force_untar = False + cmor_manager = CMORManager(self.config) + self.config.experiment.get_member_list.return_value = (('20000101', 2),) + cmor_manager.prepare() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.is_cmorized') + def test_prepare_cmorization_not_requested(self, mock_cmorized): + mock_cmorized.return_value = False + self.config.cmor.chunk_cmorization_requested.return_value = False + cmor_manager = CMORManager(self.config) + self.config.experiment.get_member_list.return_value = (('20000101', 2),) + cmor_manager.prepare() + + def test_prepare_meteofrance(self): + self._configure_meteofrance() + cmor_manager = CMORManager(self.config) + cmor_manager.prepare() + -- GitLab From 9101650edefe25de375911dd561a43915d3496d4 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 15:39:45 +0100 Subject: [PATCH 084/168] Added test for cmormanager --- earthdiagnostics/cmormanager.py | 44 +++++------ earthdiagnostics/obsreconmanager.py | 6 +- earthdiagnostics/threddsmanager.py | 4 +- test/unit/test_cmormanager.py | 117 ++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 29 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 069dfab0..c2e53bf7 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -180,7 +180,7 @@ class CMORManager(DataManager): DataFile """ - if not frequency: + if frequency is None: frequency = self.config.frequency original_name = var cmor_var = self.variable_list.get_variable(var) @@ -262,7 +262,7 @@ class CMORManager(DataManager): raise ValueError('You must provide chunk, year or date_str') elif options > 1: raise ValueError('You must provide only one parameter in chunk, year or date_str') - if not frequency: + if frequency is None: frequency = self.config.frequency folder_path = self._get_full_cmor_folder_path(startdate, member, domain, var, frequency, grid, cmor_var) @@ -384,7 +384,7 @@ class CMORManager(DataManager): move_old: bool, optional vartype: VariableType, optional """ - if not frequency: + if frequency is None: frequency = self.config.frequency filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid=grid, year=year, date_str=date_str) @@ -442,19 +442,11 @@ class CMORManager(DataManager): count = 0 if self.config.data_convention == 'specs': for freq in os.listdir(startdate_path): - domain_path = os.path.join(startdate_path, freq, - domain.name) + domain_path = os.path.join(startdate_path, freq, domain.name) if os.path.isdir(domain_path): - for var in os.listdir(domain_path): - cmor_var = self.variable_list.get_variable(var, True) - var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, - Frequency(freq)) - if os.path.isfile(var_path): - count += 1 - if count >= self.config.cmor.min_cmorized_vars: - return True - else: - continue + count = self._check_var_presence(domain_path, count, startdate, member, domain, chunk, freq) + if count >= self.config.cmor.min_cmorized_vars: + return True else: member_path = os.path.join(startdate_path, self._get_member_str(member)) if not os.path.isdir(member_path): @@ -464,17 +456,21 @@ class CMORManager(DataManager): table_dir = os.path.join(member_path, table.name) if not os.path.isdir(table_dir): return False - for var in os.listdir(table_dir): - cmor_var = self.variable_list.get_variable(var, True) - var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq) - if os.path.isfile(var_path): - count += 1 - if count >= self.config.cmor.min_cmorized_vars: - return True - else: - continue + count = self._check_var_presence(table_dir, count, startdate, member, domain, chunk, freq) + if count >= self.config.cmor.min_cmorized_vars: + return True return False + def _check_var_presence(self, folder, current_count, startdate, member, domain, chunk, freq): + for var in os.listdir(folder): + cmor_var = self.variable_list.get_variable(var, True) + var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq) + if os.path.isfile(var_path): + current_count += 1 + if current_count >= self.config.cmor.min_cmorized_vars: + break + return current_count + def _cmorize_member(self, startdate, member): start_time = datetime.now() member_str = self.experiment.get_member_str(member) diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index e680fb21..f9f35374 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -87,7 +87,7 @@ class ObsReconManager(DataManager): :param vartype: Variable type (mean, statistic) :type vartype: VariableType """ - if not frequency: + if frequency is None: frequency = self.config.frequency folder_path = self._get_folder_path(frequency, domain, var, grid, vartype) @@ -143,7 +143,7 @@ class ObsReconManager(DataManager): :type vartype: VariableType :return: """ - if not frequency: + if frequency is None: frequency = self.config.frequency var = self._get_final_var_name(box, var) full_path = os.path.join(self.config.data_dir, self.config.data_type, self.experiment.institute, @@ -250,7 +250,7 @@ class ObsReconManager(DataManager): :return: path to the copy created on the scratch folder :rtype: str """ - if not frequency: + if frequency is None: frequency = self.config.frequency original_name = var cmor_var = self.variable_list.get_variable(var) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 8064b8a4..331fd55a 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -115,7 +115,7 @@ class THREDDSManager(DataManager): :param vartype: Variable type (mean, statistic) :type vartype: VariableType """ - if not frequency: + if frequency is None: frequency = self.config.frequency var = self._get_final_var_name(box, var) @@ -180,7 +180,7 @@ class THREDDSManager(DataManager): :type vartype: VariableType :return: """ - if not frequency: + if frequency is None: frequency = self.config.frequency var = self._get_final_var_name(box, var) full_path = os.path.join(self.server_url, 'dodsC', self.config.data_type, self.experiment.institute, diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index ab414762..1931f99f 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -410,3 +410,120 @@ class TestCMORManager(TestCase): cmor_manager = CMORManager(self.config) cmor_manager.prepare() + def test_is_cmorized_false(self): + cmor_manager = CMORManager(self.config) + self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_cmorized_true(self, mock_is_file): + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + cmor_manager = CMORManager(self.config) + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S20000101/mon/ocean/var')) + self.assertTrue(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_cmorized_not_enough_vars(self, mock_is_file): + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + cmor_manager = CMORManager(self.config) + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S20000101/mon/ocean/var')) + self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_not_domain_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + cmor_manager = CMORManager(self.config) + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S20000101/mon')) + self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_cmorized_true_primavera(self, mock_is_file): + self._configure_primavera() + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + cmor_manager = CMORManager(self.config) + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var')) + self.assertTrue(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_cmorized_false_primavera(self, mock_is_file): + self._configure_primavera() + mock_is_file.return_value = False + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + cmor_manager = CMORManager(self.config) + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var')) + self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_false_not_member_folder_primavera(self): + self._configure_primavera() + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + cmor_manager = CMORManager(self.config) + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/expname/')) + self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_false_not_table_folder_primavera(self): + self._configure_primavera() + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + cmor_manager = CMORManager(self.config) + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1')) + self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') + def test_link_file(self, mock_get_file_path, mock_create_link): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + cmor_manager.link_file(ModelingRealms.ocean, 'var', cmor_var, '20010101', 1, 1) + mock_get_file_path.assert_called_once() + mock_create_link.assert_called_once() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') + def test_link_file(self, mock_get_file_path, mock_create_link): + frequency = Mock() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + cmor_manager.link_file(ModelingRealms.ocean, 'var', cmor_var, '20010101', 1, 1, frequency=frequency) + mock_get_file_path.assert_called_once() + mock_create_link.assert_called_once() + + -- GitLab From 11a241458c3d8ef5bff57542b762f95bed8e33f8 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 16:47:06 +0100 Subject: [PATCH 085/168] Added test for cmormanager --- earthdiagnostics/cmormanager.py | 20 +++--- test/unit/test_cmormanager.py | 106 ++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index c2e53bf7..7995fdee 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -631,18 +631,10 @@ class CMORManager(DataManager): for name in os.listdir(os.path.join(path, member, table, var, grid)): filepath = os.path.join(path, member, table, var, grid, name) if os.path.isfile(filepath): - original_handler = Utils.open_cdf(filepath) - if original_handler.dimensions['i'].size < original_handler.dimensions['j'].size: - original_handler.close() - Utils.rename_variables(filepath, {'i': 'j', 'j': 'i'}, False, True) - else: - original_handler.close() self.create_link(domain, filepath, frequency, var, "", False, vartype=VariableType.MEAN) else: for filename in os.listdir(filepath): - if not filename.endswith('.nc') or filename.startswith('.'): - return cmorfile = os.path.join(filepath, filename) self.create_link(domain, cmorfile, frequency, var, "", False, vartype=VariableType.MEAN) @@ -721,6 +713,18 @@ class MergeYear(Diagnostic): self.box = box self.frequency = frequency + def __str__(self): + return 'Merge year data Variable: {0.domain}:{0.var} Startdate: {0.startdate} Member: {0.member} ' \ + 'Year: {0.year} Grid: {0.grid} Box: {0.box} Frequency: {0.frequency}'.format(self) + + def __eq__(self, other): + return self.domain == other.domain and self.var == other.var and self.startdate == other.startdate and \ + self.member == other.member and self.year == other.year and self.grid == other.grid and \ + self.box == other.box and self.frequency == other.frequency + + def __hash__(self): + return hash(str(self)) + def request_data(self): """Request all the data required by the diagnostic""" for chunk in self.experiment.get_year_chunks(self.startdate, self.year): diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 1931f99f..2fe9ae56 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -526,4 +526,110 @@ class TestCMORManager(TestCase): mock_get_file_path.assert_called_once() mock_create_link.assert_called_once() + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + def test_create_links_specs(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/expname/S20010101/mon/ocean/var/r2i1p1') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + cmor_manager = CMORManager(self.config) + cmor_manager.create_links('20010101', 1) + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + def test_create_links_specs_member_not_found(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/expname/S20010101/mon/ocean/var/r1i1p1') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + cmor_manager = CMORManager(self.config) + cmor_manager.create_links('20010101', 1) + mock_create_link.assert_not_called() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + def test_create_links_specs_with_grid(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/expname/S20010101/mon/ocean/var/r2i1p1/grid') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + cmor_manager = CMORManager(self.config) + cmor_manager.create_links('20010101', 1) + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + def test_create_links_primavera(self, mock_create_link): + self._configure_primavera() + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var/gn') + os.makedirs(member_path) + self.config.var_manager.tables = {'Omon': Mock()} + tempfile.mkstemp(dir=member_path) + cmor_manager = CMORManager(self.config) + cmor_manager.create_links('20010101', 1) + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + def test_create_links_with_version_primavera(self, mock_create_link): + self._configure_primavera() + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var/gn/version') + os.makedirs(member_path) + self.config.var_manager.tables = {'Omon': Mock()} + tempfile.mkstemp(dir=member_path) + cmor_manager = CMORManager(self.config) + cmor_manager.create_links('20010101', 1) + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + def test_create_links_member_not_found_primavera(self, mock_create_link): + self._configure_primavera() + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/expname/r1i1p1f1/Omon/var/gn') + os.makedirs(member_path) + self.config.var_manager.tables = {'Omon': Mock()} + tempfile.mkstemp(dir=member_path) + cmor_manager = CMORManager(self.config) + cmor_manager.create_links('20010101', 1) + mock_create_link.assert_not_called() + + def test_create_links_meteofrance(self): + self._configure_meteofrance() + cmor_manager = CMORManager(self.config) + with self.assertRaises(ValueError): + cmor_manager.create_links('20010101', 1) + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') + def test_request_chunk(self, mock_get_file_path): + mock_get_file_path.return_value = '/path/to/file' + cmor_manager = CMORManager(self.config) + datafile = cmor_manager.request_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) + self.assertEqual(datafile.remote_file, '/path/to/file') + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') + def test_declare_chunk(self, mock_get_file_path): + mock_get_file_path.return_value = '/path/to/file' + cmor_manager = CMORManager(self.config) + datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) + self.assertEqual(datafile.remote_file, '/path/to/file') + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') + def test_request_year(self, mock_get_file_path): + self.config.experiment.get_year_chunks.return_value = (1, 2) + mock_get_file_path.return_value = '/path/to/file' + mock_diagnostic = Mock() + cmor_manager = CMORManager(self.config) + datafile = cmor_manager.request_year(mock_diagnostic, ModelingRealms.ocean, 'var', '20010101', 1, 2000) + self.assertEqual(datafile.remote_file, '/path/to/file') + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') + def test_declare_year(self, mock_get_file_path): + mock_get_file_path.return_value = '/path/to/file' + cmor_manager = CMORManager(self.config) + datafile = cmor_manager.declare_year(ModelingRealms.ocean, 'var', '20010101', 1, 2001) + self.assertEqual(datafile.remote_file, '/path/to/file') + + + + + -- GitLab From 4e89d688ce94e89a18db0fe0ae059c8cbd991e44 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 17:32:29 +0100 Subject: [PATCH 086/168] Updated tests for cmormanager --- earthdiagnostics/cmormanager.py | 5 +---- test/unit/test_cmormanager.py | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 7995fdee..a2cce129 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -345,7 +345,7 @@ class CMORManager(DataManager): 'H{0}'.format(chr(64 + int(startdate[4:6]))), startdate[0:4]) else: - raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) + raise ValueError('Data convention {0} not supported'.format(self.config.data_convention)) return folder_path def _get_chunk_time_bounds(self, startdate, chunk): @@ -668,9 +668,6 @@ class CMORManager(DataManager): template = 'r{0}i{1}p1f1' elif self.config.data_convention == 'meteofrance': return '{0:02d}'.format(member) - else: - raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) - return template.format(member + 1 - self.experiment.member_count_start, self.config.cmor.initialization_number) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 2fe9ae56..a631591f 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -76,6 +76,20 @@ class TestCMORManager(TestCase): self.assertEqual(cmor_manager.get_varfolder(ModelingRealms.ocean, 'var'), 'var_f6h') + def test_get_file_path_bad_convention(self): + self.config.cmor.version = 'version' + self.config.data_convention = 'bad_convention' + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + with self.assertRaises(ValueError): + cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, frequency) + def test_get_file_path_specs(self): cmor_manager = CMORManager(self.config) cmor_var = Mock() @@ -92,6 +106,23 @@ class TestCMORManager(TestCase): 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + def test_get_file_path_preface(self): + self._configure_preface() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901_198912.nc')) + def test_get_file_path_specs_version(self): self.config.cmor.version = 'version' cmor_manager = CMORManager(self.config) @@ -225,6 +256,9 @@ class TestCMORManager(TestCase): def _configure_meteofrance(self): self.config.data_convention = 'meteofrance' + def _configure_preface(self): + self.config.data_convention = 'preface' + def test_get_file_path_primavera_grid(self): self._configure_primavera() cmor_manager = CMORManager(self.config) -- GitLab From 0a67782fb0d63acb9b76510fc30c28e67dcdcfd2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 17:48:29 +0100 Subject: [PATCH 087/168] Updated config documentation --- earthdiagnostics/config.py | 159 +++++++++++++++++++++++++++++++++++-- 1 file changed, 154 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 74d0c9f1..4fa1d251 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Classes to manage Earth Diagnostics configuration""" import os import six @@ -13,6 +14,7 @@ from earthdiagnostics.variable import VariableManager class ConfigException(Exception): + """Exception raised when there is a problem with the configuration""" pass @@ -20,8 +22,10 @@ class Config(object): """ Class to read and manage the configuration - :param path: path to the conf file - :type path: str + Parameters + ---------- + path: str + Path to the configuration file """ def __init__(self, path): @@ -137,6 +141,14 @@ class Config(object): class CMORConfig(object): + """ + Configuration for the cmorization processes + + Parameters + ---------- + parser: ConfigParser + var_manager: VariableManager + """ def __init__(self, parser, var_manager): self.force = parser.get_bool_option('CMOR', 'FORCE', False) @@ -210,6 +222,18 @@ class CMORConfig(object): return False def chunk_cmorization_requested(self, chunk): + """ + Check if the cmorization of a given chunk is required + + Parameters + ---------- + chunk: int + + Returns + ------- + bool + + """ if len(self._chunks) == 0: return True return chunk in self._chunks @@ -241,6 +265,23 @@ class CMORConfig(object): return range(start, end, step) def get_variables(self, frequency): + """ + Get the variables to get from the grib file for a given frequency + + Parameters + ---------- + frequency: Frequency + + Returns + ------- + str + + Raises + ------ + ValueError + If the frequency passed is not supported + + """ if frequency in (Frequencies.three_hourly, Frequencies.six_hourly): return self._var_hourly elif frequency == Frequencies.daily: @@ -250,23 +291,55 @@ class CMORConfig(object): raise ValueError('Frequency not recognized: {0}'.format(frequency)) def get_requested_codes(self): + """ + Get all the codes to be extracted from the grib files + + Returns + ------- + set of int + + """ return set(list(self._var_hourly.keys()) + list(self._var_daily.keys()) + list(self._var_monthly.keys())) def get_levels(self, frequency, variable): + """ + Get the levels to extract for a given variable + + Parameters + ---------- + frequency: Frequency + variable: str + + Returns + ------- + iterable of int + + """ return self.get_variables(frequency)[variable] class THREDDSConfig(object): + """ + Configuration related to the THREDDS server + + Parameters + ---------- + parser: ConfigParser + + """ + def __init__(self, parser): self.server_url = parser.get_option('THREDDS', 'SERVER_URL', '') class ExperimentConfig(object): """ - Encapsulates all chunk related tasks + Configuration related to the experiment + + Parameters + ---------- + parser: ConfigParser - :param parser: parser for the config file - :type parser: Parser """ def __init__(self, parser): @@ -395,18 +468,86 @@ class ExperimentConfig(object): return chunks def get_chunk_start(self, startdate, chunk): + """ + Get chunk's first day + + Parameters + ---------- + startdate: str or datetime.datetime + chunk: int + + Returns + ------- + datetime.datetime + + See Also + -------- + get_chunk_start_str + + """ # noinspection PyTypeChecker if isinstance(startdate, six.string_types): startdate = parse_date(startdate) return chunk_start_date(startdate, chunk, self.chunk_size, 'month', self.calendar) def get_chunk_start_str(self, startdate, chunk): + """ + Get chunk's first day string representation + + Parameters + ---------- + startdate: str or datetime.datetime + chunk: int + + Returns + ------- + str + + See Also + -------- + get_chunk_start + + """ return date2str(self.get_chunk_start(startdate, chunk)) def get_chunk_end(self, startdate, chunk): + """ + Get chunk's last day + + Parameters + ---------- + startdate: str or datetime.datetime + chunk: int + + Returns + ------- + datetime.datetime + + See Also + -------- + get_chunk_end_str + + """ return chunk_end_date(self.get_chunk_start(startdate, chunk), self.chunk_size, 'month', self.calendar) def get_chunk_end_str(self, startdate, chunk): + """ + Get chunk's last day as a string + + Parameters + ---------- + startdate: str or datetime.datetime + chunk: int + + Returns + ------- + datetime.datetime + + See Also + -------- + get_chunk_end + + """ return date2str(self.get_chunk_end(startdate, chunk)) def get_full_years(self, startdate): @@ -446,6 +587,14 @@ class ExperimentConfig(object): class ReportConfig(object): + """ + Configuration for the reporting feature + + Parameters + ---------- + parser: ConfigParser + + """ def __init__(self, parser): self.maximum_priority = parser.get_int_option('REPORT', 'MAXIMUM_PRIORITY', 10) self.path = parser.get_path_option('REPORT', 'PATH', '') -- GitLab From 518578393c4a6f83869ea7846cb47cc6b7fd9a0c Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 17:57:11 +0100 Subject: [PATCH 088/168] Updated config in variable --- earthdiagnostics/variable.py | 68 +++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index e1ba8b78..4ff58418 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Classes to manage variable definitions and aliases""" import csv import glob import json @@ -13,10 +14,16 @@ from earthdiagnostics.modelingrealm import ModelingRealms class VariableJsonException(Exception): + """Exception to be raised when an error related to the json reading is encountered""" pass class VariableManager(object): + """ + Class for translating variable alias into standard names and provide the correct description for them + + + """ def __init__(self): self._cmor_tables_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cmor_tables') self._aliases_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'variable_alias') @@ -25,7 +32,7 @@ class VariableManager(object): def get_variable(self, original_name, silent=False): """ - Returns the cmor variable instance given a variable name + Return the cmor variable instance given a variable name :param original_name: original variable's name :type original_name: str @@ -43,7 +50,7 @@ class VariableManager(object): def get_all_variables(self): """ - Returns all variables + Return all variables :return: CMOR variable list :rtype: set[Variable] @@ -53,7 +60,7 @@ class VariableManager(object): def get_variable_and_alias(self, original_name, silent=False): """ - Returns the cmor variable instance given a variable name + Return the cmor variable instance given a variable name :param original_name: original variable's name :type original_name: str @@ -71,7 +78,12 @@ class VariableManager(object): def load_variables(self, table_name): """ - Loads the CMOR csv and creates the variables dictionary + Loads the CMOR csv and creates the variables dictionary + + Parameters + ---------- + table_name: str + """ self._dict_variables = dict() self._load_variable_list(table_name) @@ -293,7 +305,9 @@ class VariableManager(object): class Variable(object): """ - Class to characterize a CMOR variable. It also contains the static method to make the match between thje original + Class to characterize a CMOR variable. + + It also contains the static method to make the match between the original name and the standard name. Requires data _convetion to be available in cmor_tables to work. """ @@ -364,6 +378,14 @@ class Variable(object): return ModelingRealms.parse(domains[0]) def parse_csv(self, var_line): + """ + Fill the object information from a csv line + + Parameters + ---------- + var_line: list of str + + """ self.short_name = var_line[1].strip() self.standard_name = var_line[2].strip() self.long_name = var_line[3].strip() @@ -375,6 +397,26 @@ class Variable(object): self.grid = var_line[9].strip() def get_table(self, frequency, data_convention): + """ + Get a table object given the frequency and data_covention + + If the variable does not contain the table information, it uses the domain to make a guess + + Parameters + ---------- + frequency: Frequency + data_convention: str + + Returns + ------- + CMORTable + + Raises + ------ + ValueError + If a table can not be deduced from the given parameters + + """ for table, _ in self.tables: if table.frequency == frequency: return table @@ -405,6 +447,10 @@ class VariableAlias(object): """ Class to characterize a CMOR variable. It also contains the static method to make the match between thje original name and the standard name. Requires data _convetion to be available in cmor_tables to work. + + Parameters + ---------- + alias: str """ def __init__(self, alias): @@ -422,6 +468,15 @@ class VariableAlias(object): class CMORTable(object): + """ + Class to represent a CMOR table + + Parameters + ---------- + name: str + frequency: Frequency + date: str + """ def __init__(self, name, frequency, date): self.name = name self.frequency = Frequency.parse(frequency) @@ -435,6 +490,3 @@ class CMORTable(object): def __lt__(self, other): return self.name < other.name - - - -- GitLab From f1a7e58de5697f63adea1ac35b76bded2ed3fa9e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 18:04:35 +0100 Subject: [PATCH 089/168] Updated utils doc --- earthdiagnostics/config.py | 2 ++ earthdiagnostics/utils.py | 72 ++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 4fa1d251..94fa627d 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -15,6 +15,7 @@ from earthdiagnostics.variable import VariableManager class ConfigException(Exception): """Exception raised when there is a problem with the configuration""" + pass @@ -595,6 +596,7 @@ class ReportConfig(object): parser: ConfigParser """ + def __init__(self, parser): self.maximum_priority = parser.get_int_option('REPORT', 'MAXIMUM_PRIORITY', 10) self.path = parser.get_path_option('REPORT', 'PATH', '') diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index cbbf6f11..5b343d38 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -1,7 +1,5 @@ # coding=utf-8 -""" -Common utilities for multiple topics that are not big enough to have their own module -""" +"""Common utilities for multiple topics that are not big enough to have their own module""" import datetime import os import re @@ -29,7 +27,7 @@ from earthdiagnostics.constants import Basins @contextmanager def suppress_stdout(): - """Redirects the standard output to devnull""" + """Redirect the standard output to devnull""" with open(os.devnull, "w") as devnull: old_stdout = sys.stdout sys.stdout = devnull @@ -387,7 +385,7 @@ class Utils(object): @staticmethod def copy_tree(source, destiny): """ - Copies a full tree to a new location + Copy a full tree to a new location Parameters ---------- @@ -432,7 +430,6 @@ class Utils(object): """ Get the xxHash hash for a given file - Parameters ---------- filepath: str @@ -441,7 +438,6 @@ class Utils(object): save: bool, optional If True, saves the hash to a file """ - if use_stored: hash_file = Utils._get_hash_filename(filepath) if os.path.isfile(hash_file): @@ -716,12 +712,14 @@ class Utils(object): def concat_variables(source, destiny, remove_source=False): """ Add variables from a nc file to another - :param source: path to source file - :type source: str - :param destiny: path to destiny file - :type destiny: str - :param remove_source: if True, removes source file - :type remove_source: bool + + Parameters + ---------- + source: str + destiny: str + remove_source: bool + if True, removes source file + """ if os.path.exists(destiny): handler_total = Utils.open_cdf(destiny) @@ -751,9 +749,8 @@ class Utils(object): Utils.convert2netcdf4(destiny) class ExecutionError(Exception): - """ - Exception to raise when a command execution fails - """ + """Exception to raise when a command execution fails""" + pass @classmethod @@ -769,9 +766,12 @@ class Utils(object): @staticmethod def create_folder_tree(path): """ - Createas a fodle path will and parent directories if needed. - :param path: folder's path - :type path: str + Create a folder path with all parent directories if needed. + + Parameters + ---------- + path: str + """ if not os.path.exists(path): # noinspection PyBroadException @@ -824,10 +824,12 @@ class Utils(object): def untar(files, destiny_path): """ Untar files to a given destiny - :param files: files to unzip - :type files: list[Any] | Tuple[Any] - :param destiny_path: path to destination folder - :type destiny_path: str + + Parameters + ---------- + files: iterable of str + destiny_path: str + """ for filepath in files: Log.debug('Unpacking {0}', filepath) @@ -846,10 +848,11 @@ class Utils(object): def unzip(files, force=False): """ Unzip a list of files - :param files: files to unzip - :type files: list | str - :param force: if True, it will overwrite unzipped files - :type force: bool + + files: str or iterable of str + force: bool, optional + if True, it will overwrite unzipped files + """ # noinspection PyTypeChecker if isinstance(files, six.string_types): @@ -866,16 +869,13 @@ class Utils(object): raise Utils.UnzipException('Can not unzip {0}: {1}'.format(filepath, ex)) class UnzipException(Exception): - """ - Excpetion raised when unzip fails - """ + """Exception raised when unzip fails""" + pass class TempFile(object): - """ - Class to manage temporal files - """ + """Class to manage temporal files""" autoclean = True """ @@ -897,7 +897,7 @@ class TempFile(object): @staticmethod def get(filename=None, clean=None, suffix='.nc'): """ - Gets a new temporal filename, storing it for automated cleaning + Get a new temporal filename, storing it for automated cleaning :param suffix: :param filename: if it is not none, the function will use this filename instead of a random one @@ -924,9 +924,7 @@ class TempFile(object): @staticmethod def clean(): - """ - Removes all temporary files created with Tempfile until now - """ + """Remove all temporary files created with Tempfile until now""" for temp_file in TempFile.files: if os.path.exists(temp_file): os.remove(temp_file) -- GitLab From c5a100d57ab657980cf31ea115a676da6382bcb6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 13 Dec 2017 18:06:33 +0100 Subject: [PATCH 090/168] Updated variable doc --- earthdiagnostics/variable.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 4ff58418..fe5e0c57 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -15,15 +15,13 @@ from earthdiagnostics.modelingrealm import ModelingRealms class VariableJsonException(Exception): """Exception to be raised when an error related to the json reading is encountered""" + pass class VariableManager(object): - """ - Class for translating variable alias into standard names and provide the correct description for them - + """Class for translating variable alias into standard names and provide the correct description for them""" - """ def __init__(self): self._cmor_tables_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cmor_tables') self._aliases_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'variable_alias') @@ -477,6 +475,7 @@ class CMORTable(object): frequency: Frequency date: str """ + def __init__(self, name, frequency, date): self.name = name self.frequency = Frequency.parse(frequency) -- GitLab From 42b866722402a0449d247572252f660d9866cb72 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 10:28:57 +0100 Subject: [PATCH 091/168] Updated cmorizer doc --- earthdiagnostics/cmorizer.py | 102 ++++++++++++++++++----------------- setup.py | 4 +- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index b66f705a..2abe701b 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Cmorization classes""" import glob import os import shutil @@ -19,12 +20,11 @@ class Cmorizer(object): """ Class to manage CMORization - :param data_manager: experiment's data manager - :type data_manager: CMORManager - :param startdate: startdate to cmorize - :type startdate: str - :param member: member to cmorize - :type member: int + Parameters + ---------- + data_manager: DataManager + startdate: str + member: int """ @@ -64,10 +64,7 @@ class Cmorizer(object): 'tbnds': 'bnds', 'nav_lat': self.lat_name, 'nav_lon': self.lon_name, 'x': 'i', 'y': 'j'} def cmorize_ocean(self): - """ - CMORizes ocean files from MMO files - :return: - """ + """Cmorize ocean files from MMO files""" if not self.cmor.ocean: Log.info('Skipping ocean cmorization due to configuration') return @@ -87,9 +84,9 @@ class Cmorizer(object): count = 1 for tarfile in tar_files: - if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), (ModelingRealms.ocean, - ModelingRealms.seaIce, - ModelingRealms.ocnBgchem)): + if not self._cmorization_required(self._get_chunk(os.path.basename(tarfile)), (ModelingRealms.ocean, + ModelingRealms.seaIce, + ModelingRealms.ocnBgchem)): Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) count += 1 continue @@ -189,16 +186,13 @@ class Cmorizer(object): shutil.move(merged_sh, os.path.join(self.cmor_scratch, 'MMASH_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) def cmorize_atmos(self): - """ - CMORizes atmospheric data, from grib or MMA files - :return: - """ + """CMORizes atmospheric data, from grib or MMA files""" if not self.cmor.atmosphere: Log.info('Skipping atmosphere cmorization due to configuration') return Log.info('\nCMORizing atmosphere\n') - if self.cmor.use_grib and self.gribfiles_available(): + if self.cmor.use_grib and self._gribfiles_available(): self._cmorize_grib_files() else: self._cmorize_mma_files() @@ -210,7 +204,7 @@ class Cmorizer(object): if len(tar_files) == 0: Log.error('MMA files not found in {0}'.format(self.original_files_path)) for tarfile in tar_files: - if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), (ModelingRealms.atmos,)): + if not self._cmorization_required(self._get_chunk(os.path.basename(tarfile)), (ModelingRealms.atmos,)): Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) count += 1 continue @@ -230,10 +224,10 @@ class Cmorizer(object): chunk = 1 chunk_start = parse_date(self.startdate) - while os.path.exists(self.get_original_grib_path(chunk_start, 'GG')) or \ - os.path.exists(self.get_original_grib_path(chunk_start, 'SH')): + while os.path.exists(self._get_original_grib_path(chunk_start, 'GG')) or \ + os.path.exists(self._get_original_grib_path(chunk_start, 'SH')): - if self.cmorization_required(chunk, (ModelingRealms.atmos,)): + if self._cmorization_required(chunk, (ModelingRealms.atmos,)): chunk_end = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', self.experiment.calendar) chunk_end = previous_day(chunk_end, self.experiment.calendar) Log.info('CMORizing chunk {0}-{1}', date2str(chunk_start), date2str(chunk_end)) @@ -241,21 +235,21 @@ class Cmorizer(object): for grid in ('SH', 'GG'): Log.info('Processing {0} variables', grid) - if not os.path.exists(self.get_original_grib_path(chunk_start, grid)): + if not os.path.exists(self._get_original_grib_path(chunk_start, grid)): continue - self.cmorize_grib_file(chunk_end, chunk_start, grid) + self._cmorize_grib_file(chunk_end, chunk_start, grid) except Exception as ex: Log.error('Can not cmorize GRIB file for chunk {0}-{1}: {2}', date2str(chunk_start), date2str(chunk_end), ex) chunk_start = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', self.experiment.calendar) chunk += 1 - def cmorize_grib_file(self, chunk_end, chunk_start, grid): + def _cmorize_grib_file(self, chunk_end, chunk_start, grid): for month in range(0, self.experiment.chunk_size): current_date = add_months(chunk_start, month, self.experiment.calendar) - original_gribfile = self.get_original_grib_path(current_date, grid) + original_gribfile = self._get_original_grib_path(current_date, grid) Log.info('Processing month {1}', grid, date2str(current_date)) - gribfile = self.get_scratch_grib_path(current_date, grid) + gribfile = self._get_scratch_grib_path(current_date, grid) if not os.path.isfile(gribfile): Log.info('Copying file...', grid, date2str(current_date)) Utils.copy_file(original_gribfile, gribfile) @@ -266,7 +260,7 @@ class Cmorizer(object): os.remove(gribfile) return - next_gribfile = self.get_original_grib_path(add_months(current_date, 1, self.experiment.calendar), grid) + next_gribfile = self._get_original_grib_path(add_months(current_date, 1, self.experiment.calendar), grid) if not os.path.exists(next_gribfile): os.remove(gribfile) @@ -317,7 +311,7 @@ class Cmorizer(object): Utils.remove_file('ICM') def _get_monthly_grib(self, current_date, gribfile, grid): - prev_gribfile = self.get_scratch_grib_path(add_months(current_date, -1, self.experiment.calendar), grid) + prev_gribfile = self._get_scratch_grib_path(add_months(current_date, -1, self.experiment.calendar), grid) if os.path.exists(prev_gribfile): self._merge_grib_files(current_date, prev_gribfile, gribfile) full_file = 'ICM' @@ -325,14 +319,14 @@ class Cmorizer(object): full_file = gribfile return full_file - def get_scratch_grib_path(self, current_date, grid): + def _get_scratch_grib_path(self, current_date, grid): return os.path.join(self.config.scratch_dir, self._get_grib_filename(grid, current_date)) def _obtain_atmos_timestep(self, gribfile): if self.atmos_timestep is None: self.atmos_timestep = self._get_atmos_timestep(gribfile) - def get_original_grib_path(self, current_date, grid): + def _get_original_grib_path(self, current_date, grid): return os.path.join(self.original_files_path, self._get_grib_filename(grid, current_date)) @@ -369,7 +363,7 @@ class Cmorizer(object): frequency = self._get_nc_file_frequency(filename) Utils.rename_variables(filename, self.alt_coord_names, False, True) handler = Utils.open_cdf(filename) - self._remove_valid_limits(handler) + Cmorizer._remove_valid_limits(handler) self._add_common_attributes(handler, frequency) self._update_time_variables(handler) handler.sync() @@ -386,7 +380,8 @@ class Cmorizer(object): handler.close() os.remove(filename) - def _remove_valid_limits(self, filename): + @staticmethod + def _remove_valid_limits(filename): handler = Utils.open_cdf(filename) for variable in handler.variables.keys(): var = handler.variables[variable] @@ -416,16 +411,20 @@ class Cmorizer(object): def extract_variable(self, file_path, handler, frequency, variable): """ - Extracts a variable from a file and creates the CMOR file - - :param file_path: path to the file - :type file_path: str - :param handler: netCDF4 handler for the file - :type handler: netCDF4.Dataset - :param frequency: variable's frequency - :type frequency: Frequency - :param variable: variable's name - :type variable: str + Extract a variable from a file and creates the CMOR file + + Parameters + ---------- + file_path:str + handler: netCDF4.Dataset + frequency: Frequency + variable: str + + Raises + ------ + CMORException + If the filename does not match any of the recognized patterns + """ alias, var_cmor = self.config.var_manager.get_variable_and_alias(variable) if var_cmor is None: @@ -445,7 +444,7 @@ class Cmorizer(object): else: region = alias.basin.name - date_str = self.get_date_str(file_path) + date_str = self._get_date_str(file_path) if date_str is None: Log.error('Variable {0} can not be cmorized. Original filename does not match a recognized pattern', var_cmor.short_name) @@ -477,9 +476,10 @@ class Cmorizer(object): region_str = '' Log.info('Variable {0.domain}:{0.short_name} processed{1}', var_cmor, region_str) - def get_date_str(self, file_path): + def _get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') - if file_parts[0] in (self.experiment.expid, 't00o', 'MMA', 'MMASH', 'MMAGG', 'MMO') or file_parts[0].startswith('ORCA'): + if file_parts[0] in (self.experiment.expid, 't00o', 'MMA', 'MMASH', 'MMAGG', 'MMO') or \ + file_parts[0].startswith('ORCA'): # Model output if file_parts[-1].endswith('.tar'): file_parts = file_parts[-1][0:-4].split('-') @@ -492,8 +492,8 @@ class Cmorizer(object): else: return None - def get_chunk(self, file_path): - chunk_start = parse_date(self.get_date_str(file_path).split('-')[0]) + def _get_chunk(self, file_path): + chunk_start = parse_date(self._get_date_str(file_path).split('-')[0]) current_date = parse_date(self.startdate) chunk = 1 while current_date < chunk_start: @@ -683,12 +683,12 @@ class Cmorizer(object): handler.title = "{0} model output prepared for {2} {1}".format(experiment.model, experiment.experiment_name, self.config.data_convention.upper()) - def gribfiles_available(self): + def _gribfiles_available(self): grb_path = os.path.join(self.original_files_path, '*.grb') gribfiles = glob.glob(grb_path) return len(gribfiles) > 0 - def cmorization_required(self, chunk, domains): + def _cmorization_required(self, chunk, domains): if not self.config.cmor.chunk_cmorization_requested(chunk): return False if self.config.cmor.force: @@ -700,4 +700,6 @@ class Cmorizer(object): class CMORException(Exception): + """Exception to be launched when an error is encountered during cmorization""" + pass diff --git a/setup.py b/setup.py index a9ff71b6..81ea7ac8 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # coding=utf-8 -""" -Installation script for EarthDiagnostics package -""" +"""Installation script for EarthDiagnostics package""" from os import path -- GitLab From f0f1eaa0f9f59c1607216fd1b05a2cb5c379e350 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 11:05:08 +0100 Subject: [PATCH 092/168] Updated THREDDS manager doc --- earthdiagnostics/cmorizer.py | 2 +- earthdiagnostics/threddsmanager.py | 172 ++++++++++++----------------- 2 files changed, 73 insertions(+), 101 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 2abe701b..a196063f 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -479,7 +479,7 @@ class Cmorizer(object): def _get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') if file_parts[0] in (self.experiment.expid, 't00o', 'MMA', 'MMASH', 'MMAGG', 'MMO') or \ - file_parts[0].startswith('ORCA'): + file_parts[0].startswith('ORCA'): # Model output if file_parts[-1].endswith('.tar'): file_parts = file_parts[-1][0:-4].split('-') diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 331fd55a..e3b81029 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Data manager for THREDDS server""" import os from datetime import datetime from time import strptime @@ -6,7 +7,7 @@ from time import strptime import iris import netCDF4 import numpy as np -from bscearth.utils.date import parse_date, add_months, chunk_start_date, chunk_end_date +from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date from bscearth.utils.log import Log from cf_units import Unit from iris.coords import DimCoord @@ -19,7 +20,11 @@ from earthdiagnostics.variable_type import VariableType class THREDDSManager(DataManager): """ - Data manager class for CMORized experiments + Data manager class for THREDDS + + Parameters + ---------- + config: Config """ def __init__(self, config): super(THREDDSManager, self).__init__(config) @@ -38,52 +43,28 @@ class THREDDSManager(DataManager): if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size != 1: raise Exception('For obs and recon data chunk_size must be always 1') - # noinspection PyUnusedLocal - def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, vartype=VariableType.MEAN): - - aggregation_path = self.get_var_url(variable, startdate, frequency, None, vartype) - startdate = parse_date(startdate) - start_chunk = chunk_start_date(startdate, self.experiment.num_chunks, self.experiment.chunk_size, - 'month', self.experiment.calendar) - end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) - - thredds_subset = THREDDSSubset(aggregation_path, "", variable, startdate, end_chunk) - selected_months = ','.join([str(add_months(startdate, i, self.experiment.calendar).month) for i in leadtimes]) - temp = TempFile.get() - if self.config.data_type == 'exp': - select_months = '-selmonth,{0} {1}'.format(selected_months, thredds_subset) - selected_years = ','.join([str(add_months(startdate, i, self.experiment.calendar).year) for i in leadtimes]) - Utils.cdo.selyear(selected_years, input=select_months, output=temp) - else: - Utils.cdo.selmonth(selected_months, input=thredds_subset, output=temp) - return temp - # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN): - """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + """" + Check if a file exists in the storage + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None + box: Box or None + frequency: Frequency or None + vartype: VariableType + + Returns + ------- + bool - :param domain: CMOR domain - :type domain: str - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str """ aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) @@ -97,23 +78,22 @@ class THREDDSManager(DataManager): def get_file_path(self, startdate, domain, var, frequency, vartype, box=None, grid=None): """ - Returns the path to a concrete file - :param startdate: file's startdate - :type startdate: str - :param domain: file's domain - :type domain: str - :param var: file's var - :type var: str - :param frequency: file's frequency - :type frequency: Frequency - :param box: file's box - :type box: Box - :param grid: file's grid - :type grid: str - :return: path to the file - :rtype: str - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType + Return the path to a concrete file + + Parameters + ---------- + startdate: str + domain: ModelingRealm + var: str + frequency: Frequency + vartype: VariableType + box: Box or None, optional + grid: str or None, optional + + Returns + ------- + str + """ if frequency is None: frequency = self.config.frequency @@ -143,6 +123,7 @@ class THREDDSManager(DataManager): def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN): """ Ge a file containing all the data for one year for one variable + :param domain: variable's domain :type domain: str :param var: variable's name @@ -162,12 +143,13 @@ class THREDDSManager(DataManager): :return: """ aggregation_path = self.get_var_url(var, startdate, None, box, vartype) - thredds_subset = THREDDSSubset(aggregation_path, "", var, datetime(year, 1, 1), datetime(year+1, 1, 1)) + thredds_subset = THREDDSSubset(aggregation_path, "", var, datetime(year, 1, 1), datetime(year + 1, 1, 1)) return thredds_subset.download() def get_var_url(self, var, startdate, frequency, box, vartype): """ Get url for dataset + :param var: variable to retrieve :type var: str :param startdate: startdate to retrieve @@ -199,44 +181,10 @@ class THREDDSManager(DataManager): else: return '{0}.nc'.format(var) - def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, - frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): - """ - Creates the link of a given file from the CMOR repository. - - :param cmor_var: - :param move_old: - :param date_str: - :param year: if frequency is yearly, this parameter is used to give the corresponding year - :type year: int - :param domain: CMOR domain - :type domain: str - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str - :param cmor_var: variable instance describing the selected variable - :type cmor_var: Variable - """ - # THREDDSManager does not require links - pass - def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + Copy a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy :param vartype: :param domain: CMOR domain @@ -274,7 +222,7 @@ class THREDDSManager(DataManager): def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, vartype=VariableType.MEAN, diagnostic=None): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + Copy a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy :param diagnostic: :param region: @@ -319,10 +267,25 @@ class THREDDSManager(DataManager): class THREDDSError(Exception): + """Exception to be launched when a THREDDS related error is encounteredd""" + pass class THREDDSSubset(DataFile): + """ + Implementation of DataFile for the THREDDS server + + Parameters + ---------- + thredds_path: str + file_path: str + var: str + start_time: datetime + end_time: datetime + + """ + def __init__(self, thredds_path, file_path, var, start_time, end_time): """ @@ -351,6 +314,15 @@ class THREDDSSubset(DataFile): return 'THREDDS {0.thredds_path} ({0.start_time}-{0.end_time})'.format(self) def download(self): + """ + Get data from the THREDDS server + + Raises + ------ + THREDDSError + If the data can not be downloaded + + """ try: Log.debug('Downloading thredds subset {0}...', self) iris.FUTURE.netcdf_promote = True @@ -363,10 +335,10 @@ class THREDDSSubset(DataFile): self.local_file = TempFile.get() iris.save(var_cube, self.local_file, zlib=True) if not Utils.check_netcdf_file(self.local_file): - raise Exception('netcdf check for downloaded file failed') + raise THREDDSError('netcdf check for downloaded file failed') Log.info('Request {0} ready!', self) self.local_status = LocalStatus.READY - except Exception as ex: + except THREDDSError as ex: Log.error('Can not retrieve {0} from server: {1}'.format(self, ex)) self.local_status = LocalStatus.FAILED -- GitLab From 514b336e08e91f6c8d11f7d9f2f0bcb81621566a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 11:21:08 +0100 Subject: [PATCH 093/168] Updated doc --- earthdiagnostics/general/attribute.py | 22 +++++----- earthdiagnostics/general/dailymean.py | 6 +-- earthdiagnostics/general/module.py | 6 +-- earthdiagnostics/general/monthlymean.py | 6 +-- earthdiagnostics/general/relink.py | 6 +-- earthdiagnostics/general/relinkall.py | 6 +-- earthdiagnostics/general/rewrite.py | 6 +-- earthdiagnostics/general/scale.py | 6 +-- earthdiagnostics/general/select_levels.py | 6 +-- .../general/simplify_dimensions.py | 6 +-- .../general/verticalmeanmetersiris.py | 6 +-- earthdiagnostics/general/yearlymean.py | 6 +-- earthdiagnostics/ocean/areamoc.py | 6 +-- earthdiagnostics/ocean/averagesection.py | 6 +-- earthdiagnostics/ocean/convectionsites.py | 6 +-- earthdiagnostics/ocean/cutsection.py | 6 +-- earthdiagnostics/ocean/gyres.py | 6 +-- earthdiagnostics/ocean/heatcontent.py | 6 +-- earthdiagnostics/ocean/heatcontentlayer.py | 6 +-- earthdiagnostics/ocean/interpolate.py | 6 +-- earthdiagnostics/ocean/interpolatecdo.py | 42 ++++++++++++++----- earthdiagnostics/ocean/mask_land.py | 6 +-- earthdiagnostics/ocean/maxmoc.py | 7 ++-- .../ocean/mixedlayerheatcontent.py | 6 +-- .../ocean/mixedlayersaltcontent.py | 6 +-- earthdiagnostics/ocean/moc.py | 6 +-- earthdiagnostics/ocean/mxl.py | 6 +-- earthdiagnostics/ocean/psi.py | 6 +-- earthdiagnostics/ocean/regionmean.py | 6 +-- earthdiagnostics/ocean/regionsum.py | 6 +-- earthdiagnostics/ocean/rotation.py | 6 +-- earthdiagnostics/ocean/siasiesiv.py | 6 +-- earthdiagnostics/ocean/verticalgradient.py | 6 +-- earthdiagnostics/ocean/verticalmean.py | 6 +-- earthdiagnostics/ocean/verticalmeanmeters.py | 6 +-- .../statistics/climatologicalpercentile.py | 6 +-- .../statistics/daysoverpercentile.py | 6 +-- earthdiagnostics/statistics/discretize.py | 6 +-- .../statistics/monthlypercentile.py | 6 +-- 39 files changed, 153 insertions(+), 134 deletions(-) diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 1d460ac2..50b1cf8c 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -6,7 +6,7 @@ from earthdiagnostics.utils import Utils class Attribute(Diagnostic): """ - Rewrites files without doing any calculations. + Rewrite files without doing any calculations. Can be useful to convert units or to correct wrong metadata :original author: Javier Vegas-Regidor @@ -55,14 +55,14 @@ class Attribute(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Creates a job for each chunk to compute the diagnostic - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, grid - :type options: list[str] - :return: - """ + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ options_available = (DiagnosticDomainOption(), DiagnosticVariableOption(diags.data_manager.config.var_manager), @@ -78,17 +78,17 @@ class Attribute(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid, to_modify=True) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" variable_file = self.variable_file.local_file handler = Utils.open_cdf(variable_file) handler.setncattr(self.attributte_name, self.attributte_value) diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py index d2737b5c..e0db5c14 100644 --- a/earthdiagnostics/general/dailymean.py +++ b/earthdiagnostics/general/dailymean.py @@ -82,17 +82,17 @@ class DailyMean(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=self.frequency, grid=self.grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.daymean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=Frequencies.daily, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() handler = Utils.open_cdf(self.variable_file.local_file) if 'region' in handler.variables: diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index 17b01d67..5257cbd3 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -79,19 +79,19 @@ class Module(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.component_u_file = self.request_chunk(self.domain, self.componentu, self.startdate, self.member, self.chunk, grid=self.grid) self.component_v_file = self.request_chunk(self.domain, self.componentv, self.startdate, self.member, self.chunk, grid=self.grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.module_file = self.declare_chunk(self.domain, self.module, self.startdate, self.member, self.chunk, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() Utils.copy_file(self.component_u_file.local_file, temp) component_u = Utils.open_cdf(temp) diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 55caf8c8..e7fa5c4e 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -81,17 +81,17 @@ class MonthlyMean(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=self.frequency, grid=self.grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.monmean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=Frequencies.monthly, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" handler = Utils.open_cdf(self.variable_file.local_file) temp = TempFile.get() if 'region' in handler.variables: diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index b2545e1b..8e0eddf2 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -73,15 +73,15 @@ class Relink(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" pass def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" pass def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" self.data_manager.link_file(self.domain, self.variable, self.var_manager.get_variable(self.variable), self.startdate, self.member, self.chunk, move_old=self.move_old, grid=self.grid) diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index d5fffc4a..56d1eb2e 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -48,14 +48,14 @@ class RelinkAll(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" pass def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" pass def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" self.data_manager.create_links(self.startdate) diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index 7d38c0ae..19ecfd5e 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -68,16 +68,16 @@ class Rewrite(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid, to_modify=True) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" self.corrected.set_local_file(self.variable_file.local_file, self) diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 1d1d918a..40318ec2 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -95,17 +95,17 @@ class Scale(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid, frequency=self.frequency, to_modify=True) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid, frequency=self.frequency) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" variable_file = self.variable_file.local_file handler = Utils.open_cdf(variable_file) diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index 69931b87..f1069436 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -81,17 +81,17 @@ class SelectLevels(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid, to_modify=True) def declare_data_generated(self): + """Request data required by the diagnostic""" self.result = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() Utils.nco.ncks(input=self.variable_file, output=temp, diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index 4c83e376..27d1a512 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -82,17 +82,17 @@ class SimplifyDimensions(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid, to_modify=True) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.simplified = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" handler = Utils.open_cdf(self.variable_file.local_file) if 'i' not in handler.dimensions: raise Exception('Variable {0.domain}:{0.variable} does not have i,j dimensions'.format(self)) diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py index 98db4d86..1bc2dfe6 100644 --- a/earthdiagnostics/general/verticalmeanmetersiris.py +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -86,17 +86,17 @@ class VerticalMeanMetersIris(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.results = self.declare_chunk(self.domain, self.variable + 'vmean', self.startdate, self.member, self.chunk, box=self.box) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" iris.FUTURE.netcdf_no_unlimited = True iris.FUTURE.netcdf_promote = True diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py index 2f3c96d4..964c570b 100644 --- a/earthdiagnostics/general/yearlymean.py +++ b/earthdiagnostics/general/yearlymean.py @@ -82,17 +82,17 @@ class YearlyMean(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=self.frequency, grid=self.grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.yearmean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=Frequencies.yearly, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() handler = Utils.open_cdf(self.variable_file.local_file) diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index c96def76..23e69e17 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -90,18 +90,18 @@ class AreaMoc(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(ModelingRealms.ocean, AreaMoc.vsftmyz, self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.results = self.declare_chunk(ModelingRealms.ocean, AreaMoc.vsftmyz, self.startdate, self.member, self.chunk, box=self.box) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" nco = Utils.nco cdo = Utils.cdo diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 85f98ffa..40ce55f8 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -86,17 +86,17 @@ class AverageSection(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.mean = self.declare_chunk(self.domain, self.variable + 'mean', self.startdate, self.member, self.chunk, box=self.box, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() variable_file = self.variable_file.local_file Utils.cdo.zonmean(input='-sellonlatbox,{0},{1},{2},{3} {4}'.format(self.box.min_lon, self.box.max_lon, diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index 9d9eff99..dc5cad88 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -67,15 +67,15 @@ class ConvectionSites(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.mixed_layer = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.sites = self.declare_chunk(ModelingRealms.ocean, 'site', self.startdate, self.member, self.chunk) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" if self.model_version in [Models.ECEARTH_2_3_O1L42, Models.ECEARTH_3_0_O1L46, Models.NEMO_3_2_O1L42, Models.NEMO_3_3_O1L46, Models.NEMOVAR_O1L42]: diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index d5e03355..f2e9bb8c 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -92,16 +92,16 @@ class CutSection(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.section = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, box=self.box) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" nco = Utils.nco handler = Utils.open_cdf('mesh_hgr.nc') diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index abca9e55..377e630e 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -69,16 +69,16 @@ class Gyres(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.vsftbarot = self.request_chunk(ModelingRealms.ocean, 'vsftbarot', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.gyre = self.declare_chunk(ModelingRealms.ocean, 'gyre', self.startdate, self.member, self.chunk) # noinspection PyPep8Naming def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" if self.model_version in [Models.ECEARTH_2_3_O1L42, Models.ECEARTH_3_0_O1L46, Models.NEMO_3_2_O1L42, Models.NEMO_3_3_O1L46, Models.NEMOVAR_O1L42]: diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index ca572a3f..3e65830e 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -150,11 +150,13 @@ class HeatContent(Diagnostic): return depth_t, depth_w def request_data(self): + """Request data required by the diagnostic""" self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) if self.mxloption != 0: self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" if self.box.min_depth == 0: # For cdftools, this is all levels box_save = None @@ -167,9 +169,7 @@ class HeatContent(Diagnostic): box=box_save, region=self.basin.name) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" nco = Utils.nco temperature_file = TempFile.get() Utils.copy_file(self.thetao.local_file, temperature_file) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 4662bd13..33473bff 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -136,16 +136,16 @@ class HeatContentLayer(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.heatc = self.declare_chunk(ModelingRealms.ocean, 'heatc', self.startdate, self.member, self.chunk, box=self.box) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" nco = Utils.nco thetao_file = TempFile.get() results = TempFile.get() diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index d9b6275f..159ceb6c 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -100,17 +100,17 @@ class Interpolate(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.original = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.original_grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.regridded = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" variable_file = TempFile.get() Utils.copy_file(self.original.local_file, variable_file) Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False, rename_dimension=True) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index ff2ab184..c070aa2a 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -79,14 +79,14 @@ class InterpolateCDO(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: target_grid, variable, domain=ocean - :type options: list[str] - :return: - """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: target_grid, variable, domain=ocean + :type options: list[str] + :return: + """ options_available = (DiagnosticDomainOption(default_value=ModelingRealms.ocean), DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), DiagnosticOption('target_grid', diags.config.experiment.atmos_grid.lower()), @@ -214,17 +214,17 @@ class InterpolateCDO(Diagnostic): return target_grid def request_data(self): + """Request data required by the diagnostic""" self.original = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.original_grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.regridded = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" variable_file = TempFile.get() Utils.copy_file(self.original.local_file, variable_file) Utils.rename_variables(variable_file, {'jpib': 'i', 'jpjb': 'j', 'x': 'i', 'y': 'j', @@ -267,6 +267,24 @@ class InterpolateCDO(Diagnostic): class ComputeWeights(Diagnostic): + """ + Diagnostic used to compute interpolation weights + + Parameters + ---------- + data_manager: DataManager + startdate: str + member: int + chunk: int + domain: ModelingRealm + variable: str + target_grid: str + original_grid: str + weights_file: str + method: str + + """ + alias = 'computeinterpcdoweights' "Diagnostic alias for the configuration file" @@ -294,8 +312,10 @@ class ComputeWeights(Diagnostic): InterpolateCDO.compute_weights(self.method, self.grid, self.sample_data.local_file, self.weights_file) def request_data(self): + """Request data required by the diagnostic""" self.sample_data = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.original_grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" pass diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index 20b14369..c95a3a93 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -86,17 +86,17 @@ class MaskLand(Diagnostic): return mask def request_data(self): + """Request data required by the diagnostic""" self.var_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.masked_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() Utils.copy_file(self.var_file.local_file, temp) diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index bbbf1a22..b5985e4b 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -91,11 +91,12 @@ class MaxMoc(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_year(ModelingRealms.ocean, MaxMoc.vsftmyz, self.startdate, self.member, self.year) def declare_data_generated(self): - + """Declare data to be generated by the diagnostic""" self.results = {'vsftmyzmax': self.declare_year(ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, self.year, box=self.box, vartype=VariableType.STATISTIC), 'vsftmyzmaxlev': self.declare_year(ModelingRealms.ocean, 'vsftmyzmaxlev', self.startdate, @@ -114,9 +115,7 @@ class MaxMoc(Diagnostic): vartype=VariableType.STATISTIC)} def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" nco = Utils.nco temp = TempFile.get() diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index 8767298e..327d42ad 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -64,16 +64,16 @@ class MixedLayerHeatContent(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.ohcsum = self.declare_chunk(ModelingRealms.ocean, 'ohcvsumlotst', self.startdate, self.member, self.chunk) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temperature_file = TempFile.get() Utils.copy_file(self.thetao.local_file, temperature_file) Utils.nco.ncks(input=self.mlotst.local_file, output=temperature_file, options=('-A -v mlotst',)) diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index 0ffe00a5..8aa803cc 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -63,16 +63,16 @@ class MixedLayerSaltContent(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.so = self.request_chunk(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.sosum = self.declare_chunk(ModelingRealms.ocean, 'scvsummlotst', self.startdate, self.member, self.chunk) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" salinity_file = TempFile.get() Utils.copy_file(self.so.local_file, salinity_file) Utils.nco.ncks(input=self.mlotst.local_file, output=salinity_file, options=('-A -v mlotst',)) diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index 6171c975..ad3299cb 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -67,15 +67,15 @@ class Moc(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.results = self.declare_chunk(ModelingRealms.ocean, Moc.vsftmyz, self.startdate, self.member, self.chunk) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() Log.debug('Computing MOC') diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py index ff6a85e2..d11a1893 100644 --- a/earthdiagnostics/ocean/mxl.py +++ b/earthdiagnostics/ocean/mxl.py @@ -55,16 +55,16 @@ class Mxl(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.thetao_file = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) self.so_file = self.request_chunk(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.mlotst_file = self.declare_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() cdftools.run('cdfmxl', input_file=[self.thetao_file, self.so_file], output_file=temp, options='-nc4') temp2 = TempFile.get() diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index e67a25bc..11f41938 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -61,16 +61,16 @@ class Psi(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.uo = self.request_chunk(ModelingRealms.ocean, 'uo', self.startdate, self.member, self.chunk) self.vo = self.request_chunk(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.psi = self.declare_chunk(ModelingRealms.ocean, Psi.vsftbarot, self.startdate, self.member, self.chunk) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() cdftools.run('cdfpsi', input_file=[self.uo.local_file, self.vo.local_file], output_file=temp, options='-mean -mask') diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index dbff3fc5..ca43049f 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -99,10 +99,12 @@ class RegionMean(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" if self.box.min_depth == 0: # To cdftools, this means all levels box_save = None @@ -117,9 +119,7 @@ class RegionMean(Diagnostic): self.declare_var('var', True, box_save) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" mean_file = TempFile.get() variable_file = self.variable_file.local_file diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index 21a731d0..4d6435e2 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -97,10 +97,12 @@ class RegionSum(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" if self.box.min_depth == 0: # To cdftools, this means all levels box_save = None @@ -111,9 +113,7 @@ class RegionSum(Diagnostic): self.declare_var('sum', True, box_save) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" mean_file = TempFile.get() variable_file = self.variable_file.local_file diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 2ac29073..a71a4938 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -79,19 +79,19 @@ class Rotation(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.ufile = self.request_chunk(self.domain, self.variableu, self.startdate, self.member, self.chunk) self.vfile = self.request_chunk(self.domain, self.variablev, self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.urotated_file = self.declare_chunk(self.domain, self.variableu, self.startdate, self.member, self.chunk, grid='rotated') self.vrotated_file = self.declare_chunk(self.domain, self.variablev, self.startdate, self.member, self.chunk, grid='rotated') def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" handler = Utils.open_cdf(self.ufile.local_file) if 'lev' in handler.dimensions: diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 29de93d5..3f0e9f3d 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -98,6 +98,7 @@ class Siasiesiv(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" if not self.omit_volume: self.sit = self.request_chunk(ModelingRealms.seaIce, self.sit_varname, self.startdate, self.member, self.chunk) @@ -105,6 +106,7 @@ class Siasiesiv(Diagnostic): self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" if not self.omit_volume: self._declare_var('sivols') self._declare_var('sivoln') @@ -120,9 +122,7 @@ class Siasiesiv(Diagnostic): self.chunk, region=self.basin.name) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" import earthdiagnostics.cdftoolspython as cdftoolspython sic_handler = Utils.open_cdf(self.sic.local_file) diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 515aeed3..35cea5b2 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -81,17 +81,17 @@ class VerticalGradient(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.gradient_file = self.declare_chunk(ModelingRealms.ocean, self.variable + 'vgrad', self.startdate, self.member, self.chunk, box=self.box) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" handler = Utils.open_cdf(self.variable_file) if 'lev' not in handler.dimensions: raise Exception('Variable {0} does not have a level dimension') diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 040fa5b3..f8f2253a 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -82,17 +82,17 @@ class VerticalMean(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.results = self.declare_chunk(ModelingRealms.ocean, self.variable + 'vmean', self.startdate, self.member, self.chunk, box=self.box) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() handler = Utils.open_cdf(self.variable_file.local_file) if self.box.min_depth is None: diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 4eb240ea..2b72117a 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -84,17 +84,17 @@ class VerticalMeanMeters(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.results = self.declare_chunk(self.domain, self.variable + 'vmean', self.startdate, self.member, self.chunk, box=self.box) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() handler = Utils.open_cdf(self.variable_file.local_file) if self.box.min_depth is None: diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 229ad020..0b53aed4 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -84,6 +84,7 @@ class ClimatologicalPercentile(Diagnostic): return ['{0}{1:02}01'.format(year, self.forecast_month) for year in range(self.start_year, self.end_year+1)] def request_data(self): + """Request data required by the diagnostic""" for startdate in self.requested_startdates(): if startdate not in self.leadtime_files: self.leadtime_files[startdate] = {} @@ -92,14 +93,13 @@ class ClimatologicalPercentile(Diagnostic): None, None, vartype=VariableType.STATISTIC) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" var_name = '{0.variable}prct{0.start_year}{0.forecast_month}-{0.end_year}{0.forecast_month:02d}'.format(self) self.percentiles_file = self.declare_chunk(self.domain, var_name, None, None, None, frequency=Frequencies.climatology, vartype=VariableType.STATISTIC) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" iris.FUTURE.netcdf_promote = True self._get_distribution() percentile_values = self._calculate_percentiles() diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index defcbb07..beb2235b 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -77,6 +77,7 @@ class DaysOverPercentile(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" var_name = '{0.variable}prct{0.start_year}{0.forecast_month}-{0.end_year}{0.forecast_month:02d}'.format(self) self.percentiles_file = self.request_chunk(self.domain, var_name, None, None, None, frequency=Frequencies.climatology) @@ -84,6 +85,7 @@ class DaysOverPercentile(Diagnostic): self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, None, None) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" var_over = self.variable + '_daysover_q{0}_{1.start_year}-{1.end_year}' var_below = self.variable + '_daysbelow_q{0}_{1.start_year}-{1.end_year}' self.days_over_file = {} @@ -100,9 +102,7 @@ class DaysOverPercentile(Diagnostic): vartype=VariableType.STATISTIC) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" iris.FUTURE.netcdf_promote = True percentiles = iris.load_cube(self.percentiles_file.local_file) diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index 7e078db3..004628e9 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -117,17 +117,17 @@ class Discretize(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.original_data = self.request_chunk(self.domain, self.variable, self.startdate, None, None) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" var_name = '{0.variable}_dis'.format(self) self.discretized_data = self.declare_chunk(self.domain, var_name, self.startdate, None, None, vartype=VariableType.STATISTIC) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" self.print_memory_used() iris.FUTURE.netcdf_promote = True self._load_cube() diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index aa361f6d..ad1466c0 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -70,9 +70,11 @@ class MonthlyPercentile(Diagnostic): return job_list def request_data(self): + """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk) def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" self.max_file = self.declare_chunk(self.domain, self.variable_max, self.startdate, self.member, self.chunk, frequency=Frequencies.monthly, vartype=VariableType.STATISTIC) self.min_file = self.declare_chunk(self.domain, self.variable_min, self.startdate, self.member, self.chunk, @@ -99,9 +101,7 @@ class MonthlyPercentile(Diagnostic): return '{0}_q{1}'.format(self.variable, percentile) def compute(self): - """ - Runs the diagnostic - """ + """Run the diagnostic""" temp = TempFile.get() handler = Utils.open_cdf(self.variable_file.local_file) datetimes = Utils.get_datetime_from_netcdf(handler) -- GitLab From d8544cc588ba81359162e7c22e78c6f62279bec1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 11:32:55 +0100 Subject: [PATCH 094/168] Fixed small isues --- earthdiagnostics/cmorizer.py | 4 ++-- earthdiagnostics/cmormanager.py | 9 --------- earthdiagnostics/ocean/interpolatecdo.py | 4 ++-- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index a196063f..2a7a8c64 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -478,8 +478,8 @@ class Cmorizer(object): def _get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') - if file_parts[0] in (self.experiment.expid, 't00o', 'MMA', 'MMASH', 'MMAGG', 'MMO') or \ - file_parts[0].startswith('ORCA'): + valid_starts = (self.experiment.expid, 'MMA', 'MMASH', 'MMAGG', 'MMO') + if file_parts[0] in valid_starts or file_parts[0].startswith('ORCA'): # Model output if file_parts[-1].endswith('.tar'): file_parts = file_parts[-1][0:-4].split('-') diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index a2cce129..c59ed8d5 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -775,12 +775,3 @@ class MergeYear(Diagnostic): for chunk_file in self.chunk_files: os.remove(chunk_file) return temp - - def __str__(self): - return 'Create year CMOR file Startdate: {0.startdate} Member: {0.member} Year: {0.year} ' \ - 'Variable: {0.domain}:{0.var} Grid: {0.grid} Box: {0.box}'.format(self) - - def __eq__(self, other): - return self.startdate == other.startdate and self.member == other.member and self.year == other.year and \ - self.domain == other.domain and self.var == other.var and self.grid == other.grid and \ - self.box == other.box diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index c070aa2a..83004c12 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -269,7 +269,7 @@ class InterpolateCDO(Diagnostic): class ComputeWeights(Diagnostic): """ Diagnostic used to compute interpolation weights - + Parameters ---------- data_manager: DataManager @@ -284,7 +284,7 @@ class ComputeWeights(Diagnostic): method: str """ - + alias = 'computeinterpcdoweights' "Diagnostic alias for the configuration file" -- GitLab From d6e18eebf53840aa3361fc9a64bb45cae869549b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 11:42:57 +0100 Subject: [PATCH 095/168] Updated data_manager doc --- earthdiagnostics/datamanager.py | 103 +++++++++++++++++----------- earthdiagnostics/obsreconmanager.py | 37 +++++----- earthdiagnostics/threddsmanager.py | 37 +++++----- 3 files changed, 97 insertions(+), 80 deletions(-) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 92a74332..8b043fbb 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -1,4 +1,5 @@ # coding: utf-8 +"""Base data manager for Earth diagnostics""" import csv import os import re @@ -15,9 +16,11 @@ class DataManager(object): """ Class to manage the data repositories - :param config: - :type config: Config + Parameters + ---------- + config: Config """ + def __init__(self, config): self.config = config self.experiment = config.experiment @@ -58,21 +61,37 @@ class DataManager(object): return var def get_varfolder(self, domain, var, grid=None, frequency=None): + """Get variable folder name for _ folder""" if grid: var = '{0}-{1}'.format(var, grid) if domain in [ModelingRealms.ocean, ModelingRealms.seaIce, ModelingRealms.ocnBgchem]: - return self._apply_fxh(var, self.experiment.ocean_timestep, frequency) + return DataManager._apply_fxh(var, self.experiment.ocean_timestep, frequency) else: - return self._apply_fxh(var, self.experiment.atmos_timestep, frequency) + return DataManager._apply_fxh(var, self.experiment.atmos_timestep, frequency) - def _apply_fxh(self, folder_name, timestep, frequency=None): + @staticmethod + def _apply_fxh(folder_name, timestep, frequency=None): is_base_frequency = frequency is not None and frequency.frequency.endswith('hr') if not is_base_frequency and timestep > 0: return '{0}_f{1}h'.format(folder_name, timestep) return folder_name def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + """ + Create file link + + Parameters + ---------- + domain: ModelingRealm + filepath: str + frequency: Frequency + var: str + grid: str + move_old: bool + vartype: VariableType + + """ if self.config.data_convention == 'meteofrance': return freq_str = frequency.folder_name(vartype) @@ -133,7 +152,7 @@ class DataManager(object): def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ - Creates the link of a given file from the CMOR repository. + Create the link of a given file from the CMOR repository. :param cmor_var: :param move_old: @@ -162,36 +181,29 @@ class DataManager(object): pass def prepare(self): - """ - Prepares the data to be used by the diagnostic. - :return: - """ + """Prepare the data to be used by Earth Diagnostics""" pass def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + Request a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None + box: Box or None + frequency: Frequency or None + vartype: VariableType or None + + Returns + ------- + DataFile - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str|None - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|None - :return: path to the copy created on the scratch folder - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :rtype: str """ raise NotImplementedError('Class must override request_chunk method') @@ -199,14 +211,27 @@ class DataManager(object): class UnitConversion(object): """ Class to manage unit conversions + + Parameters + ---------- + source: str + destiny: str + factor: float + offset: float + """ + _dict_conversions = None + def __init__(self, source, destiny, factor, offset): + self.source = source + self.destiny = destiny + self.factor = float(factor) + self.offset = float(offset) + @classmethod def load_conversions(cls): - """ - Load conversions from the configuration file - """ + """Load conversions from the configuration file""" cls._dict_conversions = dict() with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'r') as csvfile: reader = csv.reader(csvfile, dialect='excel') @@ -225,16 +250,14 @@ class UnitConversion(object): """ cls._dict_conversions[(conversion.source, conversion.destiny)] = conversion - def __init__(self, source, destiny, factor, offset): - self.source = source - self.destiny = destiny - self.factor = float(factor) - self.offset = float(offset) + @classmethod def get_conversion_factor_offset(cls, input_units, output_units): """ - Gets the conversion factor and offset for two units . The conversion has to be done in the following way: + Get the conversion factor and offset for two units. + + The conversion has to be done in the following way: converted = original * factor + offset :param input_units: original units diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index f9f35374..d5391d61 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -194,27 +194,24 @@ class ObsReconManager(DataManager): def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + Request a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None + box: Box or None + frequency: Frequency or None + vartype: VariableType or None + + Returns + ------- + DataFile - :param vartype: - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str|None - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|None - :return: path to the copy created on the scratch folder - :rtype: str """ var = self._get_final_var_name(box, var) filepath = self.get_file_path(startdate, domain, var, frequency, vartype, box, grid) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index e3b81029..7c4a1415 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -184,27 +184,24 @@ class THREDDSManager(DataManager): def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN): """ - Copy a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + Request a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None + box: Box or None + frequency: Frequency or None + vartype: VariableType or None + + Returns + ------- + DataFile - :param vartype: - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str|None - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|None - :return: path to the copy created on the scratch folder - :rtype: str """ aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) file_path = self.get_file_path(startdate, domain, var, frequency, vartype, box=box) -- GitLab From bac8177b4a6d14ef151957d6e00496fe60a1f2d9 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 11:47:32 +0100 Subject: [PATCH 096/168] Updated box doc --- earthdiagnostics/box.py | 42 +++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/earthdiagnostics/box.py b/earthdiagnostics/box.py index 51cefe96..4d68bb10 100644 --- a/earthdiagnostics/box.py +++ b/earthdiagnostics/box.py @@ -1,9 +1,20 @@ # coding=utf-8 +"""Module to manage 3D space restrictions""" + + class Box(object): """ - Represents a box in the 3D space. Also allows easy conversion from the coordinate values to significant string + Represents a box in the 3D space. + + Also allows easy conversion from the coordinate values to significant string representations + + Parameters + ---------- + depth_in_meters: bool, optional + If True, depth is given in meters. If False, it correspond to levels """ + def __init__(self, depth_in_meters=False): self.depth_in_meters = depth_in_meters """ @@ -37,6 +48,7 @@ class Box(object): def max_lat(self): """ Maximum latitude + :rtype: float """ return self._max_lat @@ -50,8 +62,9 @@ class Box(object): @property def min_lat(self): """ - Minimum latitude - :rtype: float + Minimum latitude + + :rtype: float """ return self._min_lat @@ -64,8 +77,9 @@ class Box(object): @property def max_lon(self): """ - Maximum longitude - :rtype: float + Maximum longitude + + :rtype: float """ return self._max_lon @@ -78,8 +92,9 @@ class Box(object): @property def min_lon(self): """ - Minimum longitude - :rtype: float + Minimum longitude + + :rtype: float """ return self._min_lon @@ -91,8 +106,10 @@ class Box(object): def get_lat_str(self): """ - Gets a string representation of the latitude in the format XX{N/S}. + Get a string representation of the latitude in the format XX{N/S}. + If min_lat is different from max_lat, it concatenates the two values + :return: string representation for latitude :rtype: str """ @@ -116,8 +133,10 @@ class Box(object): def get_lon_str(self): """ - Gets a string representation of the longitude in the format XX{E/W}. + Get a string representation of the longitude in the format XX{E/W}. + If min_lon is different from max_lon, it concatenates the two values + :return: string representation for longitude :rtype: str """ @@ -140,8 +159,11 @@ class Box(object): def get_depth_str(self): """ - Gets a string representation of depth. For depth expressed in meters, it adds th character 'm' to the end + Get a string representation of depth. + + For depth expressed in meters, it adds the character 'm' to the end If min_depth is different from max_depth, it concatenates the two values + :return: string representation for depth :rtype: str """ -- GitLab From 14bbcc49a09379e76f02023814406240534c7bf2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 11:54:43 +0100 Subject: [PATCH 097/168] Updated work_manager doc --- earthdiagnostics/work_manager.py | 46 +++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index a840cd2d..153ceb33 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Earthdiagnostics workflow manager""" import datetime import operator import sys @@ -17,6 +18,14 @@ from earthdiagnostics.utils import Utils, TempFile class WorkManager(object): + """ + Class to produce and control the workflow of EarthDiagnostics + + Parameters + ---------- + config: Config + data_manager: DataManager + """ def __init__(self, config, data_manager): self.jobs = None @@ -26,6 +35,7 @@ class WorkManager(object): self.data_manager = data_manager def prepare_job_list(self): + """Create the list of jobs to run""" self._register_diagnostics() list_jobs = list() for fulldiag in self.config.get_commands(): @@ -49,6 +59,15 @@ class WorkManager(object): self.jobs = list_jobs def run(self): + """ + Run all the diagnostics + + + Returns + ------- + bool + Only True if all diagnostic were correctly executed + """ if len(self.jobs) == 0: Log.result('No diagnostics to run') return True @@ -96,8 +115,8 @@ class WorkManager(object): finish_time = datetime.datetime.now() Log.result("Diagnostics finished at {0}", finish_time) Log.result("Elapsed time: {0}\n", finish_time - start_time) - self.print_errors() - self.print_stats() + self._print_errors() + self._print_stats() return not self.had_errors def _job_status_changed(self, job): @@ -108,7 +127,7 @@ class WorkManager(object): del self.data_manager.requested_files[request.remote_file] request.unsubscribe(self) request.clean_local() - self.check_completion() + self._check_completion() def _file_object_status_changed(self, file_object): Log.debug('Checking file {0}. Local status {0.local_status} Storage status{0.storage_status}', file_object) @@ -125,9 +144,9 @@ class WorkManager(object): del self.data_manager.requested_files[file_object.remote_file] file_object.unsubscribe(self) file_object.clean_local() - self.check_completion() + self._check_completion() - def check_completion(self): + def _check_completion(self): counter = 0 for job in self.jobs: if job.status == DiagnosticStatus.READY: @@ -160,7 +179,7 @@ class WorkManager(object): pass return True - def print_stats(self): + def _print_stats(self): Log.info('Time consumed by each diagnostic class') Log.info('--------------------------------------') @@ -175,7 +194,7 @@ class WorkManager(object): for diag in sorted(times, key=operator.itemgetter(1)): Log.info('{0:23} {1:}', diag, times[diag]) - def print_errors(self): + def _print_errors(self): failed = [job for job in self.jobs if job.status == DiagnosticStatus.FAILED] if len(failed) == 0: return @@ -301,6 +320,11 @@ class WorkManager(object): class Downloader(object): + """ + Download manager for EarthDiagnostics + + We are not using a ThreadPoolExecutor because we want to be able to control priorities in the download + """ def __init__(self): self._downloads = [] self._lock = threading.Lock() @@ -308,15 +332,17 @@ class Downloader(object): self.on_hold = False def start(self): - self._thread = threading.Thread(target=self.downloader) + """Create the downloader thread and initialize it""" + self._thread = threading.Thread(target=self._downloader) self._thread.start() def submit(self, datafile): + """Add a datafile to the download queue""" self._lock.acquire() self._downloads.append(datafile) self._lock.release() - def downloader(self): + def _downloader(self): try: def suscribers_waiting(datafile): waiting = 0 @@ -363,6 +389,6 @@ class Downloader(object): Log.critical('Unhandled error at downloader: {0}\n{1}', ex, traceback.print_exc()) def shutdown(self): + """Stop the downloader after all downloads have finished""" self.stop = True self._thread.join() - -- GitLab From ffe05fd7ee0e48f1ec22830bd81db23c4e7dd02b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:02:09 +0100 Subject: [PATCH 098/168] Updated doc --- doc/source/codedoc/earthdiagnostics.rst | 62 +++++++++++++++++++++++-- earthdiagnostics/earthdiags.py | 3 +- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/doc/source/codedoc/earthdiagnostics.rst b/doc/source/codedoc/earthdiagnostics.rst index 0bd7d031..77aa310e 100644 --- a/doc/source/codedoc/earthdiagnostics.rst +++ b/doc/source/codedoc/earthdiagnostics.rst @@ -43,6 +43,13 @@ earthdiagnostics.constants :inherited-members: :members: +earthdiagnostics.datafile +------------------------- +.. automodule:: earthdiagnostics.datafile + :show-inheritance: + :inherited-members: + :members: + earthdiagnostics.datamanager ---------------------------- .. automodule:: earthdiagnostics.datamanager @@ -50,7 +57,6 @@ earthdiagnostics.datamanager :inherited-members: :members: - earthdiagnostics.diagnostic --------------------------- .. automodule:: earthdiagnostics.diagnostic @@ -65,9 +71,44 @@ earthdiagnostics.earthdiags :inherited-members: :members: -earthdiagnostics.parser +earthdiagnostics.frequency +-------------------------- +.. automodule:: earthdiagnostics.frequency + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.modellingrealm +------------------------------- +.. automodule:: earthdiagnostics.modellingrealm + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.obsreconmanager +-------------------------------- +.. automodule:: earthdiagnostics.obsreconmanager + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.publisher +-------------------------- +.. automodule:: earthdiagnostics.publisher + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.singleton +-------------------------- +.. automodule:: earthdiagnostics.singleton + :show-inheritance: + :inherited-members: + :members: + +earthdiagnostics.threddsmanager ----------------------- -.. automodule:: earthdiagnostics.parser +.. automodule:: earthdiagnostics.threddsmanager :show-inheritance: :inherited-members: :members: @@ -85,3 +126,18 @@ earthdiagnostics.variable :show-inheritance: :inherited-members: :members: + +earthdiagnostics.variable_type +------------------------------ +.. automodule:: earthdiagnostics.variable_type + :show-inheritance: + :inherited-members: + :members: + + +earthdiagnostics.workmanager +---------------------------- +.. automodule:: earthdiagnostics.work_manager + :show-inheritance: + :inherited-members: + :members: diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index db732f05..cbcbc568 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -36,7 +36,6 @@ class EarthDiags(object): version_path = os.path.join(scriptdir, 'VERSION') readme_path = os.path.join(scriptdir, 'README') changes_path = os.path.join(scriptdir, 'CHANGELOG') - documentation_path = os.path.join(scriptdir, 'EarthDiagnostics.pdf') if os.path.isfile(version_path): with open(version_path) as f: version = f.read().strip() @@ -90,7 +89,7 @@ class EarthDiags(object): args = parser.parse_args() if args.doc: Log.info('Opening documentation...') - doc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'EarthDiagnostics.pdf') + doc_path = os.path.join('http://earthdiagnostics.readthedocs.io/en/latest') Utils.execute_shell_command(('xdg-open', doc_path)) Log.result('Documentation opened!') return True -- GitLab From 408875d065ea9e92c8c7f1dfb453e8e43ef7f65b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:09:36 +0100 Subject: [PATCH 099/168] Updated config doc --- earthdiagnostics/config.py | 53 +++++++++++++++++++++++++++----------- test/unit/test_config.py | 30 ++++++++++++++------- 2 files changed, 58 insertions(+), 25 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 94fa627d..8b942928 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -99,7 +99,8 @@ class Config(object): "If True, forces the tool to copy all the mesh and mask files for the model, regardless of existence" # Read experiment config - self.experiment = ExperimentConfig(parser) + self.experiment = ExperimentConfig() + self.experiment.parse_ini(parser) """ Configuration related to the experiment @@ -134,9 +135,11 @@ class Config(object): def get_commands(self): """ - Returns the list of commands after replacing the alias - :return: full list of commands - :rtype: list(str) + Return the list of commands after replacing the alias + + Returns + ------- + iterable of str """ return self._real_commands @@ -201,11 +204,12 @@ class CMORConfig(object): def cmorize(self, var_cmor): """ - Checks if var_cmor is on variable list + Check if var_cmor is on variable list + + Parameters + ---------- + var_cmor: Variable - :param var_cmor: CMOR variable object - :rtype var_cmor: Variablle - :return: """ if self._variable_list is None: return True @@ -214,6 +218,18 @@ class CMORConfig(object): return '{0}:{1}'.format(var_cmor.domain, var_cmor.short_name) in self._variable_list def any_required(self, variables): + """ + Check if any of the given variables is needed for cmorization + + Parameters + ---------- + variables: iterable of str + + Returns + ------- + bool + + """ if self._variable_list is None: return True for var in variables: @@ -336,14 +352,17 @@ class THREDDSConfig(object): class ExperimentConfig(object): """ Configuration related to the experiment + """ - Parameters - ---------- - parser: ConfigParser + def parse_ini(self, parser): + """ + Parse experiment section from INI-like file - """ + Parameters + ---------- + parser: ConfigParser - def __init__(self, parser): + """ self.institute = parser.get_option('EXPERIMENT', 'INSTITUTE') self.expid = parser.get_option('EXPERIMENT', 'EXPID') self.experiment_name = parser.get_option('EXPERIMENT', 'NAME', self.expid) @@ -553,11 +572,13 @@ class ExperimentConfig(object): def get_full_years(self, startdate): """ - Returns the list of full years that are in the given startdate + Return the list of full years that are in the given startdate + :param startdate: startdate to use :type startdate: str :return: list of full years :rtype: list[int] + """ chunks_per_year = 12 / self.chunk_size date = parse_date(startdate) @@ -578,11 +599,13 @@ class ExperimentConfig(object): def get_member_str(self, member): """ - Returns the member name for a given member number. + Return the member name for a given member number. + :param member: member's number :type member: int :return: member's name :rtype: str + """ return '{0}{1}'.format(self.member_prefix, str(member).zfill(self.member_digits)) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 441e983f..ec3720f3 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -284,7 +284,8 @@ class TestExperimentConfig(TestCase): self.mock_parser = ParserMock() def test_basic_config(self): - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, []) self.assertEquals(config.members, []) @@ -297,41 +298,50 @@ class TestExperimentConfig(TestCase): def test_startdates(self): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '20001101 20011101') - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20011101']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200(0|1)1101') - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20011101']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200[0-2](02|05|08|11)01') - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', u'20020801', u'20021101']) def test_auto_startdates(self): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,1Y}') - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20011101']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,6M}') - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20010501', '20011101']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,1W}') - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,W}') - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7D}') - config = ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7F}') with self.assertRaises(ConfigException): - ExperimentConfig(self.mock_parser) + config = ExperimentConfig() + config.parse_ini(self.mock_parser) -- GitLab From 138fe24460933d1c9facd9b42670a9fb11dfd2c4 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:14:41 +0100 Subject: [PATCH 100/168] Fixed work_manager issues --- earthdiagnostics/work_manager.py | 83 +++++++++++++++++--------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 153ceb33..a0297dbc 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -62,7 +62,6 @@ class WorkManager(object): """ Run all the diagnostics - Returns ------- bool @@ -147,14 +146,7 @@ class WorkManager(object): self._check_completion() def _check_completion(self): - counter = 0 - for job in self.jobs: - if job.status == DiagnosticStatus.READY: - counter += 1 - if counter > 20: - break - - self.downloader.on_hold = counter > 20 + self._pause_downloader_if_required() for job in self.jobs: if job.status in (DiagnosticStatus.READY, DiagnosticStatus.RUNNING): @@ -179,6 +171,15 @@ class WorkManager(object): pass return True + def _pause_downloader_if_required(self): + counter = 0 + for job in self.jobs: + if job.status == DiagnosticStatus.READY: + counter += 1 + if counter > 20: + break + self.downloader.on_hold = counter > 20 + def _print_stats(self): Log.info('Time consumed by each diagnostic class') Log.info('--------------------------------------') @@ -325,6 +326,7 @@ class Downloader(object): We are not using a ThreadPoolExecutor because we want to be able to control priorities in the download """ + def __init__(self): self._downloads = [] self._lock = threading.Lock() @@ -344,36 +346,6 @@ class Downloader(object): def _downloader(self): try: - def suscribers_waiting(datafile): - waiting = 0 - for diag in datafile.suscribers: - if not isinstance(diag, Diagnostic): - continue - if diag.pending_requests() == 1: - waiting += 1 - return waiting - - def prioritize(datafile1, datafile2): - waiting = suscribers_waiting(datafile1) - suscribers_waiting(datafile2) - if waiting: - return -waiting - - suscribers = len(datafile1.suscribers) - len(datafile2.suscribers) - if suscribers: - return -suscribers - - if datafile1.size is None: - if datafile2.size is None: - return 0 - else: - return -1 - elif datafile2.size is None: - return 1 - size = datafile1.size - datafile2.size - if size: - return -size - return 0 - while True: with self._lock: if len(self._downloads) == 0 or self.on_hold: @@ -381,13 +353,44 @@ class Downloader(object): return time.sleep(0.01) continue - self._downloads.sort(key=cmp_to_key(prioritize)) + self._downloads.sort(key=cmp_to_key(Downloader._prioritize())) datafile = self._downloads[0] self._downloads.remove(datafile) datafile.download() except Exception as ex: Log.critical('Unhandled error at downloader: {0}\n{1}', ex, traceback.print_exc()) + @staticmethod + def _suscribers_waiting(datafile): + waiting = 0 + for diag in datafile.suscribers: + if not isinstance(diag, Diagnostic): + continue + if diag.pending_requests() == 1: + waiting += 1 + return waiting + @staticmethod + def _prioritize(datafile1, datafile2): + waiting = Downloader._suscribers_waiting(datafile1) - Downloader._suscribers_waiting(datafile2) + if waiting: + return -waiting + + suscribers = len(datafile1.suscribers) - len(datafile2.suscribers) + if suscribers: + return -suscribers + + if datafile1.size is None: + if datafile2.size is None: + return 0 + else: + return -1 + elif datafile2.size is None: + return 1 + size = datafile1.size - datafile2.size + if size: + return -size + return 0 + def shutdown(self): """Stop the downloader after all downloads have finished""" self.stop = True -- GitLab From 8a41b83b74a45e2a26b3f7704b3808c64befe2d1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:16:37 +0100 Subject: [PATCH 101/168] Fixed config issues --- earthdiagnostics/config.py | 61 ++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 8b942928..4b32d09b 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -100,12 +100,12 @@ class Config(object): # Read experiment config self.experiment = ExperimentConfig() - self.experiment.parse_ini(parser) """ Configuration related to the experiment :rtype: ExperimentConfig """ + self.experiment.parse_ini(parser) # Read aliases self._aliases = dict() if parser.has_section('ALIAS'): @@ -350,9 +350,7 @@ class THREDDSConfig(object): class ExperimentConfig(object): - """ - Configuration related to the experiment - """ + """Configuration related to the experiment""" def parse_ini(self, parser): """ @@ -371,24 +369,22 @@ class ExperimentConfig(object): self.member_prefix = parser.get_option('EXPERIMENT', 'MEMBER_PREFIX', 'fc') self.member_count_start = parser.get_int_option('EXPERIMENT', 'MEMBER_COUNT_START', 0) - members = [] - for mem in self.members: - if '-' in mem: - start, end = mem.split('-') - if start.startswith(self.member_prefix): - start = start[len(self.member_prefix):] - if end.startswith(self.member_prefix): - end = end[len(self.member_prefix):] - for member in range(int(start), int(end) + 1): - members.append(member) - else: - if mem.startswith(self.member_prefix): - mem = mem[len(self.member_prefix):] - members.append(int(mem)) - self.members = members + self._parse_members() self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') - startdates = parser.get_list_option('EXPERIMENT', 'STARTDATES') + self._parse_startdates(parser) + + self.chunk_size = parser.get_int_option('EXPERIMENT', 'CHUNK_SIZE') + self.num_chunks = parser.get_int_option('EXPERIMENT', 'CHUNKS') + self.chunk_list = parser.get_int_list_option('EXPERIMENT', 'CHUNK_LIST', []) + self.model = parser.get_option('EXPERIMENT', 'MODEL') + self.model_version = parser.get_option('EXPERIMENT', 'MODEL_VERSION', '') + self.atmos_grid = parser.get_option('EXPERIMENT', 'ATMOS_GRID', '') + self.atmos_timestep = parser.get_int_option('EXPERIMENT', 'ATMOS_TIMESTEP', 6) + self.ocean_timestep = parser.get_int_option('EXPERIMENT', 'OCEAN_TIMESTEP', 6) + + def _parse_startdates(self, parser): + startdates = parser.get_list_option('EXPERIMENT', 'STARTDATES') import exrex self.startdates = [] for startdate_pattern in startdates: @@ -403,15 +399,22 @@ class ExperimentConfig(object): if startdate: self.startdates.append(startdate) - self.chunk_size = parser.get_int_option('EXPERIMENT', 'CHUNK_SIZE') - self.num_chunks = parser.get_int_option('EXPERIMENT', 'CHUNKS') - self.chunk_list = parser.get_int_list_option('EXPERIMENT', 'CHUNK_LIST', []) - - self.model = parser.get_option('EXPERIMENT', 'MODEL') - self.model_version = parser.get_option('EXPERIMENT', 'MODEL_VERSION', '') - self.atmos_grid = parser.get_option('EXPERIMENT', 'ATMOS_GRID', '') - self.atmos_timestep = parser.get_int_option('EXPERIMENT', 'ATMOS_TIMESTEP', 6) - self.ocean_timestep = parser.get_int_option('EXPERIMENT', 'OCEAN_TIMESTEP', 6) + def _parse_members(self): + members = [] + for mem in self.members: + if '-' in mem: + start, end = mem.split('-') + if start.startswith(self.member_prefix): + start = start[len(self.member_prefix):] + if end.startswith(self.member_prefix): + end = end[len(self.member_prefix):] + for member in range(int(start), int(end) + 1): + members.append(member) + else: + if mem.startswith(self.member_prefix): + mem = mem[len(self.member_prefix):] + members.append(int(mem)) + self.members = members def _read_startdates(self, pattern): pattern = pattern.split(',') -- GitLab From ad75af882bae958ae701d497c761737b7d99853f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:21:23 +0100 Subject: [PATCH 102/168] Fixed work_manager issues --- earthdiagnostics/work_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index a0297dbc..a6fa51e7 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -353,7 +353,7 @@ class Downloader(object): return time.sleep(0.01) continue - self._downloads.sort(key=cmp_to_key(Downloader._prioritize())) + self._downloads.sort(key=cmp_to_key(Downloader._prioritize)) datafile = self._downloads[0] self._downloads.remove(datafile) datafile.download() @@ -369,6 +369,7 @@ class Downloader(object): if diag.pending_requests() == 1: waiting += 1 return waiting + @staticmethod def _prioritize(datafile1, datafile2): waiting = Downloader._suscribers_waiting(datafile1) - Downloader._suscribers_waiting(datafile2) -- GitLab From 6ccffaec9e8e4b0aa3cb0b27ade0e9779c3ec6b1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:38:52 +0100 Subject: [PATCH 103/168] Reorganized data_manager methods --- earthdiagnostics/cmormanager.py | 73 +++++++++++ earthdiagnostics/datamanager.py | 195 ++++++++++++++++++---------- earthdiagnostics/obsreconmanager.py | 170 +++++------------------- 3 files changed, 233 insertions(+), 205 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index c59ed8d5..a7e85dee 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -2,6 +2,8 @@ """Classes to manage cmorized datasets""" import glob import os +import re +import shutil from datetime import datetime from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, previous_day @@ -390,6 +392,77 @@ class CMORManager(DataManager): grid=grid, year=year, date_str=date_str) self.create_link(domain, filepath, frequency, var, grid, move_old, vartype) + def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + """ + Create file link + + Parameters + ---------- + domain: ModelingRealm + filepath: str + frequency: Frequency + var: str + grid: str + move_old: bool + vartype: VariableType + + """ + if self.config.data_convention == 'meteofrance': + return + freq_str = frequency.folder_name(vartype) + + if not grid: + grid = 'original' + + variable_folder = self.get_varfolder(domain, var) + vargrid_folder = self.get_varfolder(domain, var, grid) + + self.lock.acquire() + try: + if grid == 'original': + link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) + if os.path.islink(link_path): + link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) + + Utils.create_folder_tree(link_path) + else: + link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) + Utils.create_folder_tree(link_path) + default_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) + original_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, + vargrid_folder.replace('-{0}_f'.format(grid), '-original_f')) + + if os.path.islink(default_path): + os.remove(default_path) + elif os.path.isdir(default_path): + shutil.move(default_path, original_path) + os.symlink(link_path, default_path) + + if move_old and link_path not in self._checked_vars: + self._checked_vars.append(link_path) + old_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, + 'old_{0}_f{1}h'.format(var, self.experiment.atmos_timestep)) + regex = re.compile(var + '_[0-9]{6,8}\.nc') + for filename in os.listdir(link_path): + if regex.match(filename): + Utils.create_folder_tree(old_path) + Utils.move_file(os.path.join(link_path, filename), + os.path.join(old_path, filename)) + + link_path = os.path.join(link_path, os.path.basename(filepath)) + if os.path.lexists(link_path): + os.remove(link_path) + if not os.path.exists(filepath): + raise ValueError('Original file {0} does not exists'.format(filepath)) + if not os.path.isdir(os.path.dirname(link_path)): + Utils.create_folder_tree(os.path.dirname(link_path)) + relative_path = os.path.relpath(filepath, os.path.dirname(link_path)) + os.symlink(relative_path, link_path) + except Exception: + raise + finally: + self.lock.release() + def prepare(self): """ Prepare the data to be used by the diagnostic. diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 8b043fbb..16759134 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -2,13 +2,10 @@ """Base data manager for Earth diagnostics""" import csv import os -import re -import shutil import threading from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.utils import Utils from earthdiagnostics.variable_type import VariableType @@ -81,6 +78,8 @@ class DataManager(object): """ Create file link + Must be implementd by the derived classes. If not, this method will have no effect + Parameters ---------- domain: ModelingRealm @@ -92,63 +91,8 @@ class DataManager(object): vartype: VariableType """ - if self.config.data_convention == 'meteofrance': - return - freq_str = frequency.folder_name(vartype) - - if not grid: - grid = 'original' - - variable_folder = self.get_varfolder(domain, var) - vargrid_folder = self.get_varfolder(domain, var, grid) - - self.lock.acquire() - try: - if grid == 'original': - link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) - if os.path.islink(link_path): - link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) + pass - Utils.create_folder_tree(link_path) - else: - link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) - Utils.create_folder_tree(link_path) - default_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) - original_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, - vargrid_folder.replace('-{0}_f'.format(grid), '-original_f')) - - if os.path.islink(default_path): - os.remove(default_path) - elif os.path.isdir(default_path): - shutil.move(default_path, original_path) - os.symlink(link_path, default_path) - - if move_old and link_path not in self._checked_vars: - self._checked_vars.append(link_path) - old_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, - 'old_{0}_f{1}h'.format(var, self.experiment.atmos_timestep)) - regex = re.compile(var + '_[0-9]{6,8}\.nc') - for filename in os.listdir(link_path): - if regex.match(filename): - Utils.create_folder_tree(old_path) - Utils.move_file(os.path.join(link_path, filename), - os.path.join(old_path, filename)) - - link_path = os.path.join(link_path, os.path.basename(filepath)) - if os.path.lexists(link_path): - os.remove(link_path) - if not os.path.exists(filepath): - raise ValueError('Original file {0} does not exists'.format(filepath)) - if not os.path.isdir(os.path.dirname(link_path)): - Utils.create_folder_tree(os.path.dirname(link_path)) - relative_path = os.path.relpath(filepath, os.path.dirname(link_path)) - os.symlink(relative_path, link_path) - except Exception: - raise - finally: - self.lock.release() - - # Overridable methods (not mandatory) def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ @@ -195,18 +139,141 @@ class DataManager(object): startdate: str member: int chunk: int - grid: str or None - box: Box or None - frequency: Frequency or None - vartype: VariableType or None + grid: str or None, optional + box: Box or None, optional + frequency: Frequency or None, optional + vartype: VariableType or None, optional Returns ------- DataFile + Raises + ------ + NotImplementedError + If not implemented by derived classes + """ raise NotImplementedError('Class must override request_chunk method') + def request_year(self, diagnostic, domain, var, startdate, member, year, grid=None, box=None, frequency=None): + """ + Request a given year for a variavle from a CMOR repository + + Parameters + ---------- + diagnostic: Diagnostic + domain: ModelingRealm + var: str + startdate: str + member: int + year: int + grid: str or None, optional + box: Box or None, optional + frequency: Frequency or None, optional + + Returns + ------- + DataFile + + Raises + ------ + NotImplementedError + If not implemented by derived classes + + """ + raise NotImplementedError('Class must override request_year method') + + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Declare a variable chunk to be generated by a diagnostic + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None, optional + region: Basin or None, optional + box: Box or None, optional + frequency: Frequency or None, optional + vartype: VariableType, optional + diagnostic: Diagnostic, optional + + Returns + ------- + DataFile + + Raises + ------ + NotImplementedError + If not implemented by derived classes + + """ + raise NotImplementedError('Class must override declare_chunk method') + + def declare_year(self, domain, var, startdate, member, year, grid=None, box=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Declare a variable year to be generated by a diagnostic + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + year: int + grid: str or None, optional + box: Box or None, optional + vartype: VariableType, optional + diagnostic: Diagnostic, optional + + Returns + ------- + DataFile + + Raises + ------ + NotImplementedError + If not implemented by derived classes + + """ + raise NotImplementedError('Class must override declare_year method') + + def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN, possible_versions=None): + """ + Check if a file exists in the storage + + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None, optional + box: Box or None, optional + frequency: Frequency or None, optional + vartype: VariableType, optional + possible_versions: iterable od str or None, optional + + Raises + ------ + NotImplementedError + If not implemented by derived classes + + Returns + ------- + bool + + """ + raise NotImplementedError('Class must override file_exists method') + class UnitConversion(object): """ @@ -250,8 +317,6 @@ class UnitConversion(object): """ cls._dict_conversions[(conversion.source, conversion.destiny)] = conversion - - @classmethod def get_conversion_factor_offset(cls, input_units, output_units): """ diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index d5391d61..0f586300 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -1,4 +1,10 @@ # coding=utf-8 +""" +Data management for BSC-Earth conventions + +Focused on working with observations and reconstructions as well as with downloaded +but no cmorized models (like ECMWF System 4) +""" import os from bscearth.utils.log import Log @@ -11,6 +17,11 @@ from earthdiagnostics.variable_type import VariableType class ObsReconManager(DataManager): """ Data manager class for CMORized experiments + + Parameters + ---------- + config: Config + """ def __init__(self, config): super(ObsReconManager, self).__init__(config) @@ -28,48 +39,10 @@ class ObsReconManager(DataManager): if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size != 1: raise Exception('For obs and recon data chunk_size must be always 1') - # noinspection PyUnusedLocal - def request_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, - vartype=VariableType.MEAN): - filepath = self.get_file_path(startdate, domain, variable, frequency, vartype) - return self._get_file_from_storage(filepath) - - def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): - pass - - # noinspection PyUnusedLocal - def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VariableType.MEAN): - """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - - :param domain: CMOR domain - :type domain: str - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str - """ - return NotImplementedError - def get_file_path(self, startdate, domain, var, frequency, vartype, box=None, grid=None): """ - Returns the path to a concrete file + Return the path to a concrete file :param startdate: file's startdate :type startdate: str :param domain: file's domain @@ -105,52 +78,6 @@ class ObsReconManager(DataManager): var_folder) return folder_path - def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN): - """ - Ge a file containing all the data for one year for one variable - :param domain: variable's domain - :type domain: str - :param var: variable's name - :type var: str - :param startdate: startdate to retrieve - :type startdate: str - :param member: member to retrieve - :type member: int - :param year: year to retrieve - :type year: int - :param grid: variable's grid - :type grid: str - :param box: variable's box - :type box: Box - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: - """ - raise NotImplementedError() - - def get_var_url(self, var, startdate, frequency, box, vartype): - """ - Get url for dataset - :param var: variable to retrieve - :type var: str - :param startdate: startdate to retrieve - :type startdate: str - :param frequency: frequency to get: - :type frequency: Frequency | None - :param box: box to get - :type box: Box - :param vartype: type of variable - :type vartype: VariableType - :return: - """ - if frequency is None: - frequency = self.config.frequency - var = self._get_final_var_name(box, var) - full_path = os.path.join(self.config.data_dir, self.config.data_type, self.experiment.institute, - self.experiment.model, frequency.folder_name(vartype)) - full_path = os.path.join(full_path, var, self._get_file_name(var, startdate, frequency)) - return full_path - def _get_file_name(self, var, startdate, frequency): if startdate: if self.config.data_type != 'exp' and frequency != Frequencies.weekly: @@ -159,38 +86,6 @@ class ObsReconManager(DataManager): else: return '{0}.nc'.format(var) - def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, - frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): - """ - Creates the link of a given file from the CMOR repository. - - :param cmor_var: - :param move_old: - :param date_str: - :param year: if frequency is yearly, this parameter is used to give the corresponding year - :type year: int - :param domain: CMOR domain - :type domain: str - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str - """ - # THREDDSManager does not require links - pass - def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN): """ @@ -222,30 +117,25 @@ class ObsReconManager(DataManager): def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, vartype=VariableType.MEAN, diagnostic=None): """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + Declare a variable chunk to be generated by a diagnostic - :param diagnostic: - :param region: - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str|None - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency|None - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str + Parameters + ---------- + domain: ModelingRealm + var: str + startdate: str + member: int + chunk: int + grid: str or None, optional + region: Basin or None, optional + box: Box or None, optional + frequency: Frequency or None, optional + vartype: VariableType, optional + diagnostic: Diagnostic, optional + + Returns + ------- + DataFile """ if frequency is None: frequency = self.config.frequency -- GitLab From ecb44cae9373eebcbb7d75d0a3dbe911df41b064 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:43:21 +0100 Subject: [PATCH 104/168] Cleaned earthdiags --- earthdiagnostics/earthdiags.py | 35 +++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index cbcbc568..db8a4320 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # coding=utf-8 +"""Entry point for EarthDiagnostics""" import argparse import os import shutil @@ -25,8 +26,9 @@ class EarthDiags(object): """ Launcher class for the diagnostics - :param config_file: path to the configuration file - :type config_file: str + Parameters + ---------- + config_file: str """ # Get the version number from the relevant file. If not, from autosubmit package scriptdir = os.path.abspath(os.path.dirname(__file__)) @@ -61,7 +63,9 @@ class EarthDiags(object): @staticmethod def parse_args(): """ - Entry point for the Earth Diagnostics. For more detailed documentation, use -h option + Entry point for the Earth Diagnostics. + + For more detailed documentation, use -h option """ # try: parser = argparse.ArgumentParser(description='Main executable for Earth Diagnostics.') @@ -142,6 +146,11 @@ class EarthDiags(object): def run(self): """ Run the diagnostics + + Returns + ------- + bool + """ self.had_errors = False Log.debug('Using netCDF version {0}', netCDF4.getlibversion()) @@ -197,6 +206,14 @@ class EarthDiags(object): self.data_manager.prepare() def clean(self): + """ + Clean scratch folder + + Returns + ------- + bool + + """ Log.info('Removing scratch folder...') self._remove_scratch_dir() Log.result('Scratch folder removed') @@ -210,6 +227,13 @@ class EarthDiags(object): shutil.rmtree(self.config.scratch_dir) def report(self): + """ + Create a report of missing variables for a given experiment + + Returns + ------- + bool + """ Log.info('Looking for existing vars...') self._prepare_data_manager() base_folder = self.config.report.path @@ -224,7 +248,7 @@ class EarthDiags(object): '{0}_{1}.report'.format(startdate, self.config.experiment.get_member_str(member))) - self.create_report(report_path, results) + self._create_report(report_path, results) Log.result('Report finished') return True @@ -248,7 +272,7 @@ class EarthDiags(object): return results @staticmethod - def create_report(report_path, results): + def _create_report(report_path, results): tables = set([result[1].name for result in results]) for table in tables: file_handler = open('{0}.{1}'.format(report_path, table), 'w') @@ -384,6 +408,7 @@ class EarthDiags(object): def main(): + """Main for earthdiagnostics""" if not EarthDiags.parse_args(): exit(1) -- GitLab From 9da6c87fd0d252b0a4e35ba032616e45f1c2bd0d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:46:36 +0100 Subject: [PATCH 105/168] Updated gyres doc --- earthdiagnostics/general/attribute.py | 2 +- earthdiagnostics/general/dailymean.py | 2 +- earthdiagnostics/general/module.py | 2 +- earthdiagnostics/general/monthlymean.py | 2 +- earthdiagnostics/general/relink.py | 2 +- earthdiagnostics/general/relinkall.py | 2 +- earthdiagnostics/general/rewrite.py | 2 +- earthdiagnostics/general/scale.py | 2 +- earthdiagnostics/general/select_levels.py | 4 +-- .../general/simplify_dimensions.py | 2 +- .../general/verticalmeanmetersiris.py | 2 +- earthdiagnostics/general/yearlymean.py | 2 +- earthdiagnostics/ocean/areamoc.py | 2 +- earthdiagnostics/ocean/averagesection.py | 2 +- earthdiagnostics/ocean/convectionsites.py | 2 +- earthdiagnostics/ocean/cutsection.py | 2 +- earthdiagnostics/ocean/gyres.py | 35 ++++++++++--------- earthdiagnostics/ocean/heatcontent.py | 2 +- earthdiagnostics/ocean/heatcontentlayer.py | 2 +- earthdiagnostics/ocean/interpolate.py | 2 +- earthdiagnostics/ocean/interpolatecdo.py | 2 +- earthdiagnostics/ocean/mask_land.py | 2 +- .../ocean/mixedlayerheatcontent.py | 2 +- .../ocean/mixedlayersaltcontent.py | 2 +- earthdiagnostics/ocean/moc.py | 2 +- earthdiagnostics/ocean/mxl.py | 2 +- earthdiagnostics/ocean/psi.py | 2 +- earthdiagnostics/ocean/regionmean.py | 2 +- earthdiagnostics/ocean/regionsum.py | 2 +- earthdiagnostics/ocean/rotation.py | 2 +- earthdiagnostics/ocean/siasiesiv.py | 2 +- earthdiagnostics/ocean/verticalgradient.py | 2 +- earthdiagnostics/ocean/verticalmean.py | 2 +- earthdiagnostics/ocean/verticalmeanmeters.py | 2 +- .../statistics/climatologicalpercentile.py | 2 +- .../statistics/daysoverpercentile.py | 2 +- earthdiagnostics/statistics/discretize.py | 2 +- .../statistics/monthlypercentile.py | 2 +- 38 files changed, 56 insertions(+), 55 deletions(-) diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 50b1cf8c..c7c52fa3 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -55,7 +55,7 @@ class Attribute(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py index e0db5c14..9c99b094 100644 --- a/earthdiagnostics/general/dailymean.py +++ b/earthdiagnostics/general/dailymean.py @@ -61,7 +61,7 @@ class DailyMean(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index 5257cbd3..0defb706 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -57,7 +57,7 @@ class Module(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index e7fa5c4e..f0812e8d 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -59,7 +59,7 @@ class MonthlyMean(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 8e0eddf2..464508cf 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -53,7 +53,7 @@ class Relink(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 56d1eb2e..0aa90ff0 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -32,7 +32,7 @@ class RelinkAll(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index 19ecfd5e..2cae2046 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -49,7 +49,7 @@ class Rewrite(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 40318ec2..3cb23d47 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -67,7 +67,7 @@ class Scale(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index f1069436..caa2a017 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -1,8 +1,8 @@ # coding=utf-8 +from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption, DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.box import Box class SelectLevels(Diagnostic): @@ -56,7 +56,7 @@ class SelectLevels(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index 27d1a512..7fe3002e 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -59,7 +59,7 @@ class SimplifyDimensions(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py index 1bc2dfe6..a2a20f54 100644 --- a/earthdiagnostics/general/verticalmeanmetersiris.py +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -58,7 +58,7 @@ class VerticalMeanMetersIris(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py index 964c570b..c64e3511 100644 --- a/earthdiagnostics/general/yearlymean.py +++ b/earthdiagnostics/general/yearlymean.py @@ -61,7 +61,7 @@ class YearlyMean(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 23e69e17..6b66b1cc 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -64,7 +64,7 @@ class AreaMoc(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 40ce55f8..7047099f 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -58,7 +58,7 @@ class AverageSection(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index dc5cad88..5e23cd49 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -51,7 +51,7 @@ class ConvectionSites(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index f2e9bb8c..bf4e6d39 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -71,7 +71,7 @@ class CutSection(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index 377e630e..942a2467 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute the intensity of the subtropical and subpolar gyres""" import numpy as np from bscearth.utils.log import Log @@ -52,7 +53,7 @@ class Gyres(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags @@ -83,14 +84,14 @@ class Gyres(Diagnostic): Models.NEMO_3_2_O1L42, Models.NEMO_3_3_O1L46, Models.NEMOVAR_O1L42]: - subpolNAtl = [230, 275, 215, 245] - subpolNPac = [70, 145, 195, 235] - subtropNPac = [45, 175, 165, 220] - subtropNAtl = [195, 275, 175, 225] - subtropSPac = [70, 205, 120, 145] - subtropSAtl = [235, 300, 120, 145] - subtropInd = [320, 30, 110, 180] - ACC = [1, 361, 1, 65] + subpol_n_atl = [230, 275, 215, 245] + subpol_n_pac = [70, 145, 195, 235] + subtrop_n_pac = [45, 175, 165, 220] + subtrop_n_atl = [195, 275, 175, 225] + subtrop_s_pac = [70, 205, 120, 145] + subtrop_s_atl = [235, 300, 120, 145] + subtrop_ind = [320, 30, 110, 180] + acc = [1, 361, 1, 65] elif self.model_version in [Models.ECEARTH_3_0_O25L46, Models.ECEARTH_3_0_O25L75, Models.GLORYS2_V1_O25L75, Models.ECEARTH_3_2_O1L75, Models.ECEARTH_3_2_O25L75]: @@ -117,35 +118,35 @@ class Gyres(Diagnostic): var_gyre.units = 'm^3/s' var_region[0] = 'subpolNAtl' - var_gyre[:, 0] = self._gyre(subpolNAtl, True) + var_gyre[:, 0] = self._gyre(subpol_n_atl, True) Log.debug('subpolNAtl: {0}', var_gyre[:, 0]) var_region[1] = 'subpolNPac' - var_gyre[:, 1] = self._gyre(subpolNPac, True) + var_gyre[:, 1] = self._gyre(subpol_n_pac, True) Log.debug('subpolNPac: {0}', var_gyre[:, 1]) var_region[2] = 'subtropNPac' - var_gyre[:, 2] = self._gyre(subtropNPac) + var_gyre[:, 2] = self._gyre(subtrop_n_pac) Log.debug('subtropNPac: {0}', var_gyre[:, 2]) var_region[3] = 'subtropSPac' - var_gyre[:, 3] = self._gyre(subtropSPac) + var_gyre[:, 3] = self._gyre(subtrop_s_pac) Log.debug('subtropSPac: {0}', var_gyre[:, 3]) var_region[4] = 'subtropNAtl' - var_gyre[:, 4] = self._gyre(subtropNAtl) + var_gyre[:, 4] = self._gyre(subtrop_n_atl) Log.debug('subtropNAtl: {0}', var_gyre[:, 4]) var_region[5] = 'subtropSAtl' - var_gyre[:, 5] = self._gyre(subtropSAtl) + var_gyre[:, 5] = self._gyre(subtrop_s_atl) Log.debug('subtropSAtl: {0}', var_gyre[:, 5]) var_region[6] = 'subtropInd' - var_gyre[:, 6] = self._gyre(subtropInd) + var_gyre[:, 6] = self._gyre(subtrop_ind) Log.debug('subtropInd: {0}', var_gyre[:, 6]) var_region[7] = 'ACC' - var_gyre[:, 7] = self._gyre(ACC) + var_gyre[:, 7] = self._gyre(acc) Log.debug('ACC: {0}', var_gyre[:, 7]) handler.close() diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 3e65830e..cb122f33 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -63,7 +63,7 @@ class HeatContent(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 33473bff..1d87e1f4 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -54,7 +54,7 @@ class HeatContentLayer(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 159ceb6c..7804c0b6 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -75,7 +75,7 @@ class Interpolate(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 83004c12..7b62b8d1 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -79,7 +79,7 @@ class InterpolateCDO(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index c95a3a93..1f09b522 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -49,7 +49,7 @@ class MaskLand(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index 327d42ad..a61c13d7 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -48,7 +48,7 @@ class MixedLayerHeatContent(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index 8aa803cc..924a5d20 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -47,7 +47,7 @@ class MixedLayerSaltContent(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index ad3299cb..a94e1e00 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -51,7 +51,7 @@ class Moc(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py index d11a1893..bc2ee396 100644 --- a/earthdiagnostics/ocean/mxl.py +++ b/earthdiagnostics/ocean/mxl.py @@ -39,7 +39,7 @@ class Mxl(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index 11f41938..9773e02c 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -45,7 +45,7 @@ class Psi(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index ca43049f..77cc6cbc 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -68,7 +68,7 @@ class RegionMean(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index 4d6435e2..d3881492 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -67,7 +67,7 @@ class RegionSum(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index a71a4938..0f2c72ed 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -56,7 +56,7 @@ class Rotation(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 3f0e9f3d..320defad 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -67,7 +67,7 @@ class Siasiesiv(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 35cea5b2..8331db7b 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -55,7 +55,7 @@ class VerticalGradient(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index f8f2253a..1b3f8624 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -56,7 +56,7 @@ class VerticalMean(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 2b72117a..9cd3833c 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -56,7 +56,7 @@ class VerticalMeanMeters(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 0b53aed4..20d5ad5d 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -57,7 +57,7 @@ class ClimatologicalPercentile(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index beb2235b..10771ecf 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -53,7 +53,7 @@ class DaysOverPercentile(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index 004628e9..b762280b 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -94,7 +94,7 @@ class Discretize(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index ad1466c0..53a2bea6 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -50,7 +50,7 @@ class MonthlyPercentile(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags -- GitLab From 3858f8df81ad47f37359bd7eff3065e70982c378 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 12:58:34 +0100 Subject: [PATCH 106/168] Updated variable doc and minor refactoring --- earthdiagnostics/variable.py | 127 +++++++++++++++++++++++------------ 1 file changed, 85 insertions(+), 42 deletions(-) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index fe5e0c57..23dd29d1 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -76,7 +76,7 @@ class VariableManager(object): def load_variables(self, table_name): """ - Loads the CMOR csv and creates the variables dictionary + Load the CMOR csv and creates the variables dictionary Parameters ---------- @@ -178,9 +178,7 @@ class VariableManager(object): if line[0] == 'Aliases': continue - aliases = line[0].split(':') - if line[1] not in aliases: - aliases.append(line[1]) + aliases = self._get_aliases(line) cmor_vars = [] for alias in aliases: @@ -201,19 +199,28 @@ class VariableManager(object): Log.error('Aliases {0} can be be mapped to multiple variables ' '[{1}]'.format(aliases, ', '.join(map(str, cmor_vars)))) continue - cmor_var = cmor_vars[0] - for alias in aliases: - if alias != cmor_var.short_name and alias in self._dict_variables: - Log.error('Alias {0} for variable {1} is already a different ' - 'variable!'.format(alias, cmor_var.short_name)) - continue - alias_object = VariableAlias(alias) - if line[2]: - alias_object.basin = Basins().parse(line[2]) - if line[3]: - alias_object.grid = line[3] - cmor_var.known_aliases.append(alias_object) + cmor_var = cmor_vars[0] + self._register_aliases(aliases, cmor_var, line) + + def _get_aliases(self, line): + aliases = line[0].split(':') + if line[1] not in aliases: + aliases.append(line[1]) + return aliases + + def _register_aliases(self, aliases, cmor_var, line): + for alias in aliases: + if alias != cmor_var.short_name and alias in self._dict_variables: + Log.error('Alias {0} for variable {1} is already a different ' + 'variable!'.format(alias, cmor_var.short_name)) + continue + alias_object = VariableAlias(alias) + if line[2]: + alias_object.basin = Basins().parse(line[2]) + if line[3]: + alias_object.grid = line[3] + cmor_var.known_aliases.append(alias_object) def _get_aliases_csv_path(self, filename): csv_table_path = os.path.join(self._aliases_folder, '{0}.csv'.format(filename)) @@ -261,35 +268,35 @@ class VariableManager(object): for row in sheet.rows: if row[0].value == 'Priority' or not row[5].value: continue - - cmor_name = row[11].value - if not cmor_name: - cmor_name = row[5].value - - priority = int(row[0].value) - bsc_commitment = row[30].value - if bsc_commitment is not None and bsc_commitment.strip().lower() == 'false': - priority = priority + 3 - - if cmor_name.lower() in self._dict_variables: - var = self._dict_variables[cmor_name.lower()] - else: - var = Variable() - var.short_name = cmor_name - var.standard_name = row[6].value - var.long_name = row[1].value - - var.domain = self._process_modelling_realm(var, row[12].value) - - var.units = row[2].value - self._dict_variables[var.short_name.lower()] = var - var.add_table(table, priority) + self._parse_xlsx_var_row(row, table) except Exception as ex: Log.error('Table {0} can not be loaded: {1}', sheet_name, ex) import traceback traceback.print_exc() + def _parse_xlsx_var_row(self, row, table): + cmor_name = row[11].value + if not cmor_name: + cmor_name = row[5].value + priority = int(row[0].value) + bsc_commitment = row[30].value + if bsc_commitment is not None and bsc_commitment.strip().lower() == 'false': + priority = priority + 3 + if cmor_name.lower() in self._dict_variables: + var = self._dict_variables[cmor_name.lower()] + else: + var = Variable() + var.short_name = cmor_name + var.standard_name = row[6].value + var.long_name = row[1].value + + var.domain = self._process_modelling_realm(var, row[12].value) + + var.units = row[2].value + self._dict_variables[var.short_name.lower()] = var + var.add_table(table, priority) + @staticmethod def _process_modelling_realm(var, value): if value is None: @@ -329,14 +336,35 @@ class Variable(object): self.tables = [] def add_table(self, table, priority=None): + """ + Add table to variable + + Parameters + ---------- + table: CMORTable + priority: int or None, optional + + """ self.tables.append((table, priority)) - def parse_json(self, json_var, key): + def parse_json(self, json_var, variable): + """ + Parse variable json + + Parameters + ---------- + json_var: dict of str: str + variable: str + + Returns + ------- + + """ if 'out_name' in json_var: self.short_name = json_var['out_name'].strip() else: - raise VariableJsonException('Variable {0} has no out name defined'.format(key)) + raise VariableJsonException('Variable {0} has no out name defined'.format(variable)) self.standard_name = json_var['standard_name'].strip() self.long_name = json_var['long_name'].strip() @@ -354,6 +382,18 @@ class Variable(object): self.priority = 1 def get_modelling_realm(self, domains): + """ + Get var modelling realm + + Parameters + ---------- + domains: iterable of str + + Returns + ------- + ModelingRealm or None + + """ if len(domains) > 1: Log.warning('Multiple modeling realms assigned to variable {0}: {1}. ', self, domains) parsed = [] @@ -443,12 +483,15 @@ class Variable(object): class VariableAlias(object): """ - Class to characterize a CMOR variable. It also contains the static method to make the match between thje original + Class to characterize a CMOR variable. + + It also contains the static method to make the match between the original name and the standard name. Requires data _convetion to be available in cmor_tables to work. Parameters ---------- alias: str + """ def __init__(self, alias): -- GitLab From e3bdd01143c29384f9c4dc4609282f82d702d9f6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 13:27:09 +0100 Subject: [PATCH 107/168] Test prospector convention numpy --- .prospector.yml | 4 ++++ earthdiagnostics/variable.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.prospector.yml b/.prospector.yml index 142832e4..bd8477ca 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -12,6 +12,9 @@ member-warnings: false pyroma: run: true +pydocroma: + run: true + pep8: disable: [ E501, # Line-length, already controlled by pylint @@ -19,6 +22,7 @@ pep8: pep257: # see http://pep257.readthedocs.io/en/latest/error_codes.html + convention: numpy disable: [ # For short descriptions it makes sense not to end with a period: D400, # First line should end with a period diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 23dd29d1..a155e3b8 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -203,7 +203,8 @@ class VariableManager(object): cmor_var = cmor_vars[0] self._register_aliases(aliases, cmor_var, line) - def _get_aliases(self, line): + @staticmethod + def _get_aliases(line): aliases = line[0].split(':') if line[1] not in aliases: aliases.append(line[1]) @@ -360,7 +361,6 @@ class Variable(object): ------- """ - if 'out_name' in json_var: self.short_name = json_var['out_name'].strip() else: -- GitLab From 7ff689b3cd8bdf4be39b6cf27f0364785bc52307 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 14:40:30 +0100 Subject: [PATCH 108/168] Changed prospector config --- .prospector.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.prospector.yml b/.prospector.yml index bd8477ca..b9c6fa95 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -21,16 +21,16 @@ pep8: ] pep257: + run: true # see http://pep257.readthedocs.io/en/latest/error_codes.html - convention: numpy - disable: [ - # For short descriptions it makes sense not to end with a period: - D400, # First line should end with a period - # Disable because not part of PEP257 official convention: - D203, # 1 blank line required before class docstring - D212, # Multi-line docstring summary should start at the first line - D213, # Multi-line docstring summary should start at the second line - D404, # First word of the docstring should not be This - D107, # We are using numpy style and constructor should be documented in class docstring - D105, # Docstring in magic methods should not be required: we all now what they are for - ] + disable: [ + # For short descriptions it makes sense not to end with a period: + D400, # First line should end with a period + # Disable because not part of PEP257 official convention: + D203, # 1 blank line required before class docstring + D212, # Multi-line docstring summary should start at the first line + D213, # Multi-line docstring summary should start at the second line + D404, # First word of the docstring should not be This + D107, # We are using numpy style and constructor should be documented in class docstring + D105, # Docstring in magic methods should not be required: we all now what they are for + ] -- GitLab From cc0d646f291d16d5f5829ace1dfe2553b122ec94 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 15:23:04 +0100 Subject: [PATCH 109/168] Updated constant doc --- earthdiagnostics/constants.py | 19 ++++--- earthdiagnostics/modelingrealm.py | 82 +++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 25 deletions(-) diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index d7935838..f38c39fb 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -1,7 +1,5 @@ # coding=utf-8 -""" -Contains the enumeration-like classes used by the diagnostics -""" +"""Contains the enumeration-like classes used by the diagnostics""" from singleton import SingletonType @@ -30,7 +28,8 @@ class Basin(object): @property def name(self): """ - Basin's full name + Basin full name + :rtype: str """ return self._name @@ -131,6 +130,7 @@ class Basins(object): def get_available_basins(self, handler): """ + Read available basins from file :param handler: :type handler: netCDF4.Dataset @@ -156,9 +156,10 @@ class Basins(object): def parse(self, basin): """ - Return the basin matching the given name. If the parameter basin is a Basin instance, directly returns the same - instance. This bahaviour is intended to facilitate the development of methods that can either accept a name - or a Basin instance to characterize the basin. + Return the basin matching the given name. + If the parameter basin is a Basin instance, directly returns the same + instance. This bahaviour is intended to facilitate the development of + methods that can either accept a nameor a Basin instance to characterize the basin. :param basin: basin name or basin instance :type basin: str | Basin @@ -175,9 +176,7 @@ class Basins(object): class Models(object): - """ - Predefined models - """ + """Predefined models""" ECEARTH_2_3_O1L42 = 'Ec2.3_O1L42' """ EC-Earth 2.3 ORCA1 L42""" diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index 11d655f6..1502b55d 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -1,8 +1,21 @@ # coding=utf-8 +"""Class to manage variable domains""" from earthdiagnostics.frequency import Frequencies class ModelingRealm(object): + """ + Class to represent Modeling Relms for variables + + Parameters + ---------- + domain_name: str + + Raises + ------ + ValueError + If the domain is not supported + """ def __init__(self, domain_name): lower_name = domain_name.lower() @@ -36,13 +49,17 @@ class ModelingRealm(object): def get_table_name(self, frequency, data_convention): """ - Returns the table name for a domain-frequency pair - :param data_convention: Data convention in use - :type data_convention: str - :param frequency: variable's frequency - :type frequency: Frequency - :return: variable's table name - :rtype: str + Get table name for a domain-frequency pair + + Parameters + ---------- + data_convention: str + frequency: Frequency + + Returns + ------- + str + """ if self.name == 'seaIce': if data_convention in ('specs', 'preface'): @@ -65,12 +82,35 @@ class ModelingRealm(object): return table_name def get_table(self, frequency, data_convention): + """ + Get table name for a domain-frequency pair + + Parameters + ---------- + data_convention: str + frequency: Frequency + + Returns + ------- + str + + Parameters + ---------- + frequency + data_convention + + Returns + ------- + CMORTable + + """ table_name = self.get_table_name(frequency, data_convention) from earthdiagnostics.variable import CMORTable return CMORTable(table_name, frequency, 'December 2013') class ModelingRealms(object): + """Enumeration of supported modelling realms""" seaIce = ModelingRealm('seaice') ocean = ModelingRealm('ocean') landIce = ModelingRealm('landIce') @@ -83,14 +123,26 @@ class ModelingRealms(object): @classmethod def parse(cls, modelling_realm): """ - Return the basin matching the given name. If the parameter basin is a Basin instance, directly returns the same - instance. This bahaviour is intended to facilitate the development of methods that can either accept a name - or a Basin instance to characterize the basin. - - :param modelling_realm: basin name or basin instance - :type modelling_realm: str | Basin - :return: basin instance corresponding to the basin name - :rtype: Basin + Return the basin matching the given name. + + If the parameter modelling_realm is a ModelingRealm instance, directly returns the same + instance. This behaviour is intended to facilitate the development of + methods that can either accept a name or a ModelingRealm instance to + characterize the modelling realm. + + Parameters + ---------- + modelling_realm: ModelingRealm or str + + Returns + ------- + ModelingRealm + + Raises + ------ + ValueError + If the modelling realm is not supported + """ if isinstance(modelling_realm, ModelingRealm): return modelling_realm -- GitLab From bb92d6e19fa09b3244a3533e08d56de54905f000 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 15:28:04 +0100 Subject: [PATCH 110/168] Fixed issues in cmorizer --- earthdiagnostics/cmorizer.py | 83 +++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 2a7a8c64..affeceb3 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -186,7 +186,7 @@ class Cmorizer(object): shutil.move(merged_sh, os.path.join(self.cmor_scratch, 'MMASH_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) def cmorize_atmos(self): - """CMORizes atmospheric data, from grib or MMA files""" + """Cmorize atmospheric data, from grib or MMA files""" if not self.cmor.atmosphere: Log.info('Skipping atmosphere cmorization due to configuration') return @@ -549,40 +549,11 @@ class Cmorizer(object): new_units = None cdo_operator = '-selmon,{0}'.format(month) - if frequency == Frequencies.monthly: - if var_code == 201: - cdo_operator = "-monmean -daymax {0}".format(cdo_operator) - elif var_code == 202: - cdo_operator = "-monmean -daymax {0}".format(cdo_operator) - else: - cdo_operator = "-monmean {0} ".format(cdo_operator) - if frequency == Frequencies.daily: - if var_code == 201: - cdo_operator = "-daymax {0} ".format(cdo_operator) - elif var_code == 202: - cdo_operator = "-daymin {0} ".format(cdo_operator) - else: - cdo_operator = "-daymean {0} ".format(cdo_operator) - - if var_code in (144, 146, 147, 169, 175, 176, 177, 179, 180, 181, 182, 201, 202, 205, 212, 228): - cdo_operator = '{0} -shifttime,-{1}hours'.format(cdo_operator, self.experiment.atmos_timestep) - - if var_code == 129: - # geopotential - new_units = "m" - cdo_operator = "-divc,9.81 {0}".format(cdo_operator) - elif var_code in (146, 147, 169, 175, 176, 177, 179, 212): - # radiation - new_units = "W m-2" - cdo_operator = "-divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) - elif var_code in (180, 181): - # momentum flux - new_units = "N m-2" - cdo_operator = "-divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) - elif var_code in (144, 182, 205, 228): - # precipitation/evaporation/runoff - new_units = "kg m-2 s-1" - cdo_operator = "-mulc,1000 -divc,{0}".format(self.experiment.atmos_timestep * 3600) + cdo_operator = self._get_time_average(cdo_operator, frequency, var_code) + + cdo_operator = self._fix_time_shift(cdo_operator, var_code) + + cdo_operator, new_units = self._change_units(cdo_operator, new_units, var_code) levels = self.config.cmor.get_levels(frequency, var_code) if levels: @@ -612,6 +583,48 @@ class Cmorizer(object): output='{0}_{1}_1m.nc'.format(gribfile, var_code), options=('-O -v {0}'.format(var_name))) + def _fix_time_shift(self, cdo_operator, var_code): + if var_code in (144, 146, 147, 169, 175, 176, 177, 179, 180, 181, 182, 201, 202, 205, 212, 228): + cdo_operator = '{0} -shifttime,-{1}hours'.format(cdo_operator, self.experiment.atmos_timestep) + return cdo_operator + + @staticmethod + def _get_time_average(cdo_operator, frequency, var_code): + if frequency == Frequencies.monthly: + if var_code == 201: + cdo_operator = "-monmean -daymax {0}".format(cdo_operator) + elif var_code == 202: + cdo_operator = "-monmean -daymax {0}".format(cdo_operator) + else: + cdo_operator = "-monmean {0} ".format(cdo_operator) + if frequency == Frequencies.daily: + if var_code == 201: + cdo_operator = "-daymax {0} ".format(cdo_operator) + elif var_code == 202: + cdo_operator = "-daymin {0} ".format(cdo_operator) + else: + cdo_operator = "-daymean {0} ".format(cdo_operator) + return cdo_operator + + def _change_units(self, cdo_operator, new_units, var_code): + if var_code == 129: + # geopotential + new_units = "m" + cdo_operator = "-divc,9.81 {0}".format(cdo_operator) + elif var_code in (146, 147, 169, 175, 176, 177, 179, 212): + # radiation + new_units = "W m-2" + cdo_operator = "-divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) + elif var_code in (180, 181): + # momentum flux + new_units = "N m-2" + cdo_operator = "-divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) + elif var_code in (144, 182, 205, 228): + # precipitation/evaporation/runoff + new_units = "kg m-2 s-1" + cdo_operator = "-mulc,1000 -divc,{0}".format(self.experiment.atmos_timestep * 3600) + return cdo_operator, new_units + def _merge_and_cmorize_atmos(self, chunk_start, chunk_end, grid, frequency): merged_file = 'MMA_{0}_{1}_{2}_{3}.nc'.format(frequency, date2str(chunk_start), date2str(chunk_end), grid) files = glob.glob(os.path.join(self.config.scratch_dir, -- GitLab From 017ff8e2bcdde0aea013960427e2b37a1a2929d9 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 14 Dec 2017 17:57:24 +0100 Subject: [PATCH 111/168] Extracted TimeMean base class for all time mean diagnostics --- earthdiagnostics/cmormanager.py | 24 ++- earthdiagnostics/general/dailymean.py | 109 ---------- earthdiagnostics/general/monthlymean.py | 111 ----------- earthdiagnostics/general/timemean.py | 242 +++++++++++++++++++++++ earthdiagnostics/general/yearlymean.py | 111 ----------- earthdiagnostics/ocean/interpolatecdo.py | 2 + earthdiagnostics/work_manager.py | 4 +- test/unit/general/test_dailymean.py | 8 +- test/unit/general/test_monthlymean.py | 18 +- test/unit/general/test_yearlymean.py | 16 +- 10 files changed, 272 insertions(+), 373 deletions(-) delete mode 100644 earthdiagnostics/general/dailymean.py delete mode 100644 earthdiagnostics/general/monthlymean.py create mode 100644 earthdiagnostics/general/timemean.py delete mode 100644 earthdiagnostics/general/yearlymean.py diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index a7e85dee..9682cd2f 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -280,16 +280,7 @@ class CMORManager(DataManager): else: cmor_table = cmor_var.get_table(frequency, self.config.data_convention) - if chunk is not None: - time_bound = self._get_chunk_time_bounds(startdate, chunk) - elif year: - if frequency != Frequencies.yearly: - raise ValueError('Year may be provided instead of chunk only if frequency is "yr"') - time_bound = str(year) - elif date_str: - time_bound = date_str - else: - raise ValueError('Chunk, year and date_str can not be None at the same time') + time_bound = self._get_time_component(chunk, date_str, frequency, startdate, year) if time_bound: time_bound = '_{0}.nc'.format(time_bound) @@ -318,6 +309,19 @@ class CMORManager(DataManager): raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name + def _get_time_component(self, chunk, date_str, frequency, startdate, year): + if chunk is not None: + time_bound = self._get_chunk_time_bounds(startdate, chunk) + elif year: + if frequency != Frequencies.yearly: + raise ValueError('Year may be provided instead of chunk only if frequency is "yr"') + time_bound = str(year) + elif date_str: + time_bound = date_str + else: + raise ValueError('Chunk, year and date_str can not be None at the same time') + return time_bound + def _get_full_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): if self.config.data_convention in ('specs', 'preface'): folder_path = os.path.join(self._get_startdate_path(startdate), str(frequency), domain.name, var) diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py deleted file mode 100644 index 9c99b094..00000000 --- a/earthdiagnostics/general/dailymean.py +++ /dev/null @@ -1,109 +0,0 @@ -# coding=utf-8 - -import os - -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ - DiagnosticFrequencyOption, DiagnosticVariableOption -from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.utils import Utils, TempFile - - -class DailyMean(Diagnostic): - """ - Calculates daily mean for a given variable - - :original author: Javier Vegas-Regidor - - :created: July 2016 - - :param data_manager: data management object - :type data_manager: DataManager - :param startdate: startdate - :type startdate: str - :param member: member number - :type member: int - :param chunk: chunk's number - :type chunk: int - :param variable: variable's name - :type variable: str - :param domain: variable's domain - :type domain: ModelingRealm - :param frequency: original frequency - :type frequency: str - :param grid: original data grid - :type grid: str - """ - - alias = 'daymean' - "Diagnostic alias for the configuration file" - - def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.frequency = frequency - self.grid = grid - - def __str__(self): - return 'Calculate daily mean Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4} Original frequency: {5} Grid: {6}'.format(self.startdate, self.member, self.chunk, - self.domain, self.variable, - self.frequency, self.grid) - - def __eq__(self, other): - return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ - self.grid == other.grid - - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, frequency=day, grid='' - :type options: list[str] - :return: - """ - - options_available = (DiagnosticDomainOption(), - DiagnosticVariableOption(diags.data_manager.config.var_manager), - DiagnosticFrequencyOption(), - DiagnosticOption('grid', '')) - options = cls.process_options(options, options_available) - job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(DailyMean(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['frequency'], options['grid'])) - return job_list - - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=self.frequency, grid=self.grid) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.daymean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=Frequencies.daily, grid=self.grid) - - def compute(self): - """Run the diagnostic""" - temp = TempFile.get() - handler = Utils.open_cdf(self.variable_file.local_file) - if 'region' in handler.variables: - noregion = TempFile.get() - Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) - Utils.cdo.daymean(input=noregion, output=temp) - os.remove(noregion) - monmean_handler = Utils.open_cdf(temp) - Utils.copy_variable(handler, monmean_handler, 'region') - monmean_handler.close() - else: - Utils.cdo.daymean(input=self.variable_file.local_file, output=temp) - self.daymean.set_local_file(temp) - diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py deleted file mode 100644 index f0812e8d..00000000 --- a/earthdiagnostics/general/monthlymean.py +++ /dev/null @@ -1,111 +0,0 @@ -# coding=utf-8 - -import os - -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ - DiagnosticFrequencyOption, DiagnosticVariableOption -from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.utils import Utils, TempFile - - -class MonthlyMean(Diagnostic): - """ - Calculates monthly mean for a given variable - - :original author: Javier Vegas-Regidor - - :created: July 2016 - - :param data_manager: data management object - :type data_manager: DataManager - :param startdate: startdate - :type startdate: str - :param member: member number - :type member: int - :param chunk: chunk's number - :type chunk: int - :param variable: variable's name - :type variable: str - :param domain: variable's domain - :type domain: ModelingRealm - :param frequency: original frequency - :type frequency: Frequency - :param grid: original data grid - :type grid: str - """ - - alias = 'monmean' - "Diagnostic alias for the configuration file" - - def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.frequency = frequency - self.grid = grid - - def __str__(self): - return 'Calculate monthly mean Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable) - - def __eq__(self, other): - return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ - self.grid == other.grid - - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, frequency=day, grid='' - :type options: list[str] - :return: - """ - - options_available = (DiagnosticDomainOption(), - DiagnosticVariableOption(diags.data_manager.config.var_manager), - DiagnosticFrequencyOption('frequency', Frequencies.daily), - DiagnosticOption('grid', '')) - options = cls.process_options(options, options_available) - job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['frequency'], options['grid'])) - - return job_list - - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=self.frequency, grid=self.grid) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.monmean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=Frequencies.monthly, grid=self.grid) - - def compute(self): - """Run the diagnostic""" - handler = Utils.open_cdf(self.variable_file.local_file) - temp = TempFile.get() - if 'region' in handler.variables: - noregion = TempFile.get() - Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) - Utils.cdo.monmean(input=noregion, output=temp) - os.remove(noregion) - monmean_handler = Utils.open_cdf(temp) - Utils.copy_variable(handler, monmean_handler, 'region') - monmean_handler.close() - else: - Utils.cdo.monmean(input=self.variable_file.local_file, output=temp) - handler.close() - self.monmean.set_local_file(temp) - - - diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py new file mode 100644 index 00000000..b0de54ed --- /dev/null +++ b/earthdiagnostics/general/timemean.py @@ -0,0 +1,242 @@ +# coding=utf-8 + +import os + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ + DiagnosticFrequencyOption, DiagnosticVariableOption +from frequency import Frequencies +from utils import TempFile, Utils + + +class TimeMean(Diagnostic): + """ + Base class for all time mean diagnostics + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: str + :param grid: original data grid + :type grid: str + """ + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.frequency = frequency + self.grid = grid + self._target_frequency = None + + def __str__(self): + return 'Calculate {0._target_frequency} mean Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Variable: {0.domain}:{0.variable} Original frequency: {0.frequency} Grid: {0.grid}'.format(self) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.grid == other.grid and self._target_frequency == other._target_frequency + + @classmethod + def _process_options(cls, diags, options): + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFrequencyOption(), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + return options + + def request_data(self): + """Request data required by the diagnostic""" + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.daymean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.daily, grid=self.grid) + + def compute_mean(self, input_file, output_file): + """Method to actually compute the mean""" + raise NotImplementedError() + + def compute(self): + """Run the diagnostic""" + temp = TempFile.get() + handler = Utils.open_cdf(self.variable_file.local_file) + if 'region' in handler.variables: + noregion = TempFile.get() + Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) + self.compute_mean(noregion, temp) + os.remove(noregion) + monmean_handler = Utils.open_cdf(temp) + Utils.copy_variable(handler, monmean_handler, 'region') + monmean_handler.close() + else: + self.compute_mean(self.variable_file.local_file, temp) + self.daymean.set_local_file(temp) + + +class DailyMean(TimeMean): + """ + Calculates daily mean for a given variable + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: str + :param grid: original data grid + :type grid: str + """ + + alias = 'daymean' + "Diagnostic alias for the configuration file" + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) + self._target_frequency = 'daily' + + @classmethod + def generate_jobs(cls, diags, options): + """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options = cls._process_options(diags, options) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(DailyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + + def compute_mean(self, input_file, output_file): + Utils.cdo.daymean(input=input_file, output=output_file) + + +class MonthlyMean(TimeMean): + """ + Calculates monthly mean for a given variable + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: str + :param grid: original data grid + :type grid: str + """ + + alias = 'monmean' + "Diagnostic alias for the configuration file" + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) + self._target_frequency = 'monthly' + + @classmethod + def generate_jobs(cls, diags, options): + """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options = cls._process_options(diags, options) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + + def compute_mean(self, input_file, output_file): + Utils.cdo.monmean(input=input_file, output=output_file) + + +class YearlyMean(TimeMean): + """ + Calculates monthly mean for a given variable + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: str + :param grid: original data grid + :type grid: str + """ + + alias = 'yearmean' + "Diagnostic alias for the configuration file" + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) + self._target_frequency = 'yearly' + + @classmethod + def generate_jobs(cls, diags, options): + """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options = cls._process_options(diags, options) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(YearlyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + + def compute_mean(self, input_file, output_file): + Utils.cdo.monmean(input=input_file, output=output_file) + + diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py deleted file mode 100644 index c64e3511..00000000 --- a/earthdiagnostics/general/yearlymean.py +++ /dev/null @@ -1,111 +0,0 @@ -# coding=utf-8 - -import os - -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ - DiagnosticFrequencyOption, DiagnosticVariableOption -from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.utils import Utils, TempFile - - -class YearlyMean(Diagnostic): - """ - Calculates yearly mean for a given variable - - :original author: Javier Vegas-Regidor - - :created: July 2016 - - :param data_manager: data management object - :type data_manager: DataManager - :param startdate: startdate - :type startdate: str - :param member: member number - :type member: int - :param chunk: chunk's number - :type chunk: int - :param variable: variable's name - :type variable: str - :param domain: variable's domain - :type domain: ModelingRealm - :param frequency: original frequency - :type frequency: str - :param grid: original data grid - :type grid: str - """ - - alias = 'yearmean' - "Diagnostic alias for the configuration file" - - def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.frequency = frequency - self.grid = grid - - def __str__(self): - return 'Calculate yearly mean Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4} Original frequency: {5} Grid: {6}'.format(self.startdate, self.member, self.chunk, - self.domain, self.variable, - self.frequency, self.grid) - - def __eq__(self, other): - return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ - self.grid == other.grid - - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, frequency=day, grid='' - :type options: list[str] - :return: - """ - - options_available = (DiagnosticDomainOption(), - DiagnosticVariableOption(diags.data_manager.config.var_manager), - DiagnosticFrequencyOption(default_value=diags.config.frequency), - DiagnosticOption('grid', '')) - options = cls.process_options(options, options_available) - job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(YearlyMean(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['frequency'], options['grid'])) - return job_list - - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=self.frequency, grid=self.grid) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.yearmean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=Frequencies.yearly, grid=self.grid) - - def compute(self): - """Run the diagnostic""" - temp = TempFile.get() - - handler = Utils.open_cdf(self.variable_file.local_file) - if 'region' in handler.variables: - noregion = TempFile.get() - Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) - Utils.cdo.yearmean(input=noregion, output=temp) - monmean_handler = Utils.open_cdf(temp) - Utils.copy_variable(handler, monmean_handler, 'region') - monmean_handler.close() - else: - Utils.cdo.yearmean(input=self.variable_file.local_file, output=temp) - os.remove(self.variable_file.local_file) - - self.yearmean.set_local_file(temp) - diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 7b62b8d1..8bdb634c 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""CDO-based interpolation""" import os import numpy as np @@ -13,6 +14,7 @@ from earthdiagnostics.utils import Utils, TempFile class InterpolateCDO(Diagnostic): """ 3-dimensional conservative interpolation to the regular atmospheric grid. + It can also be used for 2D (i,j) variables :original author: Javier Vegas-Regidor diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index a6fa51e7..6326d004 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -250,10 +250,8 @@ class WorkManager(object): @staticmethod def _register_general_diagnostics(): from earthdiagnostics.general.attribute import Attribute - from earthdiagnostics.general.dailymean import DailyMean + from earthdiagnostics.general.timemean import DailyMean, MonthlyMean, YearlyMean from earthdiagnostics.general.module import Module - from earthdiagnostics.general.monthlymean import MonthlyMean - from earthdiagnostics.general.yearlymean import YearlyMean from earthdiagnostics.general.rewrite import Rewrite from earthdiagnostics.general.relink import Relink from earthdiagnostics.general.relinkall import RelinkAll diff --git a/test/unit/general/test_dailymean.py b/test/unit/general/test_dailymean.py index ec85d9dc..e076a247 100644 --- a/test/unit/general/test_dailymean.py +++ b/test/unit/general/test_dailymean.py @@ -1,12 +1,12 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.diagnostic import DiagnosticVariableOption -from earthdiagnostics.box import Box -from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.general.dailymean import DailyMean from mock import Mock, patch +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.timemean import DailyMean from earthdiagnostics.modelingrealm import ModelingRealms diff --git a/test/unit/general/test_monthlymean.py b/test/unit/general/test_monthlymean.py index b31561ac..401a8804 100644 --- a/test/unit/general/test_monthlymean.py +++ b/test/unit/general/test_monthlymean.py @@ -1,12 +1,12 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.diagnostic import DiagnosticVariableOption -from earthdiagnostics.box import Box -from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.general.monthlymean import MonthlyMean from mock import Mock, patch +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.timemean import MonthlyMean from earthdiagnostics.modelingrealm import ModelingRealms @@ -30,14 +30,6 @@ class TestMonthlyMean(TestCase): @patch.object(DiagnosticVariableOption, 'parse', fake_parse) def test_generate_jobs(self): - - jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) - self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - Frequencies.daily, '')) - self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - Frequencies.daily, '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'monthly']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', @@ -60,4 +52,4 @@ class TestMonthlyMean(TestCase): def test_str(self): self.assertEquals(str(self.mixed), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var') + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_yearlymean.py b/test/unit/general/test_yearlymean.py index dcf5ad75..bd1b7938 100644 --- a/test/unit/general/test_yearlymean.py +++ b/test/unit/general/test_yearlymean.py @@ -1,12 +1,12 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.diagnostic import DiagnosticVariableOption -from earthdiagnostics.box import Box -from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.general.yearlymean import YearlyMean from mock import Mock, patch +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.timemean import YearlyMean from earthdiagnostics.modelingrealm import ModelingRealms @@ -30,14 +30,6 @@ class TestYearlyMean(TestCase): @patch.object(DiagnosticVariableOption, 'parse', fake_parse) def test_generate_jobs(self): - - jobs = YearlyMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) - self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], YearlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - Frequencies.monthly, '')) - self.assertEqual(jobs[1], YearlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - Frequencies.monthly, '')) - jobs = YearlyMean.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'day']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], YearlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', -- GitLab From 6c95b2ad06ff4a1b519b2b69edbbd7b9e2d2fafc Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 15 Dec 2017 10:35:46 +0100 Subject: [PATCH 112/168] Further cleaning of timemean --- earthdiagnostics/general/timemean.py | 106 +++++++++++++-------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index b0de54ed..8305e0bf 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -1,5 +1,5 @@ # coding=utf-8 - +"""Time mean diagnostics""" import os from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ @@ -59,6 +59,25 @@ class TimeMean(Diagnostic): options = cls.process_options(options, options_available) return options + @classmethod + def generate_jobs(cls, diags, options): + """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options = cls._process_options(diags, options) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(cls(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + def request_data(self): """Request data required by the diagnostic""" self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, @@ -70,7 +89,15 @@ class TimeMean(Diagnostic): frequency=Frequencies.daily, grid=self.grid) def compute_mean(self, input_file, output_file): - """Method to actually compute the mean""" + """ + Computes the time mean + + Parameters + ---------- + input_file: str + output_file: str + + """ raise NotImplementedError() def compute(self): @@ -93,6 +120,7 @@ class TimeMean(Diagnostic): class DailyMean(TimeMean): """ Calculates daily mean for a given variable + :param data_manager: data management object :type data_manager: DataManager :param startdate: startdate @@ -117,32 +145,23 @@ class DailyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'daily' - @classmethod - def generate_jobs(cls, diags, options): + def compute_mean(self, input_file, output_file): """ - Create a job for each chunk to compute the diagnostic + Computes the time mean - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, frequency=day, grid='' - :type options: list[str] - :return: - """ + Parameters + ---------- + input_file: str + output_file: str - options = cls._process_options(diags, options) - job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(DailyMean(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['frequency'], options['grid'])) - return job_list - - def compute_mean(self, input_file, output_file): + """ Utils.cdo.daymean(input=input_file, output=output_file) class MonthlyMean(TimeMean): """ Calculates monthly mean for a given variable + :param data_manager: data management object :type data_manager: DataManager :param startdate: startdate @@ -167,32 +186,23 @@ class MonthlyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'monthly' - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, frequency=day, grid='' - :type options: list[str] - :return: + def compute_mean(self, input_file, output_file): """ + Computes the time mean - options = cls._process_options(diags, options) - job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['frequency'], options['grid'])) - return job_list + Parameters + ---------- + input_file: str + output_file: str - def compute_mean(self, input_file, output_file): + """ Utils.cdo.monmean(input=input_file, output=output_file) class YearlyMean(TimeMean): """ Calculates monthly mean for a given variable + :param data_manager: data management object :type data_manager: DataManager :param startdate: startdate @@ -217,26 +227,16 @@ class YearlyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'yearly' - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, frequency=day, grid='' - :type options: list[str] - :return: + def compute_mean(self, input_file, output_file): """ + Computes the time mean - options = cls._process_options(diags, options) - job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(YearlyMean(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['frequency'], options['grid'])) - return job_list + Parameters + ---------- + input_file: str + output_file: str - def compute_mean(self, input_file, output_file): + """ Utils.cdo.monmean(input=input_file, output=output_file) -- GitLab From 70805de8e37c8fa22b6b65ccced73f680755a903 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 13:30:52 +0100 Subject: [PATCH 113/168] Fixed import in timemean --- bin/earthdiags | 1 + earthdiagnostics/general/timemean.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/earthdiags b/bin/earthdiags index 6ac26072..174528e3 100644 --- a/bin/earthdiags +++ b/bin/earthdiags @@ -22,5 +22,6 @@ def main(): os._exit(1) os._exit(0) + if __name__ == "__main__": main() diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index 8305e0bf..af4d5eca 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -4,7 +4,7 @@ import os from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticFrequencyOption, DiagnosticVariableOption -from frequency import Frequencies +from earthdiagnostics.frequency import Frequencies from utils import TempFile, Utils -- GitLab From e68be16802141147f0f2865b424a0e6667aa5569 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 14:13:03 +0100 Subject: [PATCH 114/168] Fixed import in timemean --- earthdiagnostics/general/timemean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index af4d5eca..5aaa4458 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -5,7 +5,7 @@ import os from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies -from utils import TempFile, Utils +from earthdiagnostics.utils import TempFile, Utils class TimeMean(Diagnostic): -- GitLab From 4e5eea3bf0edda46c0ac613a2d3af4deaf48f2f6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 14:38:08 +0100 Subject: [PATCH 115/168] Updated regsum --- earthdiagnostics/ocean/regionsum.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index d3881492..6ba7a4e3 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Diagnostic to calculate a region total""" import os from earthdiagnostics import cdftools @@ -12,8 +13,9 @@ from earthdiagnostics.utils import Utils, TempFile class RegionSum(Diagnostic): """ - Computes the mean value of the field (3D, weighted). For 3D fields, - a horizontal mean for each level is also given. If a spatial window + Computes the sum of the field (3D, weighted). + + For 3D fields, a horizontal mean for each level is also given. If a spatial window is specified, the mean value is computed only in this window. :original author: Javier Vegas-Regidor @@ -109,8 +111,8 @@ class RegionSum(Diagnostic): else: box_save = self.box - self.declare_var('sum', False, box_save) - self.declare_var('sum', True, box_save) + self._declare_var('sum', False, box_save) + self._declare_var('sum', True, box_save) def compute(self): """Run the diagnostic""" @@ -138,12 +140,12 @@ class RegionSum(Diagnostic): options=cdfmean_options) Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) - self.send_var(False, mean_file) - self.send_var(True, mean_file) + self._send_var(False, mean_file) + self._send_var(True, mean_file) os.remove(mean_file) - def send_var(self, threed, mean_file): + def _send_var(self, threed, mean_file): var = 'sum' if threed: if not self.save3d: @@ -168,7 +170,7 @@ class RegionSum(Diagnostic): handler.close() self.declared[final_name].set_local_file(temp2, diagnostic=self, rename_var=original_name, region=self.basin) - def declare_var(self, var, threed, box_save): + def _declare_var(self, var, threed, box_save): if threed: if not self.save3d: return False -- GitLab From eab4dca08a7b97191093e96914703a4f8e1daac1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 14:42:10 +0100 Subject: [PATCH 116/168] Updated regmean and cleaned discretize --- earthdiagnostics/ocean/regionmean.py | 26 ++++++++-------- earthdiagnostics/statistics/discretize.py | 36 +++++++++++++++-------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 77cc6cbc..a1bc0388 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Diagnostic to compute regional averages""" import os from earthdiagnostics import cdftools @@ -12,8 +13,9 @@ from earthdiagnostics.utils import Utils, TempFile class RegionMean(Diagnostic): """ - Computes the mean value of the field (3D, weighted). For 3D fields, - a horizontal mean for each level is also given. If a spatial window + Computes the mean value of the field (3D, weighted). + + For 3D fields, a horizontal mean for each level is also given. If a spatial window is specified, the mean value is computed only in this window. :original author: Javier Vegas-Regidor @@ -111,12 +113,12 @@ class RegionMean(Diagnostic): else: box_save = self.box - self.declare_var('mean', False, box_save) - self.declare_var('mean', True, box_save) + self._declare_var('mean', False, box_save) + self._declare_var('mean', True, box_save) if self.variance: - self.declare_var('var', False, box_save) - self.declare_var('var', True, box_save) + self._declare_var('var', False, box_save) + self._declare_var('var', True, box_save) def compute(self): """Run the diagnostic""" @@ -143,16 +145,16 @@ class RegionMean(Diagnostic): cdftools.run('cdfmean', input_file=variable_file, output_file=mean_file, options=cdfmean_options) Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) - self.send_var('mean', False, mean_file) - self.send_var('mean', True, mean_file) + self._send_var('mean', False, mean_file) + self._send_var('mean', True, mean_file) if self.variance: - self.send_var('var', False, mean_file) - self.send_var('var', True, mean_file) + self._send_var('var', False, mean_file) + self._send_var('var', True, mean_file) os.remove(mean_file) - def send_var(self, var, threed, mean_file): + def _send_var(self, var, threed, mean_file): if threed: if not self.save3d: return False @@ -176,7 +178,7 @@ class RegionMean(Diagnostic): handler.close() self.declared[final_name].set_local_file(temp2, diagnostic=self, rename_var=original_name, region=self.basin) - def declare_var(self, var, threed, box_save): + def _declare_var(self, var, threed, box_save): if threed: if not self.save3d: return False diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index b762280b..6e5f1003 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Diagnostic to discretize time series""" import math import cf_units @@ -45,6 +46,7 @@ class Discretize(Diagnostic): self.realizations = None self.num_bins = num_bins self._bins = None + self.discretized_data = None self.cmor_var = data_manager.variable_list.get_variable(variable, silent=True) if not math.isnan(min_value): @@ -69,11 +71,18 @@ class Discretize(Diagnostic): self.process = psutil.Process() - def print_memory_used(self): + def _print_memory_used(self): Log.debug('Memory: {0:.2f} GB'.format(self.process.memory_info().rss / 1024.0**3)) @property def bins(self): + """ + Bins to use + + Returns + ------- + int or iterable of str + """ if self._bins is None: return self.num_bins return self._bins @@ -84,8 +93,11 @@ class Discretize(Diagnostic): def __eq__(self, other): return self.domain == other.domain and self.variable == other.variable and self.num_bins == other.num_bins and \ - self.min_value == other.min_value and self.max_value == other.max_value and \ - self.startdate == other.startdate + self.min_value == other.min_value and self.max_value == other.max_value and \ + self.startdate == other.startdate + + def __hash__(self): + return hash(str(self)) def __str__(self): return 'Discretizing variable: {0.domain}:{0.variable} Startdate: {0.startdate} ' \ @@ -128,20 +140,20 @@ class Discretize(Diagnostic): def compute(self): """Run the diagnostic""" - self.print_memory_used() + self._print_memory_used() iris.FUTURE.netcdf_promote = True self._load_cube() - self.print_memory_used() + self._print_memory_used() self._get_value_interval() - self.print_memory_used() + self._print_memory_used() Log.info('Range: [{0}, {1}]', self.min_value, self.max_value) self._get_distribution() - self.print_memory_used() + self._print_memory_used() self._save_results() - self.print_memory_used() + self._print_memory_used() del self.distribution del self.data_cube - self.print_memory_used() + self._print_memory_used() def _load_cube(self): @@ -180,7 +192,7 @@ class Discretize(Diagnostic): for x in range(self.num_bins): bins[x] = (self.bins[x + 1] - self.bins[x]) / 2 + self.bins[x] bins_bounds[x, 0] = self.bins[x] - bins_bounds[x, 1] = self.bins[x+1] + bins_bounds[x, 1] = self.bins[x + 1] bins_coord = iris.coords.DimCoord(bins, var_name='bin', units=self.data_cube.units, bounds=bins_bounds) @@ -220,13 +232,13 @@ class Discretize(Diagnostic): if 'realization' in leadtime_cube.coords(): for realization_cube in self.data_cube.slices_over('realization'): Log.debug('Discretizing realization {0}', realization_cube.coord('realization').points[0]) - self.print_memory_used() + self._print_memory_used() if leadtime not in self.distribution: self.distribution[leadtime] = self._calculate_distribution(realization_cube) else: self.distribution[leadtime] += self._calculate_distribution(realization_cube) else: - self.print_memory_used() + self._print_memory_used() self.distribution[leadtime] = self._calculate_distribution(leadtime_cube) # noinspection PyTypeChecker -- GitLab From c0c43b50e25cf9115fce1dc6009f2d6166b07291 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 14:49:55 +0100 Subject: [PATCH 117/168] Cleaned cdftools --- earthdiagnostics/cdftools.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index 6283753a..f720696e 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""CDFTOOLS interface""" import os import six @@ -22,7 +23,7 @@ class CDFTools(object): # noinspection PyShadowingBuiltins def run(self, command, input_file, output_file=None, options=None, log_level=Log.INFO, input_option=None): """ - Runs one of the CDFTools + Run one of the CDFTools :param command: executable to run :type command: str | iterable @@ -37,7 +38,6 @@ class CDFTools(object): :param input_option: option to add before input file :type input_option: str """ - line = [os.path.join(self.path, command)] self._check_command_existence(line[0]) @@ -81,18 +81,18 @@ class CDFTools(object): if not os.path.isfile(element): raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) - # noinspection PyMethodMayBeStatic - def is_exe(self, fpath): + @staticmethod + def _is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) def _check_command_existence(self, command): if self.path: - if self.is_exe(os.path.join(self.path, command)): + if self._is_exe(os.path.join(self.path, command)): return else: for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, command) - if self.is_exe(exe_file): + if self._is_exe(exe_file): return raise ValueError('Error executing {0}\n Command does not exist in {1}'.format(command, self.path)) -- GitLab From f8fb355d60940c8b32190057ff42ff80d900b6cc Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 14:57:24 +0100 Subject: [PATCH 118/168] Cleaned daysover --- .../statistics/daysoverpercentile.py | 92 +++++++++++-------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index 10771ecf..d6f30bc0 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -1,4 +1,8 @@ # coding=utf-8 +""" +Diagnostic to ciompute statistics about days above and below a set of +precomputed percentiles +""" import os import iris @@ -21,7 +25,9 @@ from earthdiagnostics.variable_type import VariableType class DaysOverPercentile(Diagnostic): """ - Calculates the montlhy percentiles + Calculates the days over and below the calculted percentiles + + It uses the stored percentiles and the stored discretized variables :param data_manager: data management object :type data_manager: DataManager @@ -41,10 +47,13 @@ class DaysOverPercentile(Diagnostic): self.forecast_month = forecast_month self.startdate = startdate + self.lat_coord = None + self.lon_coord = None + def __eq__(self, other): return self.startdate == other.startdate and self.domain == other.domain and \ - self.variable == other.variable and self.start_year == other.start_year and \ - self.end_year == other.end_year + self.variable == other.variable and self.start_year == other.start_year and \ + self.end_year == other.end_year def __str__(self): return 'Days over percentile Startdate: {0.startdate} Variable: {0.domain}:{0.variable} ' \ @@ -103,37 +112,12 @@ class DaysOverPercentile(Diagnostic): def compute(self): """Run the diagnostic""" - iris.FUTURE.netcdf_promote = True - percentiles = iris.load_cube(self.percentiles_file.local_file) - - handler = Utils.open_cdf(self.variable_file.local_file) - if 'realization' in handler.variables: - handler.variables[self.variable].coordinates = 'realization' - handler.close() - var = iris.load_cube(self.variable_file.local_file) - date = parse_date(self.startdate) - lead_date = add_months(date, 1, self.data_manager.config.experiment.calendar) - leadtimes = {1: PartialDateTime(lead_date.year, lead_date.month, lead_date.day)} - - def assign_leadtime(coord, x): - leadtime_month = 1 - partial_date = leadtimes[leadtime_month] - while coord.units.num2date(x) >= partial_date: - leadtime_month += 1 - try: - partial_date = leadtimes[leadtime_month] - except KeyError: - new_date = add_months(date, leadtime_month, self.data_manager.config.experiment.calendar) - partial_date = PartialDateTime(new_date.year, new_date.month, new_date.day) - leadtimes[leadtime_month] = partial_date - return leadtime_month - iris.coord_categorisation.add_categorised_coord(var, 'leadtime', 'time', assign_leadtime) - iris.coord_categorisation.add_year(var, 'time') - iris.coord_categorisation.add_day_of_year(var, 'time') + leadtimes, percentiles, var = self._load_data() try: realization_coord = var.coord('realization') except iris.exceptions.CoordinateNotFoundError: realization_coord = None + self.lat_coord = var.coord('latitude') self.lon_coord = var.coord('longitude') results_over = {perc: iris.cube.CubeList() for perc in ClimatologicalPercentile.Percentiles} @@ -163,22 +147,22 @@ class DaysOverPercentile(Diagnostic): # noinspection PyTypeChecker days_over = np.sum(leadtime_slice.data > percentile_slice.data, 0) / float(timesteps) - result = self.create_results_cube(days_over, percentile, realization_coord, - time_coord, var_daysover, long_name_days_over) + result = self._create_results_cube(days_over, percentile, realization_coord, + time_coord, var_daysover, long_name_days_over) results_over[percentile].append(result) # noinspection PyTypeChecker days_below = np.sum(leadtime_slice.data < percentile_slice.data, 0) / float(timesteps) - result = self.create_results_cube(days_below, percentile, realization_coord, - time_coord, var_days_below, long_name_days_below) + result = self._create_results_cube(days_below, percentile, realization_coord, + time_coord, var_days_below, long_name_days_below) results_below[percentile].append(result) Log.debug('Saving percentiles startdate {0}', self.startdate) for perc in ClimatologicalPercentile.Percentiles: iris.FUTURE.netcdf_no_unlimited = True - self.days_over_file[perc].set_local_file(self.save_to_file(perc, results_over, var_daysover), + self.days_over_file[perc].set_local_file(self._save_to_file(perc, results_over, var_daysover), rename_var=var_daysover) - self.days_below_file[perc].set_local_file(self.save_to_file(perc, results_below, var_days_below), + self.days_below_file[perc].set_local_file(self._save_to_file(perc, results_below, var_days_below), rename_var=var_days_below) del self.days_over_file @@ -186,8 +170,38 @@ class DaysOverPercentile(Diagnostic): del self.lat_coord del self.lon_coord + def _load_data(self): + iris.FUTURE.netcdf_promote = True + percentiles = iris.load_cube(self.percentiles_file.local_file) + handler = Utils.open_cdf(self.variable_file.local_file) + if 'realization' in handler.variables: + handler.variables[self.variable].coordinates = 'realization' + handler.close() + var = iris.load_cube(self.variable_file.local_file) + date = parse_date(self.startdate) + lead_date = add_months(date, 1, self.data_manager.config.experiment.calendar) + leadtimes = {1: PartialDateTime(lead_date.year, lead_date.month, lead_date.day)} + + def assign_leadtime(coord, x): + leadtime_month = 1 + partial_date = leadtimes[leadtime_month] + while coord.units.num2date(x) >= partial_date: + leadtime_month += 1 + try: + partial_date = leadtimes[leadtime_month] + except KeyError: + new_date = add_months(date, leadtime_month, self.data_manager.config.experiment.calendar) + partial_date = PartialDateTime(new_date.year, new_date.month, new_date.day) + leadtimes[leadtime_month] = partial_date + return leadtime_month + + iris.coord_categorisation.add_categorised_coord(var, 'leadtime', 'time', assign_leadtime) + iris.coord_categorisation.add_year(var, 'time') + iris.coord_categorisation.add_day_of_year(var, 'time') + return leadtimes, percentiles, var + @staticmethod - def save_to_file(perc, results_over, var_daysover): + def _save_to_file(perc, results_over, var_daysover): temp = TempFile.get() iris.save(results_over[perc].merge_cube(), temp, zlib=True, unlimited_dimensions=['time']) Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, @@ -227,8 +241,8 @@ class DaysOverPercentile(Diagnostic): handler.close() return temp - def create_results_cube(self, days_over, percentile, realization_coord, time_coord, - var_name, long_name): + def _create_results_cube(self, days_over, percentile, realization_coord, time_coord, + var_name, long_name): result = iris.cube.Cube(days_over.astype(np.float32), var_name=var_name, long_name=long_name, units=1.0) if realization_coord is not None: result.add_aux_coord(realization_coord, 0) -- GitLab From d27f32ac1e720132a01fd6955edce53d049b3146 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 15:33:33 +0100 Subject: [PATCH 119/168] Cleaned doc and reimplemented basins singleton --- earthdiagnostics/constants.py | 14 ++++++++---- earthdiagnostics/frequency.py | 27 +++++++++++++++++++++++ earthdiagnostics/publisher.py | 22 ++++++++++-------- earthdiagnostics/statistics/discretize.py | 2 +- test/unit/test_constants.py | 8 ++++++- 5 files changed, 58 insertions(+), 15 deletions(-) diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index f38c39fb..4d18c334 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -1,6 +1,5 @@ # coding=utf-8 """Contains the enumeration-like classes used by the diagnostics""" -from singleton import SingletonType class Basin(object): @@ -37,12 +36,17 @@ class Basin(object): class Basins(object): """ - Predefined basins + Singleton class to manage available basins """ + __instance = None - __metaclass__ = SingletonType + def __new__(cls, *args, **kwargs): + if cls.__instance is None: + cls.__instance = object.__new__(cls, *args, **kwargs) + cls.__instance.__initialize() + return cls.__instance - def __init__(self): + def __initialize(self): self.aliases = { 'Global': ('Global', 'glob'), @@ -128,6 +132,8 @@ class Basins(object): for alias in self.aliases[basin.name]: self._add_alias(alias, basin) + self.__initiated = False + def get_available_basins(self, handler): """ Read available basins from file diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py index 9a95f26e..0629d031 100644 --- a/earthdiagnostics/frequency.py +++ b/earthdiagnostics/frequency.py @@ -1,8 +1,10 @@ # coding=utf-8 +"""Data frequency management tools""" from earthdiagnostics.variable_type import VariableType class Frequency(object): + """Time frequency""" _recognized = {'f': 'fx', 'fx': 'fx', 'fixed': 'fx', 'c': 'clim', 'clim': 'clim', 'climatology': 'clim', 'monclim': 'clim', '1hrclimmon': 'clim', @@ -36,6 +38,17 @@ class Frequency(object): return self.frequency def folder_name(self, vartype): + """ + Get foder name associated to this frequency + + Parameters + ---------- + vartype: VariableType + + Returns + ------- + str + """ if self == Frequencies.daily: freq_str = 'daily_{0}'.format(VariableType.to_str(vartype)) elif self == Frequencies.weekly: @@ -52,12 +65,26 @@ class Frequency(object): @staticmethod def parse(freq): + """ + Get frequency instance from str + + If a Frequency object is passed, it is returned as usual + + Parameters + ---------- + freq: str or Frequency + + Returns + ------- + Frequency + """ if isinstance(freq, Frequency): return freq return Frequency(freq) class Frequencies(object): + """Enumeration of supported frequencies""" fixed = Frequency('fx') climatology = Frequency('clim') yearly = Frequency('year') diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py index 395a92b8..42a8d3ce 100644 --- a/earthdiagnostics/publisher.py +++ b/earthdiagnostics/publisher.py @@ -1,6 +1,10 @@ # coding=utf-8 +"""Module to allow classes to communicate when an event is produced""" + + class Publisher(object): """Base class to provide functionality to notify updates to other objects""" + def __init__(self): self._subscribers = dict() @@ -8,18 +12,20 @@ class Publisher(object): """ Add a suscriber to the current publisher - :param who: subscriber to add - :type who: object - :param callback: method to execute when publisher updates - :type callback: callable | None - """ + Parameters + ---------- + who: object + Subscriber to add + callback: callable or None, optional + Callback to call + """ if callback is None: callback = getattr(who, 'update') self._subscribers[who] = callback def unsubscribe(self, who): """ - Removes a suscriber from the current publisher + Remove a suscriber from the current publisher :param who: suscriber to remove :type who: object @@ -38,9 +44,7 @@ class Publisher(object): @property def suscribers(self): - """ - List of suscribers of this publisher - """ + """List of suscribers of this publisher""" return self._subscribers.keys() def only_suscriber(self, who): diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index 6e5f1003..a5c4df7a 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -77,7 +77,7 @@ class Discretize(Diagnostic): @property def bins(self): """ - Bins to use + Configuration of bins to use Returns ------- diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py index c538ee09..32dadcd9 100644 --- a/test/unit/test_constants.py +++ b/test/unit/test_constants.py @@ -1,7 +1,7 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.constants import Basin +from earthdiagnostics.constants import Basin, Basins class TestBasin(TestCase): @@ -18,3 +18,9 @@ class TestBasin(TestCase): def test__str__(self): self.assertEquals(str(self.basin), 'Basin') + + +class TestBasins(TestCase): + + def test_singleton(self): + self.assertIs(Basins(), Basins()) -- GitLab From 3b9ec2810eab9c49eefbbc9cec54916881bfe400 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 16:17:03 +0100 Subject: [PATCH 120/168] Finished experimentconfig tests --- earthdiagnostics/config.py | 7 ++- test/unit/test_config.py | 90 +++++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 4b32d09b..8fd49a24 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -352,6 +352,9 @@ class THREDDSConfig(object): class ExperimentConfig(object): """Configuration related to the experiment""" + def __init__(self): + self.chunk_list = None + def parse_ini(self, parser): """ Parse experiment section from INI-like file @@ -583,7 +586,7 @@ class ExperimentConfig(object): :rtype: list[int] """ - chunks_per_year = 12 / self.chunk_size + chunks_per_year = 12 // self.chunk_size date = parse_date(startdate) first_january = 0 first_year = date.year @@ -595,7 +598,7 @@ class ExperimentConfig(object): first_january += 1 years = list() - for _ in range(first_january, chunks_per_year, self.num_chunks): + for _ in range(first_january, self.num_chunks, chunks_per_year,): years.append(first_year) first_year += 1 return years diff --git a/test/unit/test_config.py b/test/unit/test_config.py index ec3720f3..552fb392 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -1,5 +1,6 @@ # coding=utf-8 from unittest import TestCase +import datetime from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig from earthdiagnostics.frequency import Frequencies @@ -296,6 +297,27 @@ class TestExperimentConfig(TestCase): self.assertEquals(config.atmos_timestep, 6) self.assertEquals(config.ocean_timestep, 6) + def test_members(self): + self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', 'fc0 1') + config = ExperimentConfig() + config.parse_ini(self.mock_parser) + self.assertEquals(config.members, [0, 1]) + + self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', 'fc00 fc01') + config = ExperimentConfig() + config.parse_ini(self.mock_parser) + self.assertEquals(config.members, [0, 1]) + + self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', 'fc1-fc3') + config = ExperimentConfig() + config.parse_ini(self.mock_parser) + self.assertEquals(config.members, [1, 2, 3]) + + self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', '1-3') + config = ExperimentConfig() + config.parse_ini(self.mock_parser) + self.assertEquals(config.members, [1, 2, 3]) + def test_startdates(self): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '20001101 20011101') config = ExperimentConfig() @@ -320,7 +342,7 @@ class TestExperimentConfig(TestCase): config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20011101']) - self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,6M}') + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,6M} ') config = ExperimentConfig() config.parse_ini(self.mock_parser) self.assertEquals(config.startdates, ['20001101', '20010501', '20011101']) @@ -345,3 +367,69 @@ class TestExperimentConfig(TestCase): config = ExperimentConfig() config.parse_ini(self.mock_parser) + def test_get_member_str(self): + config = ExperimentConfig() + config.parse_ini(self.mock_parser) + self.assertEqual(config.get_member_str(1), 'fc1') + + def test_get_full_years(self): + self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 3) + self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 15) + + config = ExperimentConfig() + config.parse_ini(self.mock_parser) + self.assertEqual(config.get_full_years('20000601'), [2001, 2002, 2003, 2004]) + self.assertEqual(config.get_full_years('20000101'), [2000, 2001, 2002, 2003]) + + def test_get_year_chunks(self): + self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 3) + self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 15) + + config = ExperimentConfig() + config.parse_ini(self.mock_parser) + self.assertEqual(config.get_year_chunks('20000601', 2001), [3, 4, 5, 6, 7]) + self.assertEqual(config.get_year_chunks('20000101', 2000), [1, 2, 3, 4]) + + self.assertEqual(config.get_year_chunks('20000601', 2000), [1, 2, 3]) + self.assertEqual(config.get_year_chunks('20000601', 1999), []) + + def test_get_chunk_list(self): + config = ExperimentConfig() + config.startdates = ('20010101', ) + config.members = (0, 1, 2) + config.chunk_list = [0] + config.num_chunks = 2 + self.assertEqual(config.get_chunk_list(), [('20010101', 0, 0), ('20010101', 1, 0), ('20010101', 2, 0)]) + + config.chunk_list = [] + self.assertEqual(config.get_chunk_list(), [('20010101', 0, 1), ('20010101', 0, 2), ('20010101', 1, 1), + ('20010101', 1, 2), ('20010101', 2, 1), ('20010101', 2, 2)]) + + def test_get_member_list(self): + config = ExperimentConfig() + config.startdates = ('20010101', ) + config.members = (0, 1, 2) + self.assertEqual(config.get_member_list(), [('20010101', 0), ('20010101', 1), ('20010101', 2)]) + + def test_get_chunk_start_str(self): + config = ExperimentConfig() + self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 12) + self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 3) + config.parse_ini(self.mock_parser) + self.assertEqual(config.get_chunk_start_str('20001101', 3), '20021101') + + def test_get_chunk_start_str_datetime(self): + config = ExperimentConfig() + self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 12) + self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 3) + date = datetime.datetime(year=2000, month=11, day=1) + config.parse_ini(self.mock_parser) + self.assertEqual(config.get_chunk_start_str(date, 3), '20021101') + + def test_get_chunk_end_str(self): + config = ExperimentConfig() + self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 12) + self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 3) + config.parse_ini(self.mock_parser) + self.assertEqual(config.get_chunk_end_str('20001101', 3), '20031101') + -- GitLab From c8f67b9b153b1f6534b1e6151da94b03bfa12bc3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 19 Dec 2017 17:00:24 +0100 Subject: [PATCH 121/168] Created first test for default config --- earthdiagnostics/config.py | 3 +-- test/unit/test_config.py | 54 ++++++++++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 8fd49a24..c04485dc 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -29,8 +29,7 @@ class Config(object): Path to the configuration file """ - def __init__(self, path): - + def parse(self, path): parser = ConfigParser() parser.optionxform = str parser.read(path) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 552fb392..7d94740f 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -1,8 +1,9 @@ # coding=utf-8 from unittest import TestCase import datetime +import mock -from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig +from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig, Config from earthdiagnostics.frequency import Frequencies from earthdiagnostics.modelingrealm import ModelingRealms @@ -25,9 +26,10 @@ class VariableManagerMock(object): return var -class ParserMock(object): +class ParserMock(mock.Mock): - def __init__(self): + def __init__(self, **kwargs): + super().__init__(**kwargs) self._values = {} def add_value(self, section, var, value): @@ -45,12 +47,14 @@ class ParserMock(object): def get_bool_option(self, section, var, default): return self.get_value(section, var, default) - def get_path_option(self, section, var, default): + def get_path_option(self, section, var, default=""): return self.get_value(section, var, default) def get_int_option(self, section, var, default=0): return self.get_value(section, var, default) + def get_choice_option(self, section, var, choices, default, ignore_case=True): + return self.get_value(section, var, default) def get_int_list_option(self, section, var, default=list(), separator=' '): try: @@ -240,7 +244,15 @@ class TestCMORConfig(TestCase): def test_bad_frequency_vars(self): config = CMORConfig(self.mock_parser, self.var_manager) with self.assertRaises(ValueError): - self.assertEquals(config.get_variables(Frequencies.climatology), {}) + config.get_variables(Frequencies.climatology) + + def test_requested_codes(self): + self.mock_parser.add_value('CMOR', 'ATMOS_HOURLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + self.mock_parser.add_value('CMOR', 'ATMOS_DAILY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + config = CMORConfig(self.mock_parser, self.var_manager) + + self.assertEquals(config.get_requested_codes(), {128, 129, 130, 131, 132}) class TestTHREDDSConfig(TestCase): @@ -433,3 +445,35 @@ class TestExperimentConfig(TestCase): config.parse_ini(self.mock_parser) self.assertEqual(config.get_chunk_end_str('20001101', 3), '20031101') + +class TestConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + + def test_parse(self): + config = Config() + + def mock_new(): + return self.mock_parser + + def mock_new_exp(): + mock_exp = mock.Mock() + mock_exp.expid = 'expid' + return mock_exp + + self.mock_parser.add_value('DIAGNOSTICS', 'FREQUENCY', 'mon') + self.mock_parser.add_value('DIAGNOSTICS', 'DIAGS', 'diag1 diag2') + self.mock_parser.add_value('DIAGNOSTICS', 'SCRATCH_DIR', 'scratch') + self.mock_parser.options.return_value = () + with mock.patch('earthdiagnostics.config.ConfigParser', new=mock_new): + with mock.patch('earthdiagnostics.config.VariableManager'): + with mock.patch('earthdiagnostics.config.ExperimentConfig', new=mock_new_exp): + with mock.patch('earthdiagnostics.config.CMORConfig'): + with mock.patch('earthdiagnostics.config.THREDDSConfig'): + config.parse('path') + self.assertEqual(config.frequency, Frequencies.monthly) + self.assertEqual(config.auto_clean, True) + self.assertEqual(config.cdftools_path, '') + self.assertEqual(config.con_files, '') + self.assertEqual(config.data_adaptor, 'CMOR') -- GitLab From 1f734a36a1a08c23f36d5f6d6537df4ecbdb80bb Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 12:40:59 +0100 Subject: [PATCH 122/168] Finished config tests --- test/unit/test_config.py | 64 +++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 7d94740f..d04aa004 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -2,6 +2,7 @@ from unittest import TestCase import datetime import mock +import os from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig, Config from earthdiagnostics.frequency import Frequencies @@ -71,6 +72,14 @@ class ParserMock(mock.Mock): def get_option(self, section, var, default=None): return self.get_value(section, var, default) + def has_section(self, section): + start = '{0}:'.format(section) + return any(x.startswith(start) for x in self._values) + + def options(self, section): + start = '{0}:'.format(section) + return [x[len(start):] for x in self._values if x.startswith(start)] + class TestCMORConfig(TestCase): @@ -450,10 +459,16 @@ class TestConfig(TestCase): def setUp(self): self.mock_parser = ParserMock() + self.mock_parser.add_value('DIAGNOSTICS', 'FREQUENCY', 'mon') + self.mock_parser.add_value('DIAGNOSTICS', 'DIAGS', 'diag1 diag2') + self.mock_parser.add_value('DIAGNOSTICS', 'SCRATCH_DIR', 'scratch') + self._environ = dict(os.environ) - def test_parse(self): - config = Config() + def tearDown(self): + os.environ.clear() + os.environ.update(self._environ) + def _parse(self, config): def mock_new(): return self.mock_parser @@ -461,19 +476,54 @@ class TestConfig(TestCase): mock_exp = mock.Mock() mock_exp.expid = 'expid' return mock_exp - - self.mock_parser.add_value('DIAGNOSTICS', 'FREQUENCY', 'mon') - self.mock_parser.add_value('DIAGNOSTICS', 'DIAGS', 'diag1 diag2') - self.mock_parser.add_value('DIAGNOSTICS', 'SCRATCH_DIR', 'scratch') - self.mock_parser.options.return_value = () with mock.patch('earthdiagnostics.config.ConfigParser', new=mock_new): with mock.patch('earthdiagnostics.config.VariableManager'): with mock.patch('earthdiagnostics.config.ExperimentConfig', new=mock_new_exp): with mock.patch('earthdiagnostics.config.CMORConfig'): with mock.patch('earthdiagnostics.config.THREDDSConfig'): config.parse('path') + + def test_diags(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'DIAGS', 'diag1 diag2,opt1,opt2 # Commented diag') + self._parse(config) + self.assertEqual(config.get_commands(), (['diag1', 'diag2,opt1,opt2'])) + + def test_parse(self): + config = Config() + self._parse(config) self.assertEqual(config.frequency, Frequencies.monthly) self.assertEqual(config.auto_clean, True) self.assertEqual(config.cdftools_path, '') self.assertEqual(config.con_files, '') self.assertEqual(config.data_adaptor, 'CMOR') + self.assertEqual(config.get_commands(), (['diag1', 'diag2'])) + + def test_alias(self): + config = Config() + self.mock_parser.add_value('ALIAS', 'diag1', 'diag3') + self._parse(config) + self.assertEqual(config.get_commands(), ['diag3', 'diag2']) + + def test_auto_clean_ram_disk(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'AUTO_CLEAN', False) + self.mock_parser.add_value('DIAGNOSTICS', 'USE_RAMDISK', True) + self._parse(config) + self.assertEqual(config.auto_clean, True) + self.assertEqual(config.use_ramdisk, True) + + def test_data_convention_primavera(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'primavera') + self._parse(config) + self.assertEqual(config.data_convention, 'primavera') + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks/primavera') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_primavera.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) + + + + + -- GitLab From 6546c5532a39621ace7216355d0b1a5c157f4b02 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 12:50:27 +0100 Subject: [PATCH 123/168] Fixed tests --- diags.conf | 2 +- earthdiagnostics/earthdiags.py | 3 ++- earthdiagnostics/frequency.py | 1 + test/unit/test_config.py | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/diags.conf b/diags.conf index a3176d96..82d63fd7 100644 --- a/diags.conf +++ b/diags.conf @@ -88,7 +88,7 @@ OCEAN_TIMESTEP = 6 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = testing_erainterim -STARTDATES = 20050101 +STARTDATES = 20010101 #{20050101,20161231,D} MEMBERS = 0 MEMBER_DIGITS = 1 diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index db8a4320..e6d602af 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -46,7 +46,8 @@ class EarthDiags(object): def __init__(self, config_file): Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) - self.config = Config(config_file) + self.config = Config() + self.config.parse(config_file) os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' TempFile.scratch_folder = self.config.scratch_dir diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py index 0629d031..9c04c072 100644 --- a/earthdiagnostics/frequency.py +++ b/earthdiagnostics/frequency.py @@ -85,6 +85,7 @@ class Frequency(object): class Frequencies(object): """Enumeration of supported frequencies""" + fixed = Frequency('fx') climatology = Frequency('clim') yearly = Frequency('year') diff --git a/test/unit/test_config.py b/test/unit/test_config.py index d04aa004..9b9c63d5 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -3,6 +3,7 @@ from unittest import TestCase import datetime import mock import os +import six from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig, Config from earthdiagnostics.frequency import Frequencies @@ -30,7 +31,7 @@ class VariableManagerMock(object): class ParserMock(mock.Mock): def __init__(self, **kwargs): - super().__init__(**kwargs) + super(mock.Mock, self).__init__(**kwargs) self._values = {} def add_value(self, section, var, value): -- GitLab From 8c0a354dbefec25c1a86b589cc0ff4f39bf41571 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 12:58:49 +0100 Subject: [PATCH 124/168] Updated doc --- earthdiagnostics/config.py | 79 ++++++++++++++++++++++++++++++++++++++ test/unit/test_config.py | 1 - 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index c04485dc..d84ea7ab 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -29,6 +29,85 @@ class Config(object): Path to the configuration file """ + def __init__(self): + # Read diags config + self.data_adaptor = None + "Scratch folder path" + self.scratch_dir = None + "Scratch folder path" + self.use_ramdisk = None + "If True, the scratch dir is created as a ram disk" + self.auto_clean = None + "If True, the scratch dir is removed after finishing" + self.scratch_masks = None + "Common scratch folder for masks" + self.data_dir = None + "Root data folder path" + self.data_type = None + "Data type (experiment, observation or reconstruction)" + self.con_files = None + "Mask and meshes folder path" + self.mesh_mask = None + "Custom mesh mask file to use" + self.new_mask_glo = None + "Custom new mask glo file to use" + self.mask_regions = None + "Custom mask regions file to use" + self.mask_regions_3d = None + "Custom mask regions 3D file to use" + self.data_convention = None + "Data convention to use" + self.var_manager = None + self.skip_diags_done = None + "Flag to control if already done diags must be recalculated" + self.frequency = None + "Default data frequency to be used by the diagnostics" + + self.cdftools_path = None + "Path to CDFTOOLS executables" + self.max_cores = None + "Maximum number of cores to use" + self.parallel_downloads = None + "Maximum number of simultaneous downloads" + self.parallel_uploads = None + "Maximum number of simultaneous uploads" + self.restore_meshes = None + "If True, forces the tool to copy all the mesh and mask files for the model, regardless of existence" + + # Read experiment config + self.experiment = ExperimentConfig() + """ + Configuration related to the experiment + + Returns + ------- + ExperimentConfig + """ + self.cmor = None + """ + CMOR related configuration + + Returns + ------- + CMORConfig + """ + self.thredds = None + """ + THREDDS server configuration + + Returns + ------- + THREDDSConfig + """ + self.report = None + """ + Reporting configuration + + Returns + ------- + ReportConfig + """ + def parse(self, path): parser = ConfigParser() parser.optionxform = str diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 9b9c63d5..0839998d 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -3,7 +3,6 @@ from unittest import TestCase import datetime import mock import os -import six from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig, Config from earthdiagnostics.frequency import Frequencies -- GitLab From 9f0c5ba2146fdda4f40aeb09b916dbbf84386e14 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 13:05:13 +0100 Subject: [PATCH 125/168] Removed unused docstrings --- diags.conf | 2 +- earthdiagnostics/config.py | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/diags.conf b/diags.conf index 82d63fd7..d0123ff8 100644 --- a/diags.conf +++ b/diags.conf @@ -88,7 +88,7 @@ OCEAN_TIMESTEP = 6 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = testing_erainterim -STARTDATES = 20010101 +STARTDATES = 20000101 #{20050101,20161231,D} MEMBERS = 0 MEMBER_DIGITS = 1 diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index d84ea7ab..fcc75f21 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -116,33 +116,21 @@ class Config(object): # Read diags config self.data_adaptor = parser.get_choice_option('DIAGNOSTICS', 'DATA_ADAPTOR', ('CMOR', 'THREDDS', 'OBSRECON'), 'CMOR') - "Scratch folder path" self.scratch_dir = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_DIR') - "Scratch folder path" self.use_ramdisk = parser.get_bool_option('DIAGNOSTICS', 'USE_RAMDISK', False) - "If True, the scratch dir is created as a ram disk" self.auto_clean = parser.get_bool_option('DIAGNOSTICS', 'AUTO_CLEAN', True) - "If True, the scratch dir is removed after finishing" if not self.auto_clean and self.use_ramdisk: Log.warning('RAM disk scratch dir is always automatically cleaned.') self.auto_clean = True self.scratch_masks = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_MASKS', '/scratch/Earth/ocean_masks') - "Common scratch folder for masks" self.data_dir = parser.get_path_option('DIAGNOSTICS', 'DATA_DIR') - "Root data folder path" self.data_type = parser.get_choice_option('DIAGNOSTICS', 'DATA_TYPE', ('exp', 'obs', 'recon'), 'exp') - "Data type (experiment, observation or reconstruction)" self.con_files = parser.get_path_option('DIAGNOSTICS', 'CON_FILES') - "Mask and meshes folder path" self.mesh_mask = parser.get_path_option('DIAGNOSTICS', 'MESH_MASK', '') - "Custom mesh mask file to use" self.new_mask_glo = parser.get_path_option('DIAGNOSTICS', 'NEW_MASK_GLO', '') - "Custom new mask glo file to use" self.mask_regions = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS', '') - "Custom mask regions file to use" self.mask_regions_3d = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS_3D', '') - "Custom mask regions 3D file to use" self.data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), @@ -163,26 +151,15 @@ class Config(object): self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') self.skip_diags_done = parser.get_bool_option('DIAGNOSTICS', 'SKIP_DIAGS_DONE', True) self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY')) - "Default data frequency to be used by the diagnostics" self.cdftools_path = parser.get_path_option('DIAGNOSTICS', 'CDFTOOLS_PATH', '') - "Path to CDFTOOLS executables" self.max_cores = parser.get_int_option('DIAGNOSTICS', 'MAX_CORES', 0) - "Maximum number of cores to use" self.parallel_downloads = parser.get_int_option('DIAGNOSTICS', 'PARALLEL_DOWNLOADS', 1) - "Maximum number of simultaneous downloads" self.parallel_uploads = parser.get_int_option('DIAGNOSTICS', 'PARALLEL_UPLOADS', 1) - "Maximum number of simultaneous uploads" self.restore_meshes = parser.get_bool_option('DIAGNOSTICS', 'RESTORE_MESHES', False) - "If True, forces the tool to copy all the mesh and mask files for the model, regardless of existence" # Read experiment config self.experiment = ExperimentConfig() - """ - Configuration related to the experiment - - :rtype: ExperimentConfig - """ self.experiment.parse_ini(parser) # Read aliases self._aliases = dict() -- GitLab From c2c019e23718a111717c6b73f5042926303db8ee Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 13:16:05 +0100 Subject: [PATCH 126/168] Updated config doc --- earthdiagnostics/config.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index fcc75f21..1afdc713 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -20,14 +20,7 @@ class ConfigException(Exception): class Config(object): - """ - Class to read and manage the configuration - - Parameters - ---------- - path: str - Path to the configuration file - """ + """Class to read and manage the configuration""" def __init__(self): # Read diags config @@ -86,7 +79,7 @@ class Config(object): self.cmor = None """ CMOR related configuration - + Returns ------- CMORConfig @@ -109,6 +102,13 @@ class Config(object): """ def parse(self, path): + """ + Read configuration from INI file + + Parameters + ---------- + path: str + """ parser = ConfigParser() parser.optionxform = str parser.read(path) -- GitLab From 17a9e16c812a3dc34f8704e0d65c2dabf35f28b8 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 13:23:02 +0100 Subject: [PATCH 127/168] Updated doc --- earthdiagnostics/general/attribute.py | 1 - earthdiagnostics/general/scale.py | 17 +++---- earthdiagnostics/general/select_levels.py | 48 ++++++++----------- .../general/simplify_dimensions.py | 43 ++++++++--------- earthdiagnostics/ocean/heatcontent.py | 1 + earthdiagnostics/ocean/heatcontentlayer.py | 4 +- earthdiagnostics/ocean/interpolate.py | 2 + earthdiagnostics/ocean/mask_land.py | 4 +- earthdiagnostics/ocean/maxmoc.py | 5 +- .../ocean/mixedlayerheatcontent.py | 1 + .../ocean/mixedlayersaltcontent.py | 1 + earthdiagnostics/ocean/moc.py | 1 + earthdiagnostics/ocean/mxl.py | 1 + earthdiagnostics/ocean/psi.py | 1 + 14 files changed, 63 insertions(+), 67 deletions(-) diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index c7c52fa3..a6801ebc 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -63,7 +63,6 @@ class Attribute(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticDomainOption(), DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticOption('name'), diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 3cb23d47..21b032b6 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -12,6 +12,7 @@ from earthdiagnostics.utils import Utils class Scale(Diagnostic): """ Scales a variable by the given value also adding at offset + Can be useful to correct units or other known errors (think of a tas file declaring K as units but with the data stored as Celsius) @@ -67,14 +68,14 @@ class Scale(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, grid - :type options: list[str] - :return: - """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ options_available = (DiagnosticDomainOption(), DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticFloatOption('value'), diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index caa2a017..947ef850 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -7,25 +7,19 @@ from earthdiagnostics.utils import Utils, TempFile class SelectLevels(Diagnostic): """ - Convert i j files to lon lat when there is no interpolation required, - i.e. lon is constant over i and lat is constat over j - - :original author: Javier Vegas-Regidor - - :created: April 2017 - - :param data_manager: data management object - :type data_manager: DataManager - :param startdate: startdate - :type startdate: str - :param member: member number - :type member: int - :param chunk: chunk's number - :type chunk: int - :param variable: variable's name - :type variable: str - :param domain: variable's domain - :type domain: ModelingRealm + Extract levels from file + + Parameters + ---------- + data_manager: DataManager + startdate: str + member: int + chunk: init + domain: ModellingRealm + variable: str + grid: str or None + first_level: int + last_level: int """ alias = 'selev' @@ -56,14 +50,14 @@ class SelectLevels(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: domain,variables,grid - :type options: list[str] - :return: - """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain,variables,grid + :type options: list[str] + :return: + """ options_available = (DiagnosticDomainOption(), DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), DiagnosticIntOption('first_level'), diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index 7fe3002e..2ed5eb45 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -9,24 +9,19 @@ from earthdiagnostics.utils import Utils, TempFile class SimplifyDimensions(Diagnostic): """ Convert i j files to lon lat when there is no interpolation required, + i.e. lon is constant over i and lat is constat over j - :original author: Javier Vegas-Regidor - - :created: April 2017 - - :param data_manager: data management object - :type data_manager: DataManager - :param startdate: startdate - :type startdate: str - :param member: member number - :type member: int - :param chunk: chunk's number - :type chunk: int - :param variable: variable's name - :type variable: str - :param domain: variable's domain - :type domain: ModelingRealm + Parameters + ---------- + data_manager: DataManager + startdate: str + member: int + chunk: init + domain: ModellingRealm + variable: str + grid: str or None + data_convention: str """ alias = 'simdim' @@ -59,14 +54,14 @@ class SimplifyDimensions(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: domain,variables,grid - :type options: list[str] - :return: - """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain,variables,grid + :type options: list[str] + :return: + """ options_available = (DiagnosticDomainOption(), DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), DiagnosticOption('grid', '')) diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index cb122f33..476c639f 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute the total ocean heat content""" import shutil import numpy as np diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 1d87e1f4..423087cc 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -100,9 +100,7 @@ class HeatContentLayer(Diagnostic): handler.close() def calculate_weight(e3t_point, depth_point, mask_point): - """ - Calculates the weight for each cell - """ + """Calculates the weight for each cell""" if not mask_point: return 0 top = depth_point diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 7804c0b6..4bc1510b 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""SCRIP based interpolation""" import os import shutil import threading @@ -14,6 +15,7 @@ from earthdiagnostics.utils import Utils, TempFile class Interpolate(Diagnostic): """ 3-dimensional conservative interpolation to the regular atmospheric grid. + It can also be used for 2D (i,j) variables :original author: Virginie Guemas diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index 1f09b522..a5807908 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Changes values present in the mask for NaNs""" import numpy as np from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableListOption, \ @@ -10,9 +11,6 @@ class MaskLand(Diagnostic): """ Changes values present in the mask for NaNs - :created: February 2012 - :last modified: June 2016 - :param data_manager: data management object :type data_manager: DataManager :param startdate: startdate diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index b5985e4b..56895479 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute an Atlantic MOC index""" import netCDF4 import numpy as np from bscearth.utils.log import Log @@ -13,7 +14,9 @@ from earthdiagnostics.variable_type import VariableType class MaxMoc(Diagnostic): """ - Compute an Atlantic MOC index by finding the maximum of the annual + Compute an Atlantic MOC index + + It finds the maximum of the annual mean meridional overturning in a latitude / depth region :original author: Virginie Guemas diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index a61c13d7..fa03a558 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute mixed layer heat content""" import os from earthdiagnostics import cdftools diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index 924a5d20..f41c7b1b 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute mixed layer salt content""" import os from earthdiagnostics import cdftools diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index a94e1e00..a300e8b1 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute the MOC for oceanic basins""" import numpy as np from bscearth.utils.log import Log diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py index bc2ee396..1c733c83 100644 --- a/earthdiagnostics/ocean/mxl.py +++ b/earthdiagnostics/ocean/mxl.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute the mixed layer depth""" import os from earthdiagnostics import cdftools diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index 9773e02c..03107eda 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute the barotropic stream function""" from earthdiagnostics import cdftools from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.modelingrealm import ModelingRealms -- GitLab From 3443c6e4592ee164743164014b4a9b9eca115de5 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 13:34:14 +0100 Subject: [PATCH 128/168] Cleaned code --- earthdiagnostics/__init__.py | 5 +-- earthdiagnostics/config.py | 2 +- earthdiagnostics/constants.py | 1 + earthdiagnostics/datamanager.py | 1 - earthdiagnostics/general/attribute.py | 1 - earthdiagnostics/general/relink.py | 2 +- earthdiagnostics/general/relinkall.py | 2 +- earthdiagnostics/general/rewrite.py | 1 - earthdiagnostics/general/scale.py | 2 -- earthdiagnostics/ocean/rotation.py | 2 +- earthdiagnostics/ocean/siasiesiv.py | 33 +++++++------------ .../statistics/climatologicalpercentile.py | 2 +- .../statistics/daysoverpercentile.py | 5 +-- 13 files changed, 20 insertions(+), 39 deletions(-) diff --git a/earthdiagnostics/__init__.py b/earthdiagnostics/__init__.py index e442aee3..b6f6a1bd 100644 --- a/earthdiagnostics/__init__.py +++ b/earthdiagnostics/__init__.py @@ -1,11 +1,8 @@ # coding=utf-8 -""" -Module containing the Earth Diagnostics. -""" +"""Module containing the Earth Diagnostics.""" import os from earthdiagnostics.cdftools import CDFTools cdftools = CDFTools() DEVNULL = open(os.devnull, 'wb') - diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 1afdc713..ab279a57 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -71,7 +71,7 @@ class Config(object): self.experiment = ExperimentConfig() """ Configuration related to the experiment - + Returns ------- ExperimentConfig diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index 4d18c334..01e479d9 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -163,6 +163,7 @@ class Basins(object): def parse(self, basin): """ Return the basin matching the given name. + If the parameter basin is a Basin instance, directly returns the same instance. This bahaviour is intended to facilitate the development of methods that can either accept a nameor a Basin instance to characterize the basin. diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 16759134..c077825c 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -377,4 +377,3 @@ class UnitConversion(object): return 1 / conversion.factor, -conversion.offset else: return None, None - diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index a6801ebc..be14607e 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -97,4 +97,3 @@ class Attribute(Diagnostic): self.attributte_value)) self.corrected.set_local_file(variable_file, self) - diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 464508cf..6479ee5b 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Create links for a variable""" from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption, \ DiagnosticVariableOption @@ -85,4 +86,3 @@ class Relink(Diagnostic): self.data_manager.link_file(self.domain, self.variable, self.var_manager.get_variable(self.variable), self.startdate, self.member, self.chunk, move_old=self.move_old, grid=self.grid) - diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 0aa90ff0..1f1dd5d3 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Create links for all variables in a startdate""" from earthdiagnostics.diagnostic import Diagnostic @@ -58,4 +59,3 @@ class RelinkAll(Diagnostic): def compute(self): """Run the diagnostic""" self.data_manager.create_links(self.startdate) - diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index 2cae2046..8a332ee9 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -80,4 +80,3 @@ class Rewrite(Diagnostic): def compute(self): """Run the diagnostic""" self.corrected.set_local_file(self.variable_file.local_file, self) - diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 21b032b6..02e51504 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -130,5 +130,3 @@ class Scale(Diagnostic): if not math.isnan(self.max_limit) and (self.original_values.max() > self.max_limit): return False return True - - diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 0f2c72ed..eeb38ff9 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -10,7 +10,7 @@ from earthdiagnostics.utils import Utils, TempFile class Rotation(Diagnostic): """ - Cuts a meridional or zonal section + Cut a meridional or zonal section :original author: Virginie Guemas :contributor: Javier Vegas-Regidor diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 320defad..c861d8f7 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute the sea ice extent , area and volume in both hemispheres or a specified region""" import os import netCDF4 @@ -18,15 +19,17 @@ class Siasiesiv(Diagnostic): """ Compute the sea ice extent , area and volume in both hemispheres or a specified region. - - :original author: Virginie Guemas - :contributor: Neven Fuckar - :contributor: Ruben Cruz - :contributor: Javier Vegas-Regidor - - :created: April 2012 - :last modified: June 2016 - + Parameters + ---------- + data_manager: DataManager + startdate: str + member: int + chunk: init + domain: ModellingRealm + variable: str + basin: Basin + mask: numpy.array + omit_vol: bool """ alias = 'siasiesiv' "Diagnostic alias for the configuration file" @@ -36,18 +39,6 @@ class Siasiesiv(Diagnostic): gphit = None def __init__(self, data_manager, startdate, member, chunk, basin, mask, var_manager, omit_vol): - """ - :param data_manager: data management object - :type data_manager: DataManager - :param startdate: startdate - :type startdate: str - :param member: member number - :type member: int - :param chunk: chunk's number - :type chunk: int - :param mask: mask to use - :type mask: numpy.array - """ Diagnostic.__init__(self, data_manager) self.basin = basin self.startdate = startdate diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 20d5ad5d..d33503b7 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -81,7 +81,7 @@ class ClimatologicalPercentile(Diagnostic): return job_list def requested_startdates(self): - return ['{0}{1:02}01'.format(year, self.forecast_month) for year in range(self.start_year, self.end_year+1)] + return ['{0}{1:02}01'.format(year, self.forecast_month) for year in range(self.start_year, self.end_year + 1)] def request_data(self): """Request data required by the diagnostic""" diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index d6f30bc0..4e11c116 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -1,8 +1,5 @@ # coding=utf-8 -""" -Diagnostic to ciompute statistics about days above and below a set of -precomputed percentiles -""" +"""Diagnostic to coompute statistics about days above and below a set ofprecomputed percentiles""" import os import iris -- GitLab From c4352253c2f091beb2707ccda2cff4bb6af4d118 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 14:44:49 +0100 Subject: [PATCH 129/168] Corrected empty lines at end of file --- earthdiagnostics/general/select_levels.py | 1 - earthdiagnostics/general/simplify_dimensions.py | 1 - earthdiagnostics/general/timemean.py | 2 -- earthdiagnostics/modelingrealm.py | 1 - earthdiagnostics/obsreconmanager.py | 1 - earthdiagnostics/ocean/mask_land.py | 1 - earthdiagnostics/ocean/regionmean.py | 1 - earthdiagnostics/ocean/rotation.py | 4 ---- earthdiagnostics/ocean/siasiesiv.py | 1 - earthdiagnostics/ocean/verticalgradient.py | 1 - earthdiagnostics/ocean/verticalmean.py | 1 - earthdiagnostics/statistics/discretize.py | 9 --------- earthdiagnostics/statistics/monthlypercentile.py | 3 --- 13 files changed, 27 deletions(-) diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index 947ef850..a0c92cfe 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -111,4 +111,3 @@ class SelectLevels(Diagnostic): dimensions=(var_name, 'vertices')) new_lat_vertices[:] = vertices_values Utils.copy_attributes(new_lat_vertices, var_vertices) - diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index 2ed5eb45..d9c038d3 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -148,4 +148,3 @@ class SimplifyDimensions(Diagnostic): dimensions=(var_name, 'vertices')) new_lat_vertices[:] = vertices_values Utils.copy_attributes(new_lat_vertices, var_vertices) - diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index 5aaa4458..c09b4280 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -238,5 +238,3 @@ class YearlyMean(TimeMean): """ Utils.cdo.monmean(input=input_file, output=output_file) - - diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index 1502b55d..f76afbcf 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -155,4 +155,3 @@ class ModelingRealms(object): if modelling_realm.lower() in [value.name.lower()]: return value raise ValueError('Modelling realm {0} not recognized!'.format(modelling_realm)) - diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index 0f586300..2301ceec 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -151,4 +151,3 @@ class ObsReconManager(DataManager): netcdf_file.frequency = frequency Log.debug('{0} will be generated', filepath) return netcdf_file - diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index a5807908..01062c11 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -107,4 +107,3 @@ class MaskLand(Diagnostic): handler.close() self.masked_file.set_local_file(temp) - diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index a1bc0388..6e4496bb 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -188,4 +188,3 @@ class RegionMean(Diagnostic): self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, box=box_save, region=self.basin, grid=self.grid) - diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index eeb38ff9..897c4a88 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -181,7 +181,3 @@ class Rotation(Diagnostic): self.tempTemplate = TempFile.get(suffix='_01.nc') # self.tempTemplate = 'temp_01.nc' return self.tempTemplate.replace('_01.nc', '_{1}_{0:02d}.nc'.format(lev + 1, direction)) - - - - diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index c861d8f7..7ecf9cce 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -163,4 +163,3 @@ class Siasiesiv(Diagnostic): new_var.valid_max = np.max(values) handler.close() self.generated[cmor_name].set_local_file(temp) - diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 8331db7b..13378783 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -114,4 +114,3 @@ class VerticalGradient(Diagnostic): new_var.standard_name += '_vertical_gradient' self.gradient_file.set_local_file(temp) - diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 1b3f8624..4f3afe5f 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -110,4 +110,3 @@ class VerticalMean(Diagnostic): options=[self.variable, 'T', lev_min, lev_max, '-debug']) Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) self.results.set_local_file(temp, rename_var='{0}_vert_mean'.format(self.variable)) - diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index a5c4df7a..b190b62d 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -263,12 +263,3 @@ class Discretize(Diagnostic): return histogram return np.apply_along_axis(calculate_histogram, 0, data_cube.data) - - - - - - - - - diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index 53a2bea6..fad51a01 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -157,6 +157,3 @@ class MonthlyPercentile(Diagnostic): handler.variables[self.variable].long_name += ' Monthly Minimum' handler.close() self.min_file.set_local_file(monmin_file, rename_var=self.variable) - - - -- GitLab From b633c8298aaf9e2688c36458f717b3d837864db6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 14:50:13 +0100 Subject: [PATCH 130/168] Fixed sepration between summary and description in docstrings --- earthdiagnostics/config.py | 3 +++ earthdiagnostics/general/attribute.py | 5 +++-- earthdiagnostics/general/module.py | 4 +--- earthdiagnostics/general/rewrite.py | 1 + earthdiagnostics/obsreconmanager.py | 1 + earthdiagnostics/ocean/areamoc.py | 4 +++- earthdiagnostics/ocean/verticalgradient.py | 3 +-- earthdiagnostics/ocean/verticalmean.py | 3 +-- earthdiagnostics/threddsmanager.py | 10 +--------- 9 files changed, 15 insertions(+), 19 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index ab279a57..5f358500 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -500,6 +500,7 @@ class ExperimentConfig(object): def get_chunk_list(self): """ Return a list with all the chunks + :return: List containing tuples of startdate, member and chunk :rtype: tuple[str, int, int] """ @@ -517,6 +518,7 @@ class ExperimentConfig(object): def get_member_list(self): """ Return a list with all the members + :return: List containing tuples of startdate and member :rtype: tuple[str, int, int] """ @@ -529,6 +531,7 @@ class ExperimentConfig(object): def get_year_chunks(self, startdate, year): """ Get the list of chunks containing timesteps from the given year + :param startdate: startdate to use :type startdate: str :param year: reference year diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index be14607e..68888ef0 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -6,8 +6,9 @@ from earthdiagnostics.utils import Utils class Attribute(Diagnostic): """ - Rewrite files without doing any calculations. - Can be useful to convert units or to correct wrong metadata + Set the value of an attribute + + Can be useful to correct wrong metadata :original author: Javier Vegas-Regidor diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index 0defb706..835f81ad 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -7,9 +7,7 @@ from earthdiagnostics.utils import Utils, TempFile class Module(Diagnostic): """ - Scales a variable by the given value also adding at offset - Can be useful to correct units or other known errors - (think of a tas file declaring K as units but with the data stored as Celsius) + Compute the module of the vector given by two scalar variables :original author: Javier Vegas-Regidor diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index 8a332ee9..e0628b00 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -5,6 +5,7 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic class Rewrite(Diagnostic): """ Rewrites files without doing any calculations. + Can be useful to convert units or to correct wrong metadata :original author: Javier Vegas-Regidor diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index 2301ceec..d576f34d 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -43,6 +43,7 @@ class ObsReconManager(DataManager): box=None, grid=None): """ Return the path to a concrete file + :param startdate: file's startdate :type startdate: str :param domain: file's domain diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 6b66b1cc..17bfe58c 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -12,7 +12,9 @@ from earthdiagnostics.utils import Utils, TempFile class AreaMoc(Diagnostic): """ - Compute an Atlantic MOC index by averaging the meridional overturning + Compute an Atlantic MOC index + + Averages the meridional overturning in a latitude band between 1km and 2km or any other index averaging the meridional overturning in a given basin and a given domain diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 13378783..2beceb51 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -7,8 +7,7 @@ from earthdiagnostics.utils import Utils, TempFile class VerticalGradient(Diagnostic): """ - Chooses vertical level in ocean, or vertically averages between - 2 or more ocean levels + Calculate the gradue between 2 ocean levels :original author: Virginie Guemas :contributor: Eleftheria Exarchou diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 4f3afe5f..4558c8a4 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -8,8 +8,7 @@ from earthdiagnostics.utils import Utils, TempFile class VerticalMean(Diagnostic): """ - Chooses vertical level in ocean, or vertically averages between - 2 or more ocean levels + Chooses vertical level in ocean, or vertically averages between 2 or more ocean levels :original author: Virginie Guemas :contributor: Eleftheria Exarchou diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 7c4a1415..6ffecbb8 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -26,6 +26,7 @@ class THREDDSManager(DataManager): ---------- config: Config """ + def __init__(self, config): super(THREDDSManager, self).__init__(config) self.server_url = config.thredds.server_url @@ -284,15 +285,6 @@ class THREDDSSubset(DataFile): """ def __init__(self, thredds_path, file_path, var, start_time, end_time): - """ - - :param thredds_path: - :param file_path: - :param var: - :type var: str - :param start_time: - :param end_time: - """ super(THREDDSSubset, self).__init__() self.thredds_path = thredds_path self.remote_file = file_path -- GitLab From 91d8b9ad8d395505ab63e86a53952e39a1148434 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 14:53:15 +0100 Subject: [PATCH 131/168] Fixed blank line after class docstring --- earthdiagnostics/constants.py | 5 ++--- earthdiagnostics/earthdiags.py | 1 + earthdiagnostics/modelingrealm.py | 1 + earthdiagnostics/obsreconmanager.py | 2 +- earthdiagnostics/ocean/mixedlayersaltcontent.py | 1 + earthdiagnostics/ocean/siasiesiv.py | 1 + earthdiagnostics/threddsmanager.py | 1 - 7 files changed, 7 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index 01e479d9..3e2bf040 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -35,9 +35,8 @@ class Basin(object): class Basins(object): - """ - Singleton class to manage available basins - """ + """Singleton class to manage available basins""" + __instance = None def __new__(cls, *args, **kwargs): diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index e6d602af..515a33a4 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -30,6 +30,7 @@ class EarthDiags(object): ---------- config_file: str """ + # Get the version number from the relevant file. If not, from autosubmit package scriptdir = os.path.abspath(os.path.dirname(__file__)) if not os.path.exists(os.path.join(scriptdir, 'VERSION')): diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index f76afbcf..0c8987d1 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -111,6 +111,7 @@ class ModelingRealm(object): class ModelingRealms(object): """Enumeration of supported modelling realms""" + seaIce = ModelingRealm('seaice') ocean = ModelingRealm('ocean') landIce = ModelingRealm('landIce') diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index d576f34d..5f3d4c5f 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -21,8 +21,8 @@ class ObsReconManager(DataManager): Parameters ---------- config: Config - """ + def __init__(self, config): super(ObsReconManager, self).__init__(config) data_folders = self.config.data_dir.split(':') diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index f41c7b1b..a6f7b455 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -27,6 +27,7 @@ class MixedLayerSaltContent(Diagnostic): :param chunk: chunk's number :type chunk: int """ + alias = 'mlotstsc' "Diagnostic alias for the configuration file" diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 7ecf9cce..53b1c728 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -31,6 +31,7 @@ class Siasiesiv(Diagnostic): mask: numpy.array omit_vol: bool """ + alias = 'siasiesiv' "Diagnostic alias for the configuration file" diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 6ffecbb8..7aa4c6a3 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -281,7 +281,6 @@ class THREDDSSubset(DataFile): var: str start_time: datetime end_time: datetime - """ def __init__(self, thredds_path, file_path, var, start_time, end_time): -- GitLab From ecefa0179831f7d698a8290883975443341e094e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 15:04:22 +0100 Subject: [PATCH 132/168] Added module docstrings --- earthdiagnostics/cmormanager.py | 2 +- earthdiagnostics/datafile.py | 2 +- earthdiagnostics/datamanager.py | 2 +- earthdiagnostics/diagnostic.py | 2 +- earthdiagnostics/frequency.py | 3 +- earthdiagnostics/general/attribute.py | 1 + earthdiagnostics/general/module.py | 1 + earthdiagnostics/general/rewrite.py | 1 + earthdiagnostics/general/scale.py | 1 + earthdiagnostics/general/select_levels.py | 1 + .../general/simplify_dimensions.py | 3 +- .../general/verticalmeanmetersiris.py | 3 +- earthdiagnostics/obsreconmanager.py | 2 +- earthdiagnostics/ocean/areamoc.py | 1 + earthdiagnostics/ocean/averagesection.py | 4 +- earthdiagnostics/ocean/convectionsites.py | 1 + earthdiagnostics/ocean/heatcontentlayer.py | 1 + earthdiagnostics/ocean/maxmoc.py | 2 +- earthdiagnostics/ocean/rotation.py | 3 +- earthdiagnostics/ocean/verticalgradient.py | 3 +- earthdiagnostics/ocean/verticalmean.py | 1 + earthdiagnostics/ocean/verticalmeanmeters.py | 1 + earthdiagnostics/singleton.py | 8 -- .../statistics/climatologicalpercentile.py | 3 +- .../statistics/daysoverpercentile.py | 2 +- earthdiagnostics/statistics/discretize.py | 2 +- .../statistics/monthlypercentile.py | 3 +- earthdiagnostics/threddsmanager.py | 2 +- earthdiagnostics/variable.py | 15 ++ earthdiagnostics/variable_type.py | 13 -- test/unit/general/test_attribute.py | 2 +- test/unit/general/test_dailymean.py | 2 +- test/unit/general/test_module.py | 2 +- test/unit/general/test_monthlymean.py | 2 +- test/unit/general/test_relink.py | 2 +- test/unit/general/test_relinkall.py | 2 +- test/unit/general/test_rewrite.py | 2 +- test/unit/general/test_scale.py | 2 +- test/unit/general/test_select_levels.py | 2 +- test/unit/general/test_simplify_dimensions.py | 2 +- .../general/test_verticalmeanmetersiris.py | 2 +- test/unit/general/test_yearlymean.py | 2 +- test/unit/ocean/test_areamoc.py | 2 +- test/unit/ocean/test_averagesection.py | 2 +- test/unit/ocean/test_convectionsites.py | 2 +- test/unit/ocean/test_cutsection.py | 2 +- test/unit/ocean/test_gyres.py | 2 +- test/unit/ocean/test_heatcontent.py | 2 +- test/unit/ocean/test_heatcontentlayer.py | 2 +- test/unit/ocean/test_interpolate.py | 2 +- test/unit/ocean/test_interpolatecdo.py | 2 +- test/unit/ocean/test_maskland.py | 2 +- test/unit/ocean/test_maxmoc.py | 2 +- test/unit/ocean/test_mixedlayerheatcontent.py | 2 +- test/unit/ocean/test_mixedlayersaltcontent.py | 2 +- test/unit/ocean/test_moc.py | 2 +- test/unit/ocean/test_mxl.py | 2 +- test/unit/ocean/test_psi.py | 2 +- test/unit/ocean/test_region_mean.py | 2 +- test/unit/ocean/test_siasiesiv.py | 2 +- test/unit/ocean/test_vertical_gradient.py | 2 +- test/unit/ocean/test_verticalmean.py | 2 +- test/unit/ocean/test_verticalmeanmeters.py | 2 +- .../test_climatologicalpercentile.py | 2 +- .../statistics/test_daysoverpercentile.py | 2 +- test/unit/statistics/test_discretize.py | 4 +- .../unit/statistics/test_monthlypercentile.py | 2 +- test/unit/test_box.py | 32 ++--- test/unit/test_config.py | 132 +++++++++--------- test/unit/test_constants.py | 4 +- test/unit/test_diagnostic.py | 2 +- test/unit/test_frequency.py | 2 +- test/unit/test_modelling_realm.py | 6 +- test/unit/test_utils.py | 10 +- test/unit/test_variable.py | 8 +- test/unit/test_variable_type.py | 2 +- 76 files changed, 188 insertions(+), 175 deletions(-) delete mode 100644 earthdiagnostics/singleton.py delete mode 100644 earthdiagnostics/variable_type.py diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 9682cd2f..d4268089 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -16,7 +16,7 @@ from earthdiagnostics.datamanager import DataManager from earthdiagnostics.frequency import Frequencies, Frequency from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import TempFile, Utils -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class CMORManager(DataManager): diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 5c19f458..f70413c3 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -12,7 +12,7 @@ from bscearth.utils.log import Log from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile from publisher import Publisher -from variable_type import VariableType +from earthdiagnostics.variable import VariableType class LocalStatus(object): diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index c077825c..a07b04f0 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -6,7 +6,7 @@ import threading from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class DataManager(object): diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 11fb3ae0..e743c807 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -9,7 +9,7 @@ from earthdiagnostics.datafile import StorageStatus, LocalStatus from earthdiagnostics.frequency import Frequency from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.publisher import Publisher -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class DiagnosticStatus(object): diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py index 9c04c072..2be71c33 100644 --- a/earthdiagnostics/frequency.py +++ b/earthdiagnostics/frequency.py @@ -1,6 +1,6 @@ # coding=utf-8 """Data frequency management tools""" -from earthdiagnostics.variable_type import VariableType + class Frequency(object): @@ -49,6 +49,7 @@ class Frequency(object): ------- str """ + from earthdiagnostics.variable import VariableType if self == Frequencies.daily: freq_str = 'daily_{0}'.format(VariableType.to_str(vartype)) elif self == Frequencies.weekly: diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 68888ef0..f80e9718 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Set attributtes in netCDF files""" from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticComplexStrOption, \ DiagnosticDomainOption, DiagnosticVariableOption from earthdiagnostics.utils import Utils diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index 835f81ad..0b07132b 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute module of two variables""" import numpy as np from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, DiagnosticDomainOption, DiagnosticOption diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index e0628b00..10bd7233 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Rewrite netCDF file""" from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 02e51504..fa540cdb 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,4 +1,5 @@ # coding=utf-8 +""" Scales a variable by with value and offset""" import math import numpy as np diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index a0c92cfe..239c2df1 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Extract levels from variable""" from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption, DiagnosticIntOption diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index d9c038d3..6f73d379 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Convert i j files to lon lat when there is no interpolation required""" import numpy as np from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ @@ -8,7 +9,7 @@ from earthdiagnostics.utils import Utils, TempFile class SimplifyDimensions(Diagnostic): """ - Convert i j files to lon lat when there is no interpolation required, + Convert i j files to lon lat when there is no interpolation required i.e. lon is constant over i and lat is constat over j diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py index a2a20f54..4cc5f394 100644 --- a/earthdiagnostics/general/verticalmeanmetersiris.py +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Average vertically any variable""" import iris import iris.analysis import iris.exceptions @@ -12,7 +13,7 @@ from earthdiagnostics.utils import TempFile class VerticalMeanMetersIris(Diagnostic): """ - Averages vertically any given variable + Average vertically any given variable :original author: Virginie Guemas :contributor: Javier Vegas-Regidor diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index 5f3d4c5f..40c9869f 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -11,7 +11,7 @@ from bscearth.utils.log import Log from earthdiagnostics.datamanager import DataManager from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class ObsReconManager(DataManager): diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 17bfe58c..4831a1b9 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute an Atlantic MOC index from the average""" import os import numpy as np diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 7047099f..5f7f9c99 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute an average of a given zone""" import os from earthdiagnostics.box import Box @@ -9,7 +10,8 @@ from earthdiagnostics.utils import Utils, TempFile class AverageSection(Diagnostic): """ - Compute an average of a given zone. The variable MUST be in a regular grid + Compute an average of a given zone. + The variable MUST be in a regular grid :original author: Virginie Guemas :contributor: Javier Vegas-Regidor diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index 5e23cd49..08523f34 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Compute the intensity of convection""" import numpy as np from bscearth.utils.log import Log diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 423087cc..998a371f 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Point-wise Ocean Heat Content in a specified ocean thickness (J/m-2)""" import numpy as np from earthdiagnostics.box import Box diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index 56895479..29468881 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -9,7 +9,7 @@ from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticFloatOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class MaxMoc(Diagnostic): diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 897c4a88..fe3aa73c 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Rotate two u v variables to align with latitude and longitude""" import shutil from bscearth.utils.log import Log @@ -10,7 +11,7 @@ from earthdiagnostics.utils import Utils, TempFile class Rotation(Diagnostic): """ - Cut a meridional or zonal section + Rotate two u v variables to align with latitude and longitude :original author: Virginie Guemas :contributor: Javier Vegas-Regidor diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 2beceb51..64a933ed 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Calculate the gradient between 2 ocean levels""" from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticVariableOption from earthdiagnostics.modelingrealm import ModelingRealms @@ -7,7 +8,7 @@ from earthdiagnostics.utils import Utils, TempFile class VerticalGradient(Diagnostic): """ - Calculate the gradue between 2 ocean levels + Calculate the gradient between 2 ocean levels :original author: Virginie Guemas :contributor: Eleftheria Exarchou diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 4558c8a4..cf05d5eb 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Chooses vertical level in ocean, or vertically averages between 2 or more ocean levels""" from earthdiagnostics import cdftools from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticVariableOption diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 9cd3833c..ee9d2084 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Averages vertically any given variable""" from earthdiagnostics import cdftools from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticFloatOption, DiagnosticDomainOption, \ diff --git a/earthdiagnostics/singleton.py b/earthdiagnostics/singleton.py deleted file mode 100644 index 2cd1bb8d..00000000 --- a/earthdiagnostics/singleton.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -class SingletonType(type): - def __call__(cls, *args): - try: - return cls.__instance - except AttributeError: - cls.__instance = super(SingletonType, cls).__call__(*args) - return cls.__instance diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index d33503b7..d31d0ea3 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Calculates the climatological percentiles for the given leadtime""" import iris import iris.coord_categorisation import iris.coords @@ -11,7 +12,7 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, Di DiagnosticIntOption, DiagnosticListIntOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import TempFile -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class ClimatologicalPercentile(Diagnostic): diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index 4e11c116..ad1b25f3 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -17,7 +17,7 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticDomainOption, Diag from earthdiagnostics.frequency import Frequencies from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class DaysOverPercentile(Diagnostic): diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index b190b62d..e577d90b 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -18,7 +18,7 @@ from iris.time import PartialDateTime from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, DiagnosticDomainOption, \ DiagnosticIntOption, DiagnosticFloatOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class Discretize(Diagnostic): diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index fad51a01..af1c8572 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Calculates the montlhy percentiles""" from calendar import monthrange from bscearth.utils.log import Log @@ -6,7 +7,7 @@ from bscearth.utils.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class MonthlyPercentile(Diagnostic): diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 7aa4c6a3..a0909af1 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -15,7 +15,7 @@ from iris.coords import DimCoord from datafile import DataFile, StorageStatus, LocalStatus from earthdiagnostics.datamanager import DataManager from earthdiagnostics.utils import TempFile, Utils -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class THREDDSManager(DataManager): diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index a155e3b8..7ee81bff 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -532,3 +532,18 @@ class CMORTable(object): def __lt__(self, other): return self.name < other.name + + +class VariableType(object): + """Enumeration of variable types""" + MEAN = 1 + STATISTIC = 2 + + @staticmethod + def to_str(vartype): + if vartype == VariableType.MEAN: + return 'mean' + elif vartype == VariableType.STATISTIC: + return 'statistics' + else: + raise ValueError('Variable type {0} not supported'.format(vartype)) diff --git a/earthdiagnostics/variable_type.py b/earthdiagnostics/variable_type.py deleted file mode 100644 index 4b3f17da..00000000 --- a/earthdiagnostics/variable_type.py +++ /dev/null @@ -1,13 +0,0 @@ -# coding=utf-8 -class VariableType(object): - MEAN = 1 - STATISTIC = 2 - - @staticmethod - def to_str(vartype): - if vartype == VariableType.MEAN: - return 'mean' - elif vartype == VariableType.STATISTIC: - return 'statistics' - else: - raise ValueError('Variable type {0} not supported'.format(vartype)) diff --git a/test/unit/general/test_attribute.py b/test/unit/general/test_attribute.py index da6d4146..5616b347 100644 --- a/test/unit/general/test_attribute.py +++ b/test/unit/general/test_attribute.py @@ -50,5 +50,5 @@ class TestAttribute(TestCase): def test_str(self): mixed = Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'att', 'value') - self.assertEquals(str(mixed), 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 ' + self.assertEqual(str(mixed), 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 ' 'Variable: atmos:var Attributte: att:value Grid: grid') diff --git a/test/unit/general/test_dailymean.py b/test/unit/general/test_dailymean.py index e076a247..8fa092c6 100644 --- a/test/unit/general/test_dailymean.py +++ b/test/unit/general/test_dailymean.py @@ -51,5 +51,5 @@ class TestDailyMean(TestCase): def test_str(self): mixed = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') - self.assertEquals(str(mixed), 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' + self.assertEqual(str(mixed), 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_module.py b/test/unit/general/test_module.py index e01ecf8c..218223b9 100644 --- a/test/unit/general/test_module.py +++ b/test/unit/general/test_module.py @@ -50,5 +50,5 @@ class TestModule(TestCase): def test_str(self): mixed = Module(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'varu', 'varv', 'varmodule', 'grid') - self.assertEquals(str(mixed), 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' + self.assertEqual(str(mixed), 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' 'Variables: atmos:varu,varv,varmodule Grid: grid') diff --git a/test/unit/general/test_monthlymean.py b/test/unit/general/test_monthlymean.py index 401a8804..adc75cab 100644 --- a/test/unit/general/test_monthlymean.py +++ b/test/unit/general/test_monthlymean.py @@ -51,5 +51,5 @@ class TestMonthlyMean(TestCase): MonthlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + self.assertEqual(str(self.mixed), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_relink.py b/test/unit/general/test_relink.py index 835b0bcd..970bb0ea 100644 --- a/test/unit/general/test_relink.py +++ b/test/unit/general/test_relink.py @@ -57,5 +57,5 @@ class TestRelink(TestCase): def test_str(self): mixed = Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', True, 'grid') - self.assertEquals(str(mixed), 'Relink output Startdate: 20010101 Member: 0 Chunk: 0 Move old: True ' + self.assertEqual(str(mixed), 'Relink output Startdate: 20010101 Member: 0 Chunk: 0 Move old: True ' 'Variable: ocean:var Grid: grid') diff --git a/test/unit/general/test_relinkall.py b/test/unit/general/test_relinkall.py index cda393c8..22bd7612 100644 --- a/test/unit/general/test_relinkall.py +++ b/test/unit/general/test_relinkall.py @@ -34,4 +34,4 @@ class TestRelinkAll(TestCase): def test_str(self): mixed = RelinkAll(self.data_manager, '20010101') - self.assertEquals(str(mixed), 'Relink all output Startdate: 20010101') + self.assertEqual(str(mixed), 'Relink all output Startdate: 20010101') diff --git a/test/unit/general/test_rewrite.py b/test/unit/general/test_rewrite.py index 202e2c39..4a6fd038 100644 --- a/test/unit/general/test_rewrite.py +++ b/test/unit/general/test_rewrite.py @@ -47,5 +47,5 @@ class TestRewrite(TestCase): Rewrite.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 ' + self.assertEqual(str(self.mixed), 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: atmos:var Grid: grid') diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py index b2fcd0d0..23e1f7b7 100644 --- a/test/unit/general/test_scale.py +++ b/test/unit/general/test_scale.py @@ -75,5 +75,5 @@ class TestScale(TestCase): def test_str(self): mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, Frequencies.three_hourly, False) - self.assertEquals(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' + self.assertEqual(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' 'Variable: atmos:var Frequency: 3hr Apply mask: False') diff --git a/test/unit/general/test_select_levels.py b/test/unit/general/test_select_levels.py index 32e7424d..bca61711 100644 --- a/test/unit/general/test_select_levels.py +++ b/test/unit/general/test_select_levels.py @@ -62,5 +62,5 @@ class TestSelectLevels(TestCase): def test_str(self): mixed = SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 20) - self.assertEquals(str(mixed), 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + self.assertEqual(str(mixed), 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Levels: 0-20 Grid: grid') diff --git a/test/unit/general/test_simplify_dimensions.py b/test/unit/general/test_simplify_dimensions.py index 48ac6f7c..5a36288e 100644 --- a/test/unit/general/test_simplify_dimensions.py +++ b/test/unit/general/test_simplify_dimensions.py @@ -52,5 +52,5 @@ class TestSimplifyDimensions(TestCase): def test_str(self): mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'convention') - self.assertEquals(str(mixed), 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + self.assertEqual(str(mixed), 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Grid: grid') diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py index 179b54fa..ca972cac 100644 --- a/test/unit/general/test_verticalmeanmetersiris.py +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -71,5 +71,5 @@ class TestVerticalMeanMetersIris(TestCase): box.min_depth = 0 box.max_depth = 100 mixed = VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', box) - self.assertEquals(str(mixed), 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + self.assertEqual(str(mixed), 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Box: 0-100m') diff --git a/test/unit/general/test_yearlymean.py b/test/unit/general/test_yearlymean.py index bd1b7938..36b48eaf 100644 --- a/test/unit/general/test_yearlymean.py +++ b/test/unit/general/test_yearlymean.py @@ -51,5 +51,5 @@ class TestYearlyMean(TestCase): YearlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + self.assertEqual(str(self.mixed), 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/ocean/test_areamoc.py b/test/unit/ocean/test_areamoc.py index c1361b83..648ce675 100644 --- a/test/unit/ocean/test_areamoc.py +++ b/test/unit/ocean/test_areamoc.py @@ -53,5 +53,5 @@ class TestAreaMoc(TestCase): AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.psi), 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0 ' + self.assertEqual(str(self.psi), 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0 ' 'Basin: Atlantic') diff --git a/test/unit/ocean/test_averagesection.py b/test/unit/ocean/test_averagesection.py index d3be4b2b..a5905589 100644 --- a/test/unit/ocean/test_averagesection.py +++ b/test/unit/ocean/test_averagesection.py @@ -51,5 +51,5 @@ class TestAverageSection(TestCase): def test_str(self): diag = AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', self.box, 'grid') - self.assertEquals(str(diag), 'Average section Startdate: 20010101 Member: 0 Chunk: 0 Box: 0N0E ' + self.assertEqual(str(diag), 'Average section Startdate: 20010101 Member: 0 Chunk: 0 Box: 0N0E ' 'Variable: ocean:var Grid: grid') diff --git a/test/unit/ocean/test_convectionsites.py b/test/unit/ocean/test_convectionsites.py index 63710d05..c9468b08 100644 --- a/test/unit/ocean/test_convectionsites.py +++ b/test/unit/ocean/test_convectionsites.py @@ -25,4 +25,4 @@ class TestConvectionSites(TestCase): ConvectionSites.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.psi), 'Convection sites Startdate: 20000101 Member: 1 Chunk: 1') + self.assertEqual(str(self.psi), 'Convection sites Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_cutsection.py b/test/unit/ocean/test_cutsection.py index a658b06d..e27102a0 100644 --- a/test/unit/ocean/test_cutsection.py +++ b/test/unit/ocean/test_cutsection.py @@ -51,5 +51,5 @@ class TestCutSection(TestCase): CutSection.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.psi), 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' + self.assertEqual(str(self.psi), 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' 'Zonal: True Value: 0') diff --git a/test/unit/ocean/test_gyres.py b/test/unit/ocean/test_gyres.py index 8a025583..9d3a13d3 100644 --- a/test/unit/ocean/test_gyres.py +++ b/test/unit/ocean/test_gyres.py @@ -26,4 +26,4 @@ class TestGyres(TestCase): Gyres.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.gyres), 'Gyres Startdate: 20000101 Member: 1 Chunk: 1 Model version: model_version') + self.assertEqual(str(self.gyres), 'Gyres Startdate: 20000101 Member: 1 Chunk: 1 Model version: model_version') diff --git a/test/unit/ocean/test_heatcontent.py b/test/unit/ocean/test_heatcontent.py index f842bc13..a2387016 100644 --- a/test/unit/ocean/test_heatcontent.py +++ b/test/unit/ocean/test_heatcontent.py @@ -44,5 +44,5 @@ class TestHeatContent(TestCase): def test_str(self): diag = HeatContent(self.data_manager, '20010101', 0, 0, Basins().Global, -1, self.box, 1, 20) - self.assertEquals(str(diag), 'Heat content Startdate: 20010101 Member: 0 Chunk: 0 Mixed layer: -1 Box: 0-100 ' + self.assertEqual(str(diag), 'Heat content Startdate: 20010101 Member: 0 Chunk: 0 Mixed layer: -1 Box: 0-100 ' 'Basin: Global') diff --git a/test/unit/ocean/test_heatcontentlayer.py b/test/unit/ocean/test_heatcontentlayer.py index bf8135c6..9c197405 100644 --- a/test/unit/ocean/test_heatcontentlayer.py +++ b/test/unit/ocean/test_heatcontentlayer.py @@ -22,4 +22,4 @@ class TestHeatContentLayer(TestCase): def test_str(self): diag = HeatContentLayer(self.data_manager, '20000101', 1, 1, self.box, self.weight, 0, 10) - self.assertEquals(str(diag), 'Heat content layer Startdate: 20000101 Member: 1 Chunk: 1 Box: 0-100m') + self.assertEqual(str(diag), 'Heat content layer Startdate: 20000101 Member: 1 Chunk: 1 Box: 0-100m') diff --git a/test/unit/ocean/test_interpolate.py b/test/unit/ocean/test_interpolate.py index 4d19949f..8bd284dc 100644 --- a/test/unit/ocean/test_interpolate.py +++ b/test/unit/ocean/test_interpolate.py @@ -71,6 +71,6 @@ class TestInterpolate(TestCase): def test_str(self): diag = Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'model_version', True, 'original_grid') - self.assertEquals(str(diag), 'Interpolate Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + self.assertEqual(str(diag), 'Interpolate Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Target grid: grid Invert lat: True Model: model_version ' 'Original grid: original_grid') diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py index b7b87cf1..fccf47de 100644 --- a/test/unit/ocean/test_interpolatecdo.py +++ b/test/unit/ocean/test_interpolatecdo.py @@ -88,6 +88,6 @@ class TestInterpolate(TestCase): def test_str(self): diag = InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'atmos_grid', 'model_version', False, 'orig', None) - self.assertEquals(str(diag), 'Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var ' + self.assertEqual(str(diag), 'Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var ' 'Target grid: atmos_grid Original grid: orig Mask ocean: False ' 'Model: model_version') diff --git a/test/unit/ocean/test_maskland.py b/test/unit/ocean/test_maskland.py index ede00911..22f4994b 100644 --- a/test/unit/ocean/test_maskland.py +++ b/test/unit/ocean/test_maskland.py @@ -62,4 +62,4 @@ class TestMaskLand(TestCase): def test_str(self): diag = MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 't', 'grid') - self.assertEquals(str(diag), 'Land mask Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var Grid: grid') + self.assertEqual(str(diag), 'Land mask Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var Grid: grid') diff --git a/test/unit/ocean/test_maxmoc.py b/test/unit/ocean/test_maxmoc.py index a1141d01..3224050e 100644 --- a/test/unit/ocean/test_maxmoc.py +++ b/test/unit/ocean/test_maxmoc.py @@ -63,5 +63,5 @@ class TestMaxMoc(TestCase): MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.maxmoc), 'Max moc Startdate: 20000101 Member: 1 Year: 2000 ' + self.assertEqual(str(self.maxmoc), 'Max moc Startdate: 20000101 Member: 1 Year: 2000 ' 'Box: 0.0N0m Basin: Global') diff --git a/test/unit/ocean/test_mixedlayerheatcontent.py b/test/unit/ocean/test_mixedlayerheatcontent.py index b7cf564b..8d94bd81 100644 --- a/test/unit/ocean/test_mixedlayerheatcontent.py +++ b/test/unit/ocean/test_mixedlayerheatcontent.py @@ -26,4 +26,4 @@ class TestMixedLayerHeatContent(TestCase): MixedLayerHeatContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'Mixed layer heat content Startdate: 20000101 Member: 1 Chunk: 1') + self.assertEqual(str(self.mixed), 'Mixed layer heat content Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_mixedlayersaltcontent.py b/test/unit/ocean/test_mixedlayersaltcontent.py index 7aa42f66..9832e3f8 100644 --- a/test/unit/ocean/test_mixedlayersaltcontent.py +++ b/test/unit/ocean/test_mixedlayersaltcontent.py @@ -26,4 +26,4 @@ class TestMixedLayerSaltContent(TestCase): MixedLayerSaltContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'Mixed layer salt content Startdate: 20000101 Member: 1 Chunk: 1') + self.assertEqual(str(self.mixed), 'Mixed layer salt content Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_moc.py b/test/unit/ocean/test_moc.py index 0c05f8c4..f86f41d4 100644 --- a/test/unit/ocean/test_moc.py +++ b/test/unit/ocean/test_moc.py @@ -26,4 +26,4 @@ class TestMoc(TestCase): Moc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'MOC Startdate: 20000101 Member: 1 Chunk: 1') + self.assertEqual(str(self.mixed), 'MOC Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_mxl.py b/test/unit/ocean/test_mxl.py index ead3fbbc..5b63aeea 100644 --- a/test/unit/ocean/test_mxl.py +++ b/test/unit/ocean/test_mxl.py @@ -25,4 +25,4 @@ class TestMxl(TestCase): def test_str(self): diag = Mxl(self.data_manager, '20010101', 0, 0) - self.assertEquals(str(diag), 'Mixed layer Startdate: 20010101 Member: 0 Chunk: 0') + self.assertEqual(str(diag), 'Mixed layer Startdate: 20010101 Member: 0 Chunk: 0') diff --git a/test/unit/ocean/test_psi.py b/test/unit/ocean/test_psi.py index 019e9339..1bdf597c 100644 --- a/test/unit/ocean/test_psi.py +++ b/test/unit/ocean/test_psi.py @@ -22,4 +22,4 @@ class TestPsi(TestCase): Psi.generate_jobs(self.diags, ['diagnostic', 'badoption']) def test_str(self): - self.assertEquals(str(self.psi), 'PSI Startdate: 20000101 Member: 1 Chunk: 1') + self.assertEqual(str(self.psi), 'PSI Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index efeeb7d7..5152c2fc 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -98,5 +98,5 @@ class TestRegionMean(TestCase): diag = RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', box, False, Basins().Global, True, 'grid') - self.assertEquals(str(diag), 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Grid point: U ' + self.assertEqual(str(diag), 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Grid point: U ' 'Box: 1-10 Save 3D: False Save variance: True Original grid: grid Basin: Global') diff --git a/test/unit/ocean/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py index c580e151..59ac3043 100644 --- a/test/unit/ocean/test_siasiesiv.py +++ b/test/unit/ocean/test_siasiesiv.py @@ -20,5 +20,5 @@ class TestSiasiesiv(TestCase): self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask, self.var_manager, False) def test_str(self): - self.assertEquals(str(self.psi), + self.assertEqual(str(self.psi), 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global Omit volume: False') diff --git a/test/unit/ocean/test_vertical_gradient.py b/test/unit/ocean/test_vertical_gradient.py index 892160a2..c55aeded 100644 --- a/test/unit/ocean/test_vertical_gradient.py +++ b/test/unit/ocean/test_vertical_gradient.py @@ -60,4 +60,4 @@ class TestVerticalGradient(TestCase): box.max_depth = 10 diag = VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box) - self.assertEquals(str(diag), 'Vertical gradient Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Box: 1-10') + self.assertEqual(str(diag), 'Vertical gradient Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Box: 1-10') diff --git a/test/unit/ocean/test_verticalmean.py b/test/unit/ocean/test_verticalmean.py index fc501afd..f6ae18c2 100644 --- a/test/unit/ocean/test_verticalmean.py +++ b/test/unit/ocean/test_verticalmean.py @@ -53,5 +53,5 @@ class TestVerticalMean(TestCase): VerticalMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var ' + self.assertEqual(str(self.mixed), 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var ' 'Box: 0-100') diff --git a/test/unit/ocean/test_verticalmeanmeters.py b/test/unit/ocean/test_verticalmeanmeters.py index f696d20d..ff6580a2 100644 --- a/test/unit/ocean/test_verticalmeanmeters.py +++ b/test/unit/ocean/test_verticalmeanmeters.py @@ -60,5 +60,5 @@ class TestVerticalMeanMeters(TestCase): VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 ' + self.assertEqual(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Box: 0-100m') diff --git a/test/unit/statistics/test_climatologicalpercentile.py b/test/unit/statistics/test_climatologicalpercentile.py index 12752caa..3a905cbb 100644 --- a/test/unit/statistics/test_climatologicalpercentile.py +++ b/test/unit/statistics/test_climatologicalpercentile.py @@ -38,5 +38,5 @@ class TestClimatologicalPercentile(TestCase): diagnostic = ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, 11, self.diags.config.experiment) - self.assertEquals(str(diagnostic), 'Climatological percentile Variable: ocean:var Period: 2000-2001 ' + self.assertEqual(str(diagnostic), 'Climatological percentile Variable: ocean:var Period: 2000-2001 ' 'Forecast month: 11') diff --git a/test/unit/statistics/test_daysoverpercentile.py b/test/unit/statistics/test_daysoverpercentile.py index 26e0506a..afc1a03e 100644 --- a/test/unit/statistics/test_daysoverpercentile.py +++ b/test/unit/statistics/test_daysoverpercentile.py @@ -30,5 +30,5 @@ class TestDaysOverPercentile(TestCase): def test_str(self): diagnostic = DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, '20001101', 11) - self.assertEquals(str(diagnostic), 'Days over percentile Startdate: 20001101 Variable: ocean:var ' + self.assertEqual(str(diagnostic), 'Days over percentile Startdate: 20001101 Variable: ocean:var ' 'Climatology: 2000-2001') diff --git a/test/unit/statistics/test_discretize.py b/test/unit/statistics/test_discretize.py index 402a3772..872c5109 100644 --- a/test/unit/statistics/test_discretize.py +++ b/test/unit/statistics/test_discretize.py @@ -50,11 +50,11 @@ class TestClimatologicalPercentile(TestCase): def test_str(self): diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, 10, 40) - self.assertEquals(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' + self.assertEqual(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' 'Range: [10, 40]') diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, float('nan'), float('nan')) - self.assertEquals(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' + self.assertEqual(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' 'Range: [None, None]') diff --git a/test/unit/statistics/test_monthlypercentile.py b/test/unit/statistics/test_monthlypercentile.py index a902ec89..8127d5a4 100644 --- a/test/unit/statistics/test_monthlypercentile.py +++ b/test/unit/statistics/test_monthlypercentile.py @@ -37,5 +37,5 @@ class TestMonthlyPercentile(TestCase): MonthlyPercentile.generate_jobs(self.diags, ['monpercent', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.diagnostic), 'Monthly percentile Startdate: 20000101 Member: 1 Chunk: 1 ' + self.assertEqual(str(self.diagnostic), 'Monthly percentile Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Percentiles: 10, 90') diff --git a/test/unit/test_box.py b/test/unit/test_box.py index 37b5ce49..8f69f2fa 100644 --- a/test/unit/test_box.py +++ b/test/unit/test_box.py @@ -69,25 +69,25 @@ class TestBox(TestCase): Box().min_lon = 80 def test_get_lat_str(self): - self.assertEquals('20S0N', self.box1.get_lat_str()) - self.assertEquals('20N', self.box2.get_lat_str()) - self.assertEquals('', self.box3.get_lat_str()) - self.assertEquals('20S10S', self.box4.get_lat_str()) + self.assertEqual('20S0N', self.box1.get_lat_str()) + self.assertEqual('20N', self.box2.get_lat_str()) + self.assertEqual('', self.box3.get_lat_str()) + self.assertEqual('20S10S', self.box4.get_lat_str()) def test_get_lon_str(self): - self.assertEquals('20W0E', self.box1.get_lon_str()) - self.assertEquals('20E', self.box2.get_lon_str()) - self.assertEquals('', self.box3.get_lon_str()) - self.assertEquals('20W10W', self.box4.get_lon_str()) + self.assertEqual('20W0E', self.box1.get_lon_str()) + self.assertEqual('20E', self.box2.get_lon_str()) + self.assertEqual('', self.box3.get_lon_str()) + self.assertEqual('20W10W', self.box4.get_lon_str()) def test_get_depth_str(self): - self.assertEquals('0-20', self.box1.get_depth_str()) - self.assertEquals('20m', self.box2.get_depth_str()) - self.assertEquals('', self.box3.get_depth_str()) - self.assertEquals('0-20', self.box4.get_depth_str()) + self.assertEqual('0-20', self.box1.get_depth_str()) + self.assertEqual('20m', self.box2.get_depth_str()) + self.assertEqual('', self.box3.get_depth_str()) + self.assertEqual('0-20', self.box4.get_depth_str()) def test__str__(self): - self.assertEquals('20S0N20W0E0-20', str(self.box1)) - self.assertEquals('20N20E20m', str(self.box2)) - self.assertEquals('', str(self.box3)) - self.assertEquals('20S10S20W10W0-20', str(self.box4)) + self.assertEqual('20S0N20W0E0-20', str(self.box1)) + self.assertEqual('20N20E20m', str(self.box2)) + self.assertEqual('', str(self.box3)) + self.assertEqual('20S10S20W10W0-20', str(self.box4)) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 0839998d..dbb21830 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -89,25 +89,25 @@ class TestCMORConfig(TestCase): def test_basic_config(self): config = CMORConfig(self.mock_parser, self.var_manager) - self.assertEquals(config.ocean, True) - self.assertEquals(config.atmosphere, True) - self.assertEquals(config.force, False) - self.assertEquals(config.force_untar, False) - self.assertEquals(config.use_grib, True) - self.assertEquals(config.activity, 'CMIP') - self.assertEquals(config.associated_experiment, 'to be filled') - self.assertEquals(config.associated_model, 'to be filled') - self.assertEquals(config.initialization_description, 'to be filled') - self.assertEquals(config.initialization_method, '1') - self.assertEquals(config.initialization_number, 1) - self.assertEquals(config.source, 'to be filled') - self.assertEquals(config.version, '') - self.assertEquals(config.physics_version, '1') - self.assertEquals(config.physics_description, 'to be filled') - self.assertEquals(config.filter_files, '') - self.assertEquals(config.default_atmos_grid, 'gr') - self.assertEquals(config.default_ocean_grid, 'gn') - self.assertEquals(config.min_cmorized_vars, 10) + self.assertEqual(config.ocean, True) + self.assertEqual(config.atmosphere, True) + self.assertEqual(config.force, False) + self.assertEqual(config.force_untar, False) + self.assertEqual(config.use_grib, True) + self.assertEqual(config.activity, 'CMIP') + self.assertEqual(config.associated_experiment, 'to be filled') + self.assertEqual(config.associated_model, 'to be filled') + self.assertEqual(config.initialization_description, 'to be filled') + self.assertEqual(config.initialization_method, '1') + self.assertEqual(config.initialization_number, 1) + self.assertEqual(config.source, 'to be filled') + self.assertEqual(config.version, '') + self.assertEqual(config.physics_version, '1') + self.assertEqual(config.physics_description, 'to be filled') + self.assertEqual(config.filter_files, '') + self.assertEqual(config.default_atmos_grid, 'gr') + self.assertEqual(config.default_ocean_grid, 'gn') + self.assertEqual(config.min_cmorized_vars, 10) def test_cmorize(self): config = CMORConfig(self.mock_parser, self.var_manager) @@ -198,57 +198,57 @@ class TestCMORConfig(TestCase): def test_hourly_vars(self): config = CMORConfig(self.mock_parser, self.var_manager) - self.assertEquals(config.get_variables(Frequencies.six_hourly), {}) + self.assertEqual(config.get_variables(Frequencies.six_hourly), {}) self.mock_parser.add_value('CMOR', 'ATMOS_HOURLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) - self.assertEquals(config.get_variables(Frequencies.six_hourly), {128: None, + self.assertEqual(config.get_variables(Frequencies.six_hourly), {128: None, 129: '1', 130: '1,2', 131: '1,2,3,4,5,6,7,8,9', 132: '0,5'}) - self.assertEquals(config.get_levels(Frequencies.six_hourly, 128), None) - self.assertEquals(config.get_levels(Frequencies.six_hourly, 129), '1') - self.assertEquals(config.get_levels(Frequencies.six_hourly, 130), '1,2') - self.assertEquals(config.get_levels(Frequencies.six_hourly, 131), '1,2,3,4,5,6,7,8,9',) - self.assertEquals(config.get_levels(Frequencies.six_hourly, 132), '0,5') + self.assertEqual(config.get_levels(Frequencies.six_hourly, 128), None) + self.assertEqual(config.get_levels(Frequencies.six_hourly, 129), '1') + self.assertEqual(config.get_levels(Frequencies.six_hourly, 130), '1,2') + self.assertEqual(config.get_levels(Frequencies.six_hourly, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEqual(config.get_levels(Frequencies.six_hourly, 132), '0,5') def test_daily_vars(self): config = CMORConfig(self.mock_parser, self.var_manager) - self.assertEquals(config.get_variables(Frequencies.daily), {}) + self.assertEqual(config.get_variables(Frequencies.daily), {}) self.mock_parser.add_value('CMOR', 'ATMOS_DAILY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) - self.assertEquals(config.get_variables(Frequencies.daily), {128: None, + self.assertEqual(config.get_variables(Frequencies.daily), {128: None, 129: '1', 130: '1,2', 131: '1,2,3,4,5,6,7,8,9', 132: '0,5'}) - self.assertEquals(config.get_levels(Frequencies.daily, 128), None) - self.assertEquals(config.get_levels(Frequencies.daily, 129), '1') - self.assertEquals(config.get_levels(Frequencies.daily, 130), '1,2') - self.assertEquals(config.get_levels(Frequencies.daily, 131), '1,2,3,4,5,6,7,8,9',) - self.assertEquals(config.get_levels(Frequencies.daily, 132), '0,5') + self.assertEqual(config.get_levels(Frequencies.daily, 128), None) + self.assertEqual(config.get_levels(Frequencies.daily, 129), '1') + self.assertEqual(config.get_levels(Frequencies.daily, 130), '1,2') + self.assertEqual(config.get_levels(Frequencies.daily, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEqual(config.get_levels(Frequencies.daily, 132), '0,5') def test_monthly_vars(self): config = CMORConfig(self.mock_parser, self.var_manager) - self.assertEquals(config.get_variables(Frequencies.monthly), {}) + self.assertEqual(config.get_variables(Frequencies.monthly), {}) self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) - self.assertEquals(config.get_variables(Frequencies.monthly), {128: None, + self.assertEqual(config.get_variables(Frequencies.monthly), {128: None, 129: '1', 130: '1,2', 131: '1,2,3,4,5,6,7,8,9', 132: '0,5'}) - self.assertEquals(config.get_levels(Frequencies.monthly, 128), None) - self.assertEquals(config.get_levels(Frequencies.monthly, 129), '1') - self.assertEquals(config.get_levels(Frequencies.monthly, 130), '1,2') - self.assertEquals(config.get_levels(Frequencies.monthly, 131), '1,2,3,4,5,6,7,8,9',) - self.assertEquals(config.get_levels(Frequencies.monthly, 132), '0,5') + self.assertEqual(config.get_levels(Frequencies.monthly, 128), None) + self.assertEqual(config.get_levels(Frequencies.monthly, 129), '1') + self.assertEqual(config.get_levels(Frequencies.monthly, 130), '1,2') + self.assertEqual(config.get_levels(Frequencies.monthly, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEqual(config.get_levels(Frequencies.monthly, 132), '0,5') def test_bad_frequency_vars(self): config = CMORConfig(self.mock_parser, self.var_manager) @@ -261,7 +261,7 @@ class TestCMORConfig(TestCase): self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) - self.assertEquals(config.get_requested_codes(), {128, 129, 130, 131, 132}) + self.assertEqual(config.get_requested_codes(), {128, 129, 130, 131, 132}) class TestTHREDDSConfig(TestCase): @@ -271,12 +271,12 @@ class TestTHREDDSConfig(TestCase): def test_basic_config(self): config = THREDDSConfig(self.mock_parser) - self.assertEquals(config.server_url, '') + self.assertEqual(config.server_url, '') def test_url(self): self.mock_parser.add_value('THREDDS', 'SERVER_URL', 'test_url') config = THREDDSConfig(self.mock_parser) - self.assertEquals(config.server_url, 'test_url') + self.assertEqual(config.server_url, 'test_url') class TestReportConfig(TestCase): @@ -286,18 +286,18 @@ class TestReportConfig(TestCase): def test_basic_config(self): config = ReportConfig(self.mock_parser) - self.assertEquals(config.path, '') - self.assertEquals(config.maximum_priority, 10) + self.assertEqual(config.path, '') + self.assertEqual(config.maximum_priority, 10) def test_path(self): self.mock_parser.add_value('REPORT', 'PATH', 'new_path') config = ReportConfig(self.mock_parser) - self.assertEquals(config.path, 'new_path') + self.assertEqual(config.path, 'new_path') def test_priority(self): self.mock_parser.add_value('REPORT', 'MAXIMUM_PRIORITY', 3) config = ReportConfig(self.mock_parser) - self.assertEquals(config.maximum_priority, 3) + self.assertEqual(config.maximum_priority, 3) class TestExperimentConfig(TestCase): @@ -309,51 +309,51 @@ class TestExperimentConfig(TestCase): config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, []) - self.assertEquals(config.members, []) - self.assertEquals(config.chunk_size, 0) - self.assertEquals(config.num_chunks, 0) + self.assertEqual(config.startdates, []) + self.assertEqual(config.members, []) + self.assertEqual(config.chunk_size, 0) + self.assertEqual(config.num_chunks, 0) - self.assertEquals(config.atmos_grid, '') - self.assertEquals(config.atmos_timestep, 6) - self.assertEquals(config.ocean_timestep, 6) + self.assertEqual(config.atmos_grid, '') + self.assertEqual(config.atmos_timestep, 6) + self.assertEqual(config.ocean_timestep, 6) def test_members(self): self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', 'fc0 1') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.members, [0, 1]) + self.assertEqual(config.members, [0, 1]) self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', 'fc00 fc01') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.members, [0, 1]) + self.assertEqual(config.members, [0, 1]) self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', 'fc1-fc3') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.members, [1, 2, 3]) + self.assertEqual(config.members, [1, 2, 3]) self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', '1-3') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.members, [1, 2, 3]) + self.assertEqual(config.members, [1, 2, 3]) def test_startdates(self): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '20001101 20011101') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, ['20001101', '20011101']) + self.assertEqual(config.startdates, ['20001101', '20011101']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200(0|1)1101') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, ['20001101', '20011101']) + self.assertEqual(config.startdates, ['20001101', '20011101']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200[0-2](02|05|08|11)01') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', + self.assertEqual(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', u'20020801', u'20021101']) @@ -361,27 +361,27 @@ class TestExperimentConfig(TestCase): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,1Y}') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, ['20001101', '20011101']) + self.assertEqual(config.startdates, ['20001101', '20011101']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,6M} ') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, ['20001101', '20010501', '20011101']) + self.assertEqual(config.startdates, ['20001101', '20010501', '20011101']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,1W}') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + self.assertEqual(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,W}') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + self.assertEqual(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7D}') config = ExperimentConfig() config.parse_ini(self.mock_parser) - self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + self.assertEqual(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7F}') with self.assertRaises(ConfigException): diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py index 32dadcd9..943a2b3e 100644 --- a/test/unit/test_constants.py +++ b/test/unit/test_constants.py @@ -10,14 +10,14 @@ class TestBasin(TestCase): self.basin = Basin('Basin') def test_name(self): - self.assertEquals('Basin', self.basin.name) + self.assertEqual('Basin', self.basin.name) def test__eq__(self): self.assertTrue(Basin('Basin') == self.basin) self.assertFalse(Basin('bas') == self.basin) def test__str__(self): - self.assertEquals(str(self.basin), 'Basin') + self.assertEqual(str(self.basin), 'Basin') class TestBasins(TestCase): diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index e3cdcd83..b1ff6c82 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -362,7 +362,7 @@ class TestDiagnostic(TestCase): Diagnostic(None).compute() def test_repr(self): - self.assertEquals(Diagnostic(None).__repr__(), str(Diagnostic(None))) + self.assertEqual(Diagnostic(None).__repr__(), str(Diagnostic(None))) def test_empty_process_options(self): self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) diff --git a/test/unit/test_frequency.py b/test/unit/test_frequency.py index 34e32a36..0aa78f90 100644 --- a/test/unit/test_frequency.py +++ b/test/unit/test_frequency.py @@ -2,7 +2,7 @@ from unittest import TestCase from earthdiagnostics.frequency import Frequency -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class TestFrequency(TestCase): diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index eec9d0ce..4f252a0d 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -8,9 +8,9 @@ from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms class TestModellingRealms(TestCase): def test_parse(self): - self.assertEquals(ModelingRealms.parse('atmos'), ModelingRealms.atmos) - self.assertEquals(ModelingRealms.parse('atmoschem'), ModelingRealms.atmosChem) - self.assertEquals(ModelingRealms.parse('atmoSChem'), ModelingRealms.atmosChem) + self.assertEqual(ModelingRealms.parse('atmos'), ModelingRealms.atmos) + self.assertEqual(ModelingRealms.parse('atmoschem'), ModelingRealms.atmosChem) + self.assertEqual(ModelingRealms.parse('atmoSChem'), ModelingRealms.atmosChem) with self.assertRaises(ValueError): ModelingRealms.parse('badrealm') diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 70664ab4..2162cab9 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -11,17 +11,17 @@ class TestTempFile(TestCase): TempFile.prefix = 'prefix' def test_get(self): - self.assertEquals(TempFile.get('tempfile', clean=False), '/tmp/tempfile') - self.assertEquals(TempFile.get('tempfile2', clean=True), '/tmp/tempfile2') + self.assertEqual(TempFile.get('tempfile', clean=False), '/tmp/tempfile') + self.assertEqual(TempFile.get('tempfile2', clean=True), '/tmp/tempfile2') self.assertNotIn('/tmp/tempfile', TempFile.files) self.assertIn('/tmp/tempfile2', TempFile.files) TempFile.autoclean = True - self.assertEquals(TempFile.get('tempfile3'), '/tmp/tempfile3') + self.assertEqual(TempFile.get('tempfile3'), '/tmp/tempfile3') self.assertIn('/tmp/tempfile3', TempFile.files) TempFile.autoclean = False - self.assertEquals(TempFile.get('tempfile4'), '/tmp/tempfile4') + self.assertEqual(TempFile.get('tempfile4'), '/tmp/tempfile4') self.assertNotIn('/tmp/tempfile4', TempFile.files) with mock.patch('tempfile.mkstemp') as mkstemp_mock: @@ -46,7 +46,7 @@ class TestTempFile(TestCase): TempFile.get('tempfile') TempFile.get('tempfile2') TempFile.clean() - self.assertEquals(len(TempFile.files), 0) + self.assertEqual(len(TempFile.files), 0) class TestUtils(TestCase): diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index c53baa63..fddc8f2a 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -14,18 +14,18 @@ class TestCMORTable(TestCase): self.frequency = Mock() def test_str(self): - self.assertEquals(str(CMORTable('name', 'm', 'Month YEAR')), 'name') + self.assertEqual(str(CMORTable('name', 'm', 'Month YEAR')), 'name') class TestVariableAlias(TestCase): def test_str(self): alias = VariableAlias('alias') - self.assertEquals(str(alias), 'alias') + self.assertEqual(str(alias), 'alias') alias.basin = 'basin' - self.assertEquals(str(alias), 'alias Basin: basin') + self.assertEqual(str(alias), 'alias Basin: basin') alias.grid = 'grid' - self.assertEquals(str(alias), 'alias Basin: basin Grid: grid') + self.assertEqual(str(alias), 'alias Basin: basin Grid: grid') class TestVariable(TestCase): diff --git a/test/unit/test_variable_type.py b/test/unit/test_variable_type.py index 28dd44f5..5b98e6ae 100644 --- a/test/unit/test_variable_type.py +++ b/test/unit/test_variable_type.py @@ -1,7 +1,7 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.variable import VariableType class TestVariableType(TestCase): -- GitLab From 7435dfed131388c2e6ee196383afe8e0d153ea7a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 15:13:55 +0100 Subject: [PATCH 133/168] Fixed some codacy warnings --- earthdiagnostics/datamanager.py | 2 +- earthdiagnostics/earthdiags.py | 60 +++++++++++----------- earthdiagnostics/general/timemean.py | 3 +- earthdiagnostics/ocean/areamoc.py | 1 - earthdiagnostics/ocean/heatcontentlayer.py | 22 ++++---- earthdiagnostics/ocean/maxmoc.py | 2 +- earthdiagnostics/ocean/rotation.py | 1 - earthdiagnostics/work_manager.py | 33 ++++++++---- 8 files changed, 67 insertions(+), 57 deletions(-) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index a07b04f0..e099ade2 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -310,7 +310,7 @@ class UnitConversion(object): @classmethod def add_conversion(cls, conversion): """ - Adds a conversion to the dictionary + Add a conversion to the dictionary :param conversion: conversion to add :type conversion: UnitConversion diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 515a33a4..d4b91083 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -332,35 +332,9 @@ class EarthDiags(object): mask_regions_3d_path = os.path.join(con_files, mask_regions_3d) if self.config.scratch_masks: - Utils.create_folder_tree(self.config.scratch_masks) - Utils.give_group_write_permissions(self.config.scratch_masks) - - mesh_mask_scratch_path = os.path.join(self.config.scratch_masks, mesh_mask) - - if self._copy_file(mesh_mask_path, mesh_mask_scratch_path, - restore_meshes): - Utils.give_group_write_permissions(mesh_mask_scratch_path) - self._link_file(mesh_mask_scratch_path, 'mesh_hgr.nc') - self._link_file(mesh_mask_scratch_path, 'mesh_zgr.nc') - self._link_file(mesh_mask_scratch_path, 'mask.nc') - - new_maskglo_scratch_path = os.path.join(self.config.scratch_masks, new_mask_glo) - if self._copy_file(new_mask_glo_path, - new_maskglo_scratch_path, restore_meshes): - Utils.give_group_write_permissions(new_maskglo_scratch_path) - self._link_file(new_maskglo_scratch_path, 'new_maskglo.nc') - - mask_regions_scratch_path = os.path.join(self.config.scratch_masks, mask_regions) - if self._copy_file(mask_regions_path, - mask_regions_scratch_path, restore_meshes): - Utils.give_group_write_permissions(mask_regions_scratch_path) - self._link_file(mask_regions_scratch_path, 'mask_regions.nc') - - mask_regions3d_scratch_path = os.path.join(self.config.scratch_masks, mask_regions_3d) - if self._copy_file(mask_regions_3d_path, - mask_regions3d_scratch_path, restore_meshes): - Utils.give_group_write_permissions(mask_regions3d_scratch_path) - self._link_file(mask_regions3d_scratch_path, 'mask_regions.3d.nc') + self._prepare_mesh_using_scratch(mask_regions, mask_regions_3d, mask_regions_3d_path, mask_regions_path, + mesh_mask, mesh_mask_path, new_mask_glo, new_mask_glo_path, + restore_meshes) else: self._copy_file(mesh_mask_path, 'mesh_hgr.nc', restore_meshes) self._link_file('mesh_hgr.nc', 'mesh_zgr.nc') @@ -374,6 +348,34 @@ class EarthDiags(object): Log.result('Mesh files ready!') + def _prepare_mesh_using_scratch(self, mask_regions, mask_regions_3d, mask_regions_3d_path, mask_regions_path, + mesh_mask, mesh_mask_path, new_mask_glo, new_mask_glo_path, + restore_meshes): + Utils.create_folder_tree(self.config.scratch_masks) + Utils.give_group_write_permissions(self.config.scratch_masks) + mesh_mask_scratch_path = os.path.join(self.config.scratch_masks, mesh_mask) + if self._copy_file(mesh_mask_path, mesh_mask_scratch_path, + restore_meshes): + Utils.give_group_write_permissions(mesh_mask_scratch_path) + self._link_file(mesh_mask_scratch_path, 'mesh_hgr.nc') + self._link_file(mesh_mask_scratch_path, 'mesh_zgr.nc') + self._link_file(mesh_mask_scratch_path, 'mask.nc') + new_maskglo_scratch_path = os.path.join(self.config.scratch_masks, new_mask_glo) + if self._copy_file(new_mask_glo_path, + new_maskglo_scratch_path, restore_meshes): + Utils.give_group_write_permissions(new_maskglo_scratch_path) + self._link_file(new_maskglo_scratch_path, 'new_maskglo.nc') + mask_regions_scratch_path = os.path.join(self.config.scratch_masks, mask_regions) + if self._copy_file(mask_regions_path, + mask_regions_scratch_path, restore_meshes): + Utils.give_group_write_permissions(mask_regions_scratch_path) + self._link_file(mask_regions_scratch_path, 'mask_regions.nc') + mask_regions3d_scratch_path = os.path.join(self.config.scratch_masks, mask_regions_3d) + if self._copy_file(mask_regions_3d_path, + mask_regions3d_scratch_path, restore_meshes): + Utils.give_group_write_permissions(mask_regions3d_scratch_path) + self._link_file(mask_regions3d_scratch_path, 'mask_regions.3d.nc') + def _copy_file(self, source, destiny, force): if not os.path.exists(source): Log.user_warning('File {0} is not available for {1}', destiny, self.config.experiment.model_version) diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index c09b4280..67b1a86d 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -220,7 +220,6 @@ class YearlyMean(TimeMean): :param grid: original data grid :type grid: str """ - alias = 'yearmean' "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): @@ -229,7 +228,7 @@ class YearlyMean(TimeMean): def compute_mean(self, input_file, output_file): """ - Computes the time mean + Compute the time mean Parameters ---------- diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 4831a1b9..0f20d28d 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -75,7 +75,6 @@ class AreaMoc(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticIntOption('min_lat'), DiagnosticIntOption('max_lat'), DiagnosticIntOption('min_depth'), diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 998a371f..88a74a7c 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -72,36 +72,41 @@ class HeatContentLayer(Diagnostic): box.max_depth = options['max_depth'] job_list = list() + max_level, min_level, weight = cls._compute_weights(box) + + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(HeatContentLayer(diags.data_manager, startdate, member, chunk, box, + weight, min_level, max_level)) + return job_list + + @classmethod + def _compute_weights(cls, box): handler = Utils.open_cdf('mesh_zgr.nc') # mask = Utils.get_mask(options['basin']) mask = handler.variables['tmask'][:] - if 'e3t' in handler.variables: e3t = handler.variables['e3t'][:] elif 'e3t_0' in handler.variables: e3t = handler.variables['e3t_0'][:] else: raise Exception('e3t variable can not be found') - if 'gdepw' in handler.variables: depth = handler.variables['gdepw'][:] elif 'gdepw_0' in handler.variables: depth = handler.variables['gdepw_0'][:] else: raise Exception('gdepw variable can not be found') - e3t_3d = e3t.shape != depth.shape if e3t_3d: mask = e3t_3d * mask else: e3t = e3t[0, :] - while len(depth.shape) < 4: depth = np.expand_dims(depth, -1) handler.close() def calculate_weight(e3t_point, depth_point, mask_point): - """Calculates the weight for each cell""" + """Calculate the weight for each cell""" if not mask_point: return 0 top = depth_point @@ -118,7 +123,6 @@ class HeatContentLayer(Diagnostic): calc = np.vectorize(calculate_weight, otypes='f') weight = calc(e3t, depth, mask) - # Now we will reduce to the levels with any weight != 0 to avoid loading too much data on memory levels = weight.shape[1] min_level = 0 @@ -128,11 +132,7 @@ class HeatContentLayer(Diagnostic): while max_level < (levels - 1) and weight[:, max_level + 1, :].any(): max_level += 1 weight = weight[:, min_level:max_level, :] - - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(HeatContentLayer(diags.data_manager, startdate, member, chunk, box, - weight, min_level, max_level)) - return job_list + return max_level, min_level, weight def request_data(self): """Request data required by the diagnostic""" diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index 29468881..eba04a78 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -62,7 +62,7 @@ class MaxMoc(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Creates a job for each complete year to compute the diagnostic + Create a job for each complete year to compute the diagnostic :param diags: Diagnostics manager class :type diags: Diags diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index fe3aa73c..5caa1931 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -93,7 +93,6 @@ class Rotation(Diagnostic): def compute(self): """Run the diagnostic""" - handler = Utils.open_cdf(self.ufile.local_file) if 'lev' in handler.dimensions: self.num_levels = handler.dimensions['lev'].size diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 6326d004..588c66be 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -148,28 +148,39 @@ class WorkManager(object): def _check_completion(self): self._pause_downloader_if_required() + if self._jobs_running_or_ready(): + return False + if self._data_downloading_or_uploading(): + return False + + try: + self.lock.release() + except threading.ThreadError: + pass + return True + + def _jobs_running_or_ready(self): for job in self.jobs: if job.status in (DiagnosticStatus.READY, DiagnosticStatus.RUNNING): - return False + return True if job.status == DiagnosticStatus.WAITING: if job.all_requests_in_storage(): - return False + return True + return False + def _data_downloading_or_uploading(self): for request in self.data_manager.requested_files.values(): if request.storage_status == StorageStatus.UPLOADING: - return False + return True if request.local_status == LocalStatus.DOWNLOADING: - return False + return True if request.upload_required(): - return False + return True if request.download_required(): - return False - try: - self.lock.release() - except threading.ThreadError: - pass - return True + return True + return False + def _pause_downloader_if_required(self): counter = 0 -- GitLab From 092d29a90a85a21bbd6c3fa4a87fd172e77baadf Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 15:15:57 +0100 Subject: [PATCH 134/168] Fixed some codacy warnings --- earthdiagnostics/frequency.py | 1 - earthdiagnostics/general/scale.py | 2 +- earthdiagnostics/ocean/averagesection.py | 2 +- earthdiagnostics/variable.py | 2 ++ 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py index 2be71c33..5cd15f48 100644 --- a/earthdiagnostics/frequency.py +++ b/earthdiagnostics/frequency.py @@ -2,7 +2,6 @@ """Data frequency management tools""" - class Frequency(object): """Time frequency""" diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index fa540cdb..b7d27de2 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,5 +1,5 @@ # coding=utf-8 -""" Scales a variable by with value and offset""" +"""Scales a variable by with value and offset""" import math import numpy as np diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 5f7f9c99..27eac1aa 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -11,6 +11,7 @@ from earthdiagnostics.utils import Utils, TempFile class AverageSection(Diagnostic): """ Compute an average of a given zone. + The variable MUST be in a regular grid :original author: Virginie Guemas @@ -33,7 +34,6 @@ class AverageSection(Diagnostic): :type domain: ModelingRealm :param box: box to use for the average :type box: Box - """ alias = 'avgsection' diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 7ee81bff..0af78c00 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -536,11 +536,13 @@ class CMORTable(object): class VariableType(object): """Enumeration of variable types""" + MEAN = 1 STATISTIC = 2 @staticmethod def to_str(vartype): + """Get str representation of vartype for the folder convention""" if vartype == VariableType.MEAN: return 'mean' elif vartype == VariableType.STATISTIC: -- GitLab From 9bd206e69ac09896b90e459718c4446b68b4ae28 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 15:19:23 +0100 Subject: [PATCH 135/168] Fixed some codacy warnings --- earthdiagnostics/general/timemean.py | 5 +++-- earthdiagnostics/ocean/cutsection.py | 1 + earthdiagnostics/statistics/climatologicalpercentile.py | 4 ++-- earthdiagnostics/work_manager.py | 1 - 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index 67b1a86d..b437b7f0 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -90,7 +90,7 @@ class TimeMean(Diagnostic): def compute_mean(self, input_file, output_file): """ - Computes the time mean + Compute the time mean Parameters ---------- @@ -147,7 +147,7 @@ class DailyMean(TimeMean): def compute_mean(self, input_file, output_file): """ - Computes the time mean + Compute the time mean Parameters ---------- @@ -220,6 +220,7 @@ class YearlyMean(TimeMean): :param grid: original data grid :type grid: str """ + alias = 'yearmean' "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index bf4e6d39..2690e6b5 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -1,4 +1,5 @@ # coding=utf-8 +"""Cut meridional or zonal sections""" import numpy as np from bscearth.utils.log import Log diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index d31d0ea3..d97e24c9 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -48,8 +48,8 @@ class ClimatologicalPercentile(Diagnostic): def __eq__(self, other): return self.domain == other.domain and self.variable == other.variable and \ - self.start_year == other.start_year and self.end_year == other.end_year and \ - self.forecast_month == other.forecast_month + self.start_year == other.start_year and self.end_year == other.end_year and \ + self.forecast_month == other.forecast_month def __str__(self): return 'Climatological percentile Variable: {0.domain}:{0.variable} Period: {0.start_year}-{0.end_year} ' \ diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 588c66be..5fb8eb35 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -181,7 +181,6 @@ class WorkManager(object): return True return False - def _pause_downloader_if_required(self): counter = 0 for job in self.jobs: -- GitLab From 5f4b7fb451f31035283ec430b5075430d8a4b977 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 15:22:18 +0100 Subject: [PATCH 136/168] Fixed some codacy warnings --- earthdiagnostics/general/__init__.py | 1 + earthdiagnostics/general/module.py | 16 ++++++++-------- earthdiagnostics/general/relink.py | 14 +++++++------- earthdiagnostics/general/relinkall.py | 16 ++++++++-------- earthdiagnostics/general/rewrite.py | 14 +++++++------- earthdiagnostics/ocean/__init__.py | 2 +- earthdiagnostics/ocean/interpolate.py | 16 ++++++++-------- earthdiagnostics/statistics/__init__.py | 1 + 8 files changed, 41 insertions(+), 39 deletions(-) diff --git a/earthdiagnostics/general/__init__.py b/earthdiagnostics/general/__init__.py index 9bad5790..1b9cbfa8 100644 --- a/earthdiagnostics/general/__init__.py +++ b/earthdiagnostics/general/__init__.py @@ -1 +1,2 @@ # coding=utf-8 +"""Package containing general purpose diagnostics""" diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index 0b07132b..b8b31146 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -56,14 +56,14 @@ class Module(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, grid - :type options: list[str] - :return: - """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ options_available = (DiagnosticDomainOption(), DiagnosticVariableOption(diags.data_manager.config.var_manager, 'componentu'), DiagnosticVariableOption(diags.data_manager.config.var_manager, 'componentv'), diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 6479ee5b..1a9162d9 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -54,14 +54,14 @@ class Relink(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Create a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, move_old=False - :type options: list[str] - :return: - """ + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, move_old=False + :type options: list[str] + :return: + """ options_available = (DiagnosticDomainOption(), DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticBoolOption('move_old', True), diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 1f1dd5d3..17d05df8 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -33,14 +33,14 @@ class RelinkAll(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, move_old=False - :type options: list[str] - :return: - """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, move_old=False + :type options: list[str] + :return: + """ if len(options) > 1: raise Exception('The Relink All diagnostic has no options') job_list = list() diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index 10bd7233..db19ee02 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -51,14 +51,14 @@ class Rewrite(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Create a job for each chunk to compute the diagnostic + Create a job for each chunk to compute the diagnostic - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, grid - :type options: list[str] - :return: - """ + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ options_available = (DiagnosticDomainOption(), DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticOption('grid', '')) diff --git a/earthdiagnostics/ocean/__init__.py b/earthdiagnostics/ocean/__init__.py index d21438cd..4196e870 100644 --- a/earthdiagnostics/ocean/__init__.py +++ b/earthdiagnostics/ocean/__init__.py @@ -1,2 +1,2 @@ # coding=utf-8 - +"""Package containing diagnostics related to the ocean model""" diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 4bc1510b..acb114e6 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -77,14 +77,14 @@ class Interpolate(Diagnostic): @classmethod def generate_jobs(cls, diags, options): """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: target_grid, variable, domain=ocean - :type options: list[str] - :return: - """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: target_grid, variable, domain=ocean + :type options: list[str] + :return: + """ options_available = (DiagnosticOption('target_grid'), DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variable'), DiagnosticDomainOption(default_value=ModelingRealms.ocean), diff --git a/earthdiagnostics/statistics/__init__.py b/earthdiagnostics/statistics/__init__.py index 9bad5790..bc73837e 100644 --- a/earthdiagnostics/statistics/__init__.py +++ b/earthdiagnostics/statistics/__init__.py @@ -1 +1,2 @@ # coding=utf-8 +"""Package containing statistical diagnostics""" -- GitLab From 1e3afea09550225c7cf6bf5a8e98f3430d379909 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 15:31:10 +0100 Subject: [PATCH 137/168] Fixed some codacy warnings --- earthdiagnostics/general/timemean.py | 3 +-- earthdiagnostics/threddsmanager.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index b437b7f0..0f0b5322 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -70,7 +70,6 @@ class TimeMean(Diagnostic): :type options: list[str] :return: """ - options = cls._process_options(diags, options) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): @@ -188,7 +187,7 @@ class MonthlyMean(TimeMean): def compute_mean(self, input_file, output_file): """ - Computes the time mean + Compute the time mean Parameters ---------- diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index a0909af1..6650e1f9 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -46,7 +46,7 @@ class THREDDSManager(DataManager): # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VariableType.MEAN): + vartype=VariableType.MEAN, possible_versions=None): """" Check if a file exists in the storage @@ -65,7 +65,6 @@ class THREDDSManager(DataManager): Returns ------- bool - """ aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) -- GitLab From 60b5f7d4fdb0f099a2b96f678928fa1d47177dc3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 15:46:50 +0100 Subject: [PATCH 138/168] Fixed some codacy warnings --- earthdiagnostics/threddsmanager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 6650e1f9..b6606524 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -286,6 +286,7 @@ class THREDDSSubset(DataFile): super(THREDDSSubset, self).__init__() self.thredds_path = thredds_path self.remote_file = file_path + self.local_file = None if '_f' in var: self.var = var[:var.index('_f')] self.hourly = var[var.index('_f'):] @@ -329,8 +330,9 @@ class THREDDSSubset(DataFile): Log.error('Can not retrieve {0} from server: {1}'.format(self, ex)) self.local_status = LocalStatus.FAILED - # noinspection PyUnusedLocal,PyMethodMayBeStatic - def _correct_cube(self, cube, field, filename): + # noinspection PyUnusedLocal + @staticmethod + def _correct_cube(cube, field, filename): if not cube.coords('time'): return time = cube.coord('time') -- GitLab From 578ae769c2ed304282c98186e2c83d264b9a836b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 Dec 2017 17:05:20 +0100 Subject: [PATCH 139/168] Added tests for rotation --- earthdiagnostics/cmormanager.py | 4 +-- earthdiagnostics/datafile.py | 2 +- earthdiagnostics/ocean/rotation.py | 7 +++-- test/unit/ocean/test_rotation.py | 46 ++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 test/unit/ocean/test_rotation.py diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index d4268089..c39234ea 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -9,8 +9,8 @@ from datetime import datetime from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, previous_day from bscearth.utils.log import Log -from datafile import StorageStatus -from diagnostic import Diagnostic +from earthdiagnostics.datafile import StorageStatus +from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.cmorizer import Cmorizer from earthdiagnostics.datamanager import DataManager from earthdiagnostics.frequency import Frequencies, Frequency diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index f70413c3..6b82b3b6 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -11,7 +11,7 @@ from bscearth.utils.log import Log from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile -from publisher import Publisher +from earthdiagnostics.publisher import Publisher from earthdiagnostics.variable import VariableType diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 5caa1931..12654b3b 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -47,7 +47,8 @@ class Rotation(Diagnostic): def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variableu == other.variableu and self.variablev == other.variablev + self.domain == other.domain and self.variableu == other.variableu and \ + self.variablev == other.variablev and self.executable == other.executable def __str__(self): return 'Rotate variables Startdate: {0} Member: {1} Chunk: {2} Variables: {3}:{4} , ' \ @@ -65,9 +66,9 @@ class Rotation(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager, 'variableu'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'variableu'), DiagnosticVariableOption(diags.data_manager.config.var_manager, 'variablev'), - DiagnosticDomainOption(default_value=ModelingRealms.ocean), DiagnosticOption('executable', '/home/Earth/jvegas/pyCharm/cfutools/interpolation/rotateUVorca')) options = cls.process_options(options, options_available) diff --git a/test/unit/ocean/test_rotation.py b/test/unit/ocean/test_rotation.py new file mode 100644 index 00000000..08c61fb1 --- /dev/null +++ b/test/unit/ocean/test_rotation.py @@ -0,0 +1,46 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.rotation import Rotation +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption +from mock import Mock, patch + + +class TestMixedLayerHeatContent(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.mixed = Rotation(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'varu', 'varv', 'exe') + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = Rotation.generate_jobs(self.diags, ['diagnostic', 'ocean', 'varu', 'varv']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Rotation(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'varu', 'varv', + '/home/Earth/jvegas/pyCharm/cfutools/interpolation/rotateUVorca')) + self.assertEqual(jobs[1], Rotation(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'varu', 'varv', + '/home/Earth/jvegas/pyCharm/cfutools/interpolation/rotateUVorca')) + + jobs = Rotation.generate_jobs(self.diags, ['diagnostic', 'ocean', 'varu', 'varv', 'exe']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], + Rotation(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'varu', 'varv', 'exe')) + self.assertEqual(jobs[1], + Rotation(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'varu', 'varv', 'exe')) + + with self.assertRaises(Exception): + Rotation.generate_jobs(self.diags, ['diagnostic', 'ocean', 'varu', 'varv', 'exe', 'extra']) + + def test_str(self): + self.assertEqual(str(self.mixed), 'Rotate variables Startdate: 20000101 Member: 1 Chunk: 1 Variables: ocean:varu , ocean:varv') -- GitLab From ef5aa600ee647b0e23fdce110771aef1b78fa9c5 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 11:39:56 +0100 Subject: [PATCH 140/168] Added some test --- test/unit/test_cdftools.py | 9 +++-- test/unit/test_cmormanager.py | 9 +---- test/unit/test_data_manager.py | 65 ++++++++++++++++++++++++++++++- test/unit/test_frequency.py | 3 ++ test/unit/test_modelling_realm.py | 3 ++ 5 files changed, 77 insertions(+), 12 deletions(-) diff --git a/test/unit/test_cdftools.py b/test/unit/test_cdftools.py index bc7ed38e..26150c2f 100644 --- a/test/unit/test_cdftools.py +++ b/test/unit/test_cdftools.py @@ -13,13 +13,12 @@ def bad_file(path, access=None): class TestCDFTools(TestCase): - # noinspection PyUnusedLocal @mock.patch('os.path.isfile', side_effect=bad_file) @mock.patch('os.access', side_effect=bad_file) @mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') def test_run(self, mock_path, mock_exists, execute_mock): - self.cdftools = CDFTools('/test/path') + self.cdftools = CDFTools('') execute_mock.return_value = ['Command output'] with self.assertRaises(ValueError): self.cdftools.run('badcommand', input_file='input_file', output_file='output_file') @@ -33,6 +32,7 @@ class TestCDFTools(TestCase): self.cdftools.run('command', input_file='input_file', output_file='badoutput_file') self.cdftools.run('command', input_file='input_file', output_file='output_file') + self.cdftools.run('command', input_option='-i', input_file='input_file', output_file='output_file') self.cdftools.run('command', input_file='input_file') self.cdftools.run('command', input_file=None) self.cdftools.run('command', input_file=['input_file', 'input_file2']) @@ -43,8 +43,8 @@ class TestCDFTools(TestCase): @mock.patch('os.path.isfile', side_effect=bad_file) @mock.patch('os.access', side_effect=bad_file) @mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') - def test_run(self, mock_path, mock_exists, execute_mock): - self.cdftools = CDFTools('') + def test_run_with_path(self, mock_path, mock_exists, execute_mock): + self.cdftools = CDFTools('/some/path') execute_mock.return_value = ['Command output'] with self.assertRaises(ValueError): self.cdftools.run('badcommand', input_file='input_file', output_file='output_file') @@ -58,6 +58,7 @@ class TestCDFTools(TestCase): self.cdftools.run('command', input_file='input_file', output_file='badoutput_file') self.cdftools.run('command', input_file='input_file', output_file='output_file') + self.cdftools.run('command', input_option='-i', input_file='input_file', output_file='output_file') self.cdftools.run('command', input_file='input_file') self.cdftools.run('command', input_file=None) self.cdftools.run('command', input_file=['input_file', 'input_file2']) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index a631591f..90fb7c59 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -22,11 +22,11 @@ class TestCMORManager(TestCase): self.config.experiment.experiment_name = 'expname' self.config.experiment.institute = 'institute' self.config.experiment.member_count_start = 0 - self.config.experiment.atmos_timestep = 6 - self.config.experiment.ocean_timestep = 6 self.config.experiment.chunk_size = 12 self.config.experiment.num_chunks = 1 self.config.experiment.calendar = 'standard' + self.config.experiment.atmos_timestep = 3 + self.config.experiment.ocean_timestep = 6 self.config.cmor.initialization_number = 1 self.config.cmor.version = '' @@ -71,11 +71,6 @@ class TestCMORManager(TestCase): cmor_manager = CMORManager(self.config) self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'exp', 'model', 'expid')) - def test_get_varfolder(self): - cmor_manager = CMORManager(self.config) - self.assertEqual(cmor_manager.get_varfolder(ModelingRealms.ocean, 'var'), - 'var_f6h') - def test_get_file_path_bad_convention(self): self.config.cmor.version = 'version' self.config.data_convention = 'bad_convention' diff --git a/test/unit/test_data_manager.py b/test/unit/test_data_manager.py index 6bdc84b3..8277719e 100644 --- a/test/unit/test_data_manager.py +++ b/test/unit/test_data_manager.py @@ -2,8 +2,10 @@ from unittest import TestCase +from mock import Mock -from earthdiagnostics.datamanager import UnitConversion +from earthdiagnostics.datamanager import UnitConversion, DataManager +from earthdiagnostics.modelingrealm import ModelingRealms class TestConversion(TestCase): @@ -36,3 +38,64 @@ class TestConversion(TestCase): self.assertEqual(UnitConversion.get_conversion_factor_offset('m²', 'km'), (None, None)) UnitConversion._dict_conversions = dict() + + +class TestDataManager(TestCase): + + def setUp(self): + self.config = Mock() + self.config.experiment.atmos_timestep = 3 + self.config.experiment.ocean_timestep = 6 + + self.data_manager = DataManager(self.config) + + def test_prepare(self): + self.data_manager.prepare() + + def test_create_link(self): + self.data_manager.create_link(None, '', None, '', '', False, Mock()) + + def test_link_file(self): + self.data_manager.link_file(None, '', None, '', 0, 0) + + def test_file_exists(self): + with self.assertRaises(NotImplementedError): + self.data_manager.file_exists(None, '', '', 0, 0) + + def test_request_chunk(self): + with self.assertRaises(NotImplementedError): + self.data_manager.request_chunk(None, '', '', 0, 0) + + def test_request_year(self): + with self.assertRaises(NotImplementedError): + self.data_manager.request_year(None, '', '', '', 0, 0) + + def test_declare_chunk(self): + with self.assertRaises(NotImplementedError): + self.data_manager.declare_chunk(None, '', '', 0, 0) + + def test_declare_year(self): + with self.assertRaises(NotImplementedError): + self.data_manager.declare_year(None, '', '', 0, 0) + + def test_get_varfolder(self): + self.assertEqual(self.data_manager.get_varfolder(ModelingRealms.ocean, 'var'), + 'var_f6h') + self.assertEqual(self.data_manager.get_varfolder(ModelingRealms.atmos, 'var'), + 'var_f3h') + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = '3hr' + frequency.frequency = '3hr' + self.assertEqual(self.data_manager.get_varfolder(ModelingRealms.atmos, 'var', frequency=frequency), + 'var') + + def test_get_final_varname(self): + self.assertEqual(self.data_manager._get_final_var_name(None, 'var'), 'var') + box = Mock() + box.get_lon_str.return_value = '_lon' + box.get_lat_str.return_value = '_lat' + box.get_depth_str.return_value = '_depth' + self.assertEqual(self.data_manager._get_final_var_name(box, 'var'), 'var_lon_lat_depth') + + diff --git a/test/unit/test_frequency.py b/test/unit/test_frequency.py index 0aa78f90..d556e36f 100644 --- a/test/unit/test_frequency.py +++ b/test/unit/test_frequency.py @@ -20,6 +20,9 @@ class TestFrequency(TestCase): def test_get_daily_mean(self): self.assertEqual(Frequency('d').folder_name(VariableType.MEAN), 'daily_mean') + def test_get_weekly_mean(self): + self.assertEqual(Frequency('w').folder_name(VariableType.MEAN), 'weekly_mean') + def test_get_daily_stats(self): self.assertEqual(Frequency('d').folder_name(VariableType.STATISTIC), 'daily_statistics') diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 4f252a0d..3752346f 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -20,6 +20,9 @@ class TestModellingRealm(TestCase): def setUp(self): self.realm = ModelingRealm('ocean') + def test__repr__(self): + self.assertEqual(str(self.realm), repr(self.realm)) + def test_constructor_fail_on_bad_realm(self): with self.assertRaises(ValueError): ModelingRealm('badrealm') -- GitLab From 33fcf8f6b242c131da1172c732f608f7b2d83d24 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 11:53:49 +0100 Subject: [PATCH 141/168] Finished config tests --- earthdiagnostics/config.py | 3 +-- test/unit/test_config.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 5f358500..68ca08b2 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -454,8 +454,7 @@ class ExperimentConfig(object): else: for startdate in exrex.generate(startdate_pattern): startdate = startdate.strip() - if startdate: - self.startdates.append(startdate) + self.startdates.append(startdate) def _parse_members(self): members = [] diff --git a/test/unit/test_config.py b/test/unit/test_config.py index dbb21830..6b38a194 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -404,10 +404,11 @@ class TestExperimentConfig(TestCase): def test_get_year_chunks(self): self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 3) - self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 15) + self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 13) config = ExperimentConfig() config.parse_ini(self.mock_parser) + self.assertEqual(config.get_year_chunks('20000601', 2003), [11, 12, 13]) self.assertEqual(config.get_year_chunks('20000601', 2001), [3, 4, 5, 6, 7]) self.assertEqual(config.get_year_chunks('20000101', 2000), [1, 2, 3, 4]) -- GitLab From f692b6cbaca0d4fb67d96ea2856bbc979a88999d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 12:38:53 +0100 Subject: [PATCH 142/168] Added tests to workmanager --- test/unit/test_publisher.py | 11 +++++++ test/unit/test_workmanager.py | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 test/unit/test_workmanager.py diff --git a/test/unit/test_publisher.py b/test/unit/test_publisher.py index 5fe325ff..a9124832 100644 --- a/test/unit/test_publisher.py +++ b/test/unit/test_publisher.py @@ -35,3 +35,14 @@ class TestPublisher(TestCase): pub.dispatch(1, 2, 3) suscriber.callback.assert_called_with(1, 2, 3) + def test_only_suscriber(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber, callback=suscriber.callback) + self.assertTrue(pub.only_suscriber(suscriber)) + + suscriber2 = Mock() + pub.subscribe(suscriber2, callback=suscriber.callback) + self.assertFalse(pub.only_suscriber(suscriber)) + self.assertFalse(pub.only_suscriber(suscriber2)) + diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py new file mode 100644 index 00000000..ad5c88b6 --- /dev/null +++ b/test/unit/test_workmanager.py @@ -0,0 +1,54 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.work_manager import Downloader, WorkManager +from earthdiagnostics.diagnostic import Diagnostic +from mock import Mock + + +class TestDownloader(TestCase): + + def setUp(self): + self.downloader = Downloader() + + def test_start_and_stop(self): + self.downloader.start() + self.assertTrue(self.downloader._thread.is_alive()) + self.downloader.shutdown() + self.assertFalse(self.downloader._thread.is_alive()) + + def test_submit(self): + datafile = Mock() + self.downloader.submit(datafile) + + def test_download(self): + datafile = Mock() + self.downloader.submit(datafile) + self.downloader.start() + self.downloader.shutdown() + + datafile.download.assert_called_once() + + +class TestWorkManager(TestCase): + + + def setUp(self): + self.config = Mock() + self.data_manager = Mock() + self.work_manager = WorkManager(self.config, self.data_manager) + + def test_prepare_job_list(self): + + class Diag1(Diagnostic): + alias = 'diag1' + + @classmethod + def generate_jobs(cls, diags, options): + cls.process_options(options, []) + diag = Diag1(self.data_manager) + diag.add_subjob(Diag1(self.data_manager)) + return (diag,) + + self.config.get_commands.return_value = ['diag1', 'baddiag', 'diag1,badoption'] + Diagnostic.register(Diag1) + self.work_manager.prepare_job_list() -- GitLab From 048a141db9644a73b1ae1e83df9b305f1419c68e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 12:46:02 +0100 Subject: [PATCH 143/168] Removed duplicated UnitConversion class --- earthdiagnostics/datafile.py | 55 ++++++------- earthdiagnostics/datamanager.py | 108 +------------------------- test/unit/statistics/test_datafile.py | 34 ++++++++ test/unit/test_data_manager.py | 34 +------- 4 files changed, 65 insertions(+), 166 deletions(-) create mode 100644 test/unit/statistics/test_datafile.py diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 6b82b3b6..8bd25b4d 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -505,10 +505,26 @@ class DataFile(Publisher): class UnitConversion(object): - """Class to manage unit conversions""" + """ + Class to manage unit conversions + + Parameters + ---------- + source: str + destiny: str + factor: float + offset: float + + """ _dict_conversions = None + def __init__(self, source, destiny, factor, offset): + self.source = source + self.destiny = destiny + self.factor = float(factor) + self.offset = float(offset) + @classmethod def load_conversions(cls): """Load conversions from the configuration file""" @@ -525,19 +541,11 @@ class UnitConversion(object): """ Add a conversion to the dictionary - Parameters - ---------- - conversion: UnitConversion - + :param conversion: conversion to add + :type conversion: UnitConversion """ cls._dict_conversions[(conversion.source, conversion.destiny)] = conversion - def __init__(self, source, destiny, factor, offset): - self.source = source - self.destiny = destiny - self.factor = float(factor) - self.offset = float(offset) - @classmethod def get_conversion_factor_offset(cls, input_units, output_units): """ @@ -546,18 +554,12 @@ class UnitConversion(object): The conversion has to be done in the following way: converted = original * factor + offset - Parameters - ---------- - input_units: str - output_units = str - - Returns - ------- - float - Factor - float - Offset - + :param input_units: original units + :type input_units: str + :param output_units: destiny units + :type output_units: str + :return: factor and offset + :rtype: [float, float] """ units = input_units.split() if len(units) == 1: @@ -596,13 +598,14 @@ class UnitConversion(object): # Add only the conversions with a factor greater than 1 if unit == new_unit: return 1, 0 - if (unit, new_unit) in cls._dict_conversions: + elif (unit, new_unit) in cls._dict_conversions: conversion = cls._dict_conversions[(unit, new_unit)] return conversion.factor, conversion.offset - if (new_unit, unit) in cls._dict_conversions: + elif (new_unit, unit) in cls._dict_conversions: conversion = cls._dict_conversions[(new_unit, unit)] return 1 / conversion.factor, -conversion.offset - return None, None + else: + return None, None class NetCDFFile(DataFile): diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index e099ade2..ebf6e8a8 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -1,10 +1,8 @@ # coding: utf-8 """Base data manager for Earth diagnostics""" -import csv -import os import threading -from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus +from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus, UnitConversion from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.variable import VariableType @@ -273,107 +271,3 @@ class DataManager(object): """ raise NotImplementedError('Class must override file_exists method') - - -class UnitConversion(object): - """ - Class to manage unit conversions - - Parameters - ---------- - source: str - destiny: str - factor: float - offset: float - - """ - - _dict_conversions = None - - def __init__(self, source, destiny, factor, offset): - self.source = source - self.destiny = destiny - self.factor = float(factor) - self.offset = float(offset) - - @classmethod - def load_conversions(cls): - """Load conversions from the configuration file""" - cls._dict_conversions = dict() - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'r') as csvfile: - reader = csv.reader(csvfile, dialect='excel') - for line in reader: - if line[0] == 'original': - continue - cls.add_conversion(UnitConversion(line[0], line[1], line[2], line[3])) - - @classmethod - def add_conversion(cls, conversion): - """ - Add a conversion to the dictionary - - :param conversion: conversion to add - :type conversion: UnitConversion - """ - cls._dict_conversions[(conversion.source, conversion.destiny)] = conversion - - @classmethod - def get_conversion_factor_offset(cls, input_units, output_units): - """ - Get the conversion factor and offset for two units. - - The conversion has to be done in the following way: - converted = original * factor + offset - - :param input_units: original units - :type input_units: str - :param output_units: destiny units - :type output_units: str - :return: factor and offset - :rtype: [float, float] - """ - units = input_units.split() - if len(units) == 1: - scale_unit = 1 - unit = units[0] - else: - if '^' in units[0]: - values = units[0].split('^') - scale_unit = pow(int(values[0]), int(values[1])) - else: - scale_unit = float(units[0]) - unit = units[1] - - units = output_units.split() - if len(units) == 1: - scale_new_unit = 1 - new_unit = units[0] - else: - if '^' in units[0]: - values = units[0].split('^') - scale_new_unit = pow(int(values[0]), int(values[1])) - else: - scale_new_unit = float(units[0]) - new_unit = units[1] - - factor, offset = UnitConversion._get_factor(new_unit, unit) - if factor is None: - return None, None - factor = factor * scale_unit / float(scale_new_unit) - offset /= float(scale_new_unit) - - return factor, offset - - @classmethod - def _get_factor(cls, new_unit, unit): - # Add only the conversions with a factor greater than 1 - if unit == new_unit: - return 1, 0 - elif (unit, new_unit) in cls._dict_conversions: - conversion = cls._dict_conversions[(unit, new_unit)] - return conversion.factor, conversion.offset - elif (new_unit, unit) in cls._dict_conversions: - conversion = cls._dict_conversions[(new_unit, unit)] - return 1 / conversion.factor, -conversion.offset - else: - return None, None diff --git a/test/unit/statistics/test_datafile.py b/test/unit/statistics/test_datafile.py new file mode 100644 index 00000000..6b4b70fe --- /dev/null +++ b/test/unit/statistics/test_datafile.py @@ -0,0 +1,34 @@ +from unittest import TestCase + +from earthdiagnostics.datafile import UnitConversion + +class TestConversion(TestCase): + + def test__init(self): + conversion = UnitConversion('km', 'm', 1000, 0) + self.assertEqual(conversion.source, 'km') + self.assertEqual(conversion.destiny, 'm') + self.assertEqual(conversion.factor, 1000) + self.assertEqual(conversion.offset, 0) + + def test_add_conversion(self): + UnitConversion._dict_conversions = dict() + conversion = UnitConversion('km', 'm', 1000, 0) + UnitConversion.add_conversion(conversion) + self.assertIs(UnitConversion._dict_conversions[('km', 'm')], conversion) + UnitConversion._dict_conversions = dict() + + def test_get_factor_offset(self): + UnitConversion._dict_conversions = dict() + conversion = UnitConversion('km', 'm', 1000, 0) + UnitConversion.add_conversion(conversion) + self.assertEqual(UnitConversion.get_conversion_factor_offset('km', 'm'), (1000, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('m', 'km'), (1 / 1000.0, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('1e3 m', 'km'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('10^3 m', 'km'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('km', '1e3 m'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('km', '10^3 m'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('m', 'm'), (1, 0)) + self.assertEqual(UnitConversion.get_conversion_factor_offset('m²', 'km'), (None, None)) + + UnitConversion._dict_conversions = dict() diff --git a/test/unit/test_data_manager.py b/test/unit/test_data_manager.py index 8277719e..00d03b8c 100644 --- a/test/unit/test_data_manager.py +++ b/test/unit/test_data_manager.py @@ -4,42 +4,10 @@ from unittest import TestCase from mock import Mock -from earthdiagnostics.datamanager import UnitConversion, DataManager +from earthdiagnostics.datamanager import DataManager from earthdiagnostics.modelingrealm import ModelingRealms -class TestConversion(TestCase): - - def test__init(self): - conversion = UnitConversion('km', 'm', 1000, 0) - self.assertEqual(conversion.source, 'km') - self.assertEqual(conversion.destiny, 'm') - self.assertEqual(conversion.factor, 1000) - self.assertEqual(conversion.offset, 0) - - def test_add_conversion(self): - UnitConversion._dict_conversions = dict() - conversion = UnitConversion('km', 'm', 1000, 0) - UnitConversion.add_conversion(conversion) - self.assertIs(UnitConversion._dict_conversions[('km', 'm')], conversion) - UnitConversion._dict_conversions = dict() - - def test_get_factor_offset(self): - UnitConversion._dict_conversions = dict() - conversion = UnitConversion('km', 'm', 1000, 0) - UnitConversion.add_conversion(conversion) - self.assertEqual(UnitConversion.get_conversion_factor_offset('km', 'm'), (1000, 0)) - self.assertEqual(UnitConversion.get_conversion_factor_offset('m', 'km'), (1 / 1000.0, 0)) - self.assertEqual(UnitConversion.get_conversion_factor_offset('1e3 m', 'km'), (1, 0)) - self.assertEqual(UnitConversion.get_conversion_factor_offset('10^3 m', 'km'), (1, 0)) - self.assertEqual(UnitConversion.get_conversion_factor_offset('km', '1e3 m'), (1, 0)) - self.assertEqual(UnitConversion.get_conversion_factor_offset('km', '10^3 m'), (1, 0)) - self.assertEqual(UnitConversion.get_conversion_factor_offset('m', 'm'), (1, 0)) - self.assertEqual(UnitConversion.get_conversion_factor_offset('m²', 'km'), (None, None)) - - UnitConversion._dict_conversions = dict() - - class TestDataManager(TestCase): def setUp(self): -- GitLab From 5ef52ef423a6cdba52559a0927d8707aeec92c9f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 12:49:33 +0100 Subject: [PATCH 144/168] Removed duplication for int diagnostic options --- earthdiagnostics/diagnostic.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index e743c807..34125249 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -507,14 +507,17 @@ class DiagnosticIntOption(DiagnosticOption): def parse(self, option_value): value = int(self._check_default(option_value)) + self._check_limits(value) + return value + + def _check_limits(self, value): if self.min_limit is not None and value < self.min_limit: raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) if self.max_limit is not None and value > self.max_limit: raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) - return value -class DiagnosticListIntOption(DiagnosticOption): +class DiagnosticListIntOption(DiagnosticIntOption): """ Class for parsing integer list options @@ -542,12 +545,7 @@ class DiagnosticListIntOption(DiagnosticOption): return option_value values = [int(i) for i in option_value.split('-')] for value in values: - # noinspection PyTypeChecker - if self.min_limit is not None and value < self.min_limit: - raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) - # noinspection PyTypeChecker - if self.max_limit is not None and value > self.max_limit: - raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) + self._check_limits(value) return values -- GitLab From ee47081199c573aee501572db496db2a041de487 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 13:36:48 +0100 Subject: [PATCH 145/168] Added var_manager tests --- earthdiagnostics/modelingrealm.py | 2 ++ earthdiagnostics/variable.py | 23 +++++++++++---------- earthdiagnostics/variable_alias/cmip6.csv | 1 - test/unit/{statistics => }/test_datafile.py | 2 ++ test/unit/test_variable.py | 17 ++++++++++++++- 5 files changed, 32 insertions(+), 13 deletions(-) rename test/unit/{statistics => }/test_datafile.py (99%) diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index 0c8987d1..af0856f4 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -147,6 +147,8 @@ class ModelingRealms(object): """ if isinstance(modelling_realm, ModelingRealm): return modelling_realm + if modelling_realm == '': + return None for name in cls.__dict__.keys(): if name.startswith('_'): continue diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 0af78c00..1aa7735e 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -135,17 +135,18 @@ class VariableManager(object): for file_name in os.listdir(json_folder): if file_name in ('CMIP6_grids.json', 'CMIP6_formula_terms.json'): continue - json_data = open(os.path.join(json_folder, file_name)).read() - try: - data = json.loads(json_data) - except ValueError: - continue - if 'variable_entry' in data: - Log.debug('Parsing file {0}'.format(file_name)) - table_id = data['Header']['table_id'][6:] - table = CMORTable(table_id, Frequency(data['Header']['frequency']), data['Header']['table_date']) - self.tables[table_id] = table - self._load_json_variables(data['variable_entry'], table) + with open(os.path.join(json_folder, file_name)) as json_file: + json_data = json_file.read() + try: + data = json.loads(json_data) + except ValueError: + continue + if 'variable_entry' in data: + Log.debug('Parsing file {0}'.format(file_name)) + table_id = data['Header']['table_id'][6:] + table = CMORTable(table_id, Frequency(data['Header']['frequency']), data['Header']['table_date']) + self.tables[table_id] = table + self._load_json_variables(data['variable_entry'], table) def _load_json_variables(self, json_data, table): for short_name in json_data.keys(): diff --git a/earthdiagnostics/variable_alias/cmip6.csv b/earthdiagnostics/variable_alias/cmip6.csv index e6c8a679..62d12675 100644 --- a/earthdiagnostics/variable_alias/cmip6.csv +++ b/earthdiagnostics/variable_alias/cmip6.csv @@ -78,4 +78,3 @@ hflx_rain_cea,hfrainds,, hflx_cal_cea,hfibthermds2d,, rain,prra,, calving,ficeberg2d,, - diff --git a/test/unit/statistics/test_datafile.py b/test/unit/test_datafile.py similarity index 99% rename from test/unit/statistics/test_datafile.py rename to test/unit/test_datafile.py index 6b4b70fe..4a17c691 100644 --- a/test/unit/statistics/test_datafile.py +++ b/test/unit/test_datafile.py @@ -1,7 +1,9 @@ +# coding=utf-8 from unittest import TestCase from earthdiagnostics.datafile import UnitConversion + class TestConversion(TestCase): def test__init(self): diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index fddc8f2a..43e4579d 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -2,7 +2,7 @@ from mock import Mock from unittest import TestCase -from earthdiagnostics.variable import CMORTable, VariableAlias, Variable, VariableJsonException +from earthdiagnostics.variable import CMORTable, VariableAlias, Variable, VariableJsonException, VariableManager from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.constants import Basins from earthdiagnostics.frequency import Frequencies @@ -182,3 +182,18 @@ class TestVariable(TestCase): var = Variable() with self.assertRaises(ValueError): var.get_table(Frequencies.daily, 'specs') + + +class TestVariableManager(TestCase): + + def setUp(self): + self.var_manager = VariableManager() + + def test_load_primavera(self): + self.var_manager.load_variables('primavera') + + def test_load_cmip6(self): + self.var_manager.load_variables('cmip6') + + def test_load_specs(self): + self.var_manager.load_variables('specs') -- GitLab From e647bc151ccfbd9fdcd5def5bbe75dfa3592ea27 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 16:36:44 +0100 Subject: [PATCH 146/168] Test runner --- .gitlab-ci.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a913fdad..ecde0ea8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,21 +1,16 @@ test_python2: script: - - echo $USER - export PATH="$HOME/miniconda2/bin:$PATH" - - echo $PATH - conda env update -f environment.yml -n earthdiagnostics2 python=2 - source activate earthdiagnostics - - echo $PWD + - tree - coverage run -m unittest discover test_python3: script: - - echo $USER - export PATH="$HOME/miniconda2/bin:$PATH" - - echo $PATH - conda env update -f environment.yml -n earthdiagnostics3 python=3 - source activate earthdiagnostics - - echo $PWD - coverage run -m unittest discover - coverage xml - python-codacy-coverage -r coverage.xml -- GitLab From cc4758ec48eac62030adcaced85e5b057467a151 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 16:40:09 +0100 Subject: [PATCH 147/168] added git submodule config --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecde0ea8..18dc9afc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,6 @@ +variables: + GIT_SUBMODULE_STRATEGY: recursive + test_python2: script: - export PATH="$HOME/miniconda2/bin:$PATH" -- GitLab From 3f33803f989921bb4f44b5f10122470c540d3c9f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 16:40:45 +0100 Subject: [PATCH 148/168] removed tree command --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18dc9afc..02ff16f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,6 @@ test_python2: - export PATH="$HOME/miniconda2/bin:$PATH" - conda env update -f environment.yml -n earthdiagnostics2 python=2 - source activate earthdiagnostics - - tree - coverage run -m unittest discover test_python3: -- GitLab From 43afe844dd1a861079dd651456abfa557103b13b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 16:43:49 +0100 Subject: [PATCH 149/168] Added no SSL verify to runner --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 02ff16f7..910334bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,18 @@ variables: GIT_SUBMODULE_STRATEGY: recursive +before_script: + - export GIT_SSL_NO_VERIFY=1 + - export PATH="$HOME/miniconda2/bin:$PATH" + test_python2: script: - - export PATH="$HOME/miniconda2/bin:$PATH" - conda env update -f environment.yml -n earthdiagnostics2 python=2 - source activate earthdiagnostics - coverage run -m unittest discover test_python3: script: - - export PATH="$HOME/miniconda2/bin:$PATH" - conda env update -f environment.yml -n earthdiagnostics3 python=3 - source activate earthdiagnostics - coverage run -m unittest discover -- GitLab From a98b1affa9c30ba2d757ceb14953b62756a00bce Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 16:48:58 +0100 Subject: [PATCH 150/168] Added git clone by hand for submodules --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 910334bb..fdc37552 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,7 @@ -variables: - GIT_SUBMODULE_STRATEGY: recursive - before_script: - export GIT_SSL_NO_VERIFY=1 + - git submodule sync --recursive + - git submodule update --init --recursive - export PATH="$HOME/miniconda2/bin:$PATH" test_python2: -- GitLab From e968d8436a9946401d35293a732d3ad0d2356b70 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 9 Jan 2018 18:03:05 +0100 Subject: [PATCH 151/168] Updated variable manager tests --- earthdiagnostics/constants.py | 2 ++ earthdiagnostics/variable.py | 30 ++++++++++++++----- test/unit/test_variable.py | 56 +++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index 3e2bf040..a244ca5c 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -14,6 +14,8 @@ class Basin(object): self._name = name def __eq__(self, other): + if other is None: + return False if self.name != other.name: return False return True diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 1aa7735e..52ab3262 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -25,7 +25,11 @@ class VariableManager(object): def __init__(self): self._cmor_tables_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cmor_tables') self._aliases_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'variable_alias') + self.clean() + + def clean(self): self._dict_variables = {} + self._dict_aliases = {} self.tables = {} def get_variable(self, original_name, silent=False): @@ -87,7 +91,7 @@ class VariableManager(object): self._load_variable_list(table_name) self._load_missing_defaults() self._load_known_aliases(table_name) - self._construct_aliases_dict() + self.create_aliases_dict() def _load_variable_list(self, table_name): @@ -129,7 +133,10 @@ class VariableManager(object): continue var.default = default - self._dict_variables[var.short_name.lower()] = var + self.register_variable(var) + + def register_variable(self, var): + self._dict_variables[var.short_name.lower()] = var def _load_json(self, json_folder): for file_name in os.listdir(json_folder): @@ -160,7 +167,7 @@ class VariableManager(object): try: variable.parse_json(json_data[short_name], short_name) variable.add_table(table) - self._dict_variables[variable.short_name.lower()] = variable + self.register_variable(variable) except VariableJsonException: Log.error('Could not read variable {0}'.format(short_name)) @@ -228,12 +235,13 @@ class VariableManager(object): csv_table_path = os.path.join(self._aliases_folder, '{0}.csv'.format(filename)) return csv_table_path - def _construct_aliases_dict(self): + def create_aliases_dict(self): self._dict_aliases = {} for cmor_var_name in self._dict_variables: cmor_var = self._dict_variables[cmor_var_name] - if cmor_var_name not in cmor_var.known_aliases: - cmor_var.known_aliases.append(VariableAlias(cmor_var_name)) + base_alias = VariableAlias(cmor_var_name) + if base_alias not in cmor_var.known_aliases: + cmor_var.known_aliases.append(base_alias) for alias in cmor_var.known_aliases: self._dict_aliases[alias.alias] = (alias, cmor_var) @@ -296,7 +304,7 @@ class VariableManager(object): var.domain = self._process_modelling_realm(var, row[12].value) var.units = row[2].value - self._dict_variables[var.short_name.lower()] = var + self.register_variable(var) var.add_table(table, priority) @staticmethod @@ -508,6 +516,14 @@ class VariableAlias(object): string += ' Grid: {0}'.format(self.grid) return string + def __eq__(self, other): + if other is None: + return False + return self.alias == other.alias and self.grid == other.grid and self.basin == other.basin + + def __ne__(self, other): + return not self == other + class CMORTable(object): """ diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index 43e4579d..81e17115 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -189,6 +189,9 @@ class TestVariableManager(TestCase): def setUp(self): self.var_manager = VariableManager() + def tearDown(self): + self.var_manager.clean() + def test_load_primavera(self): self.var_manager.load_variables('primavera') @@ -197,3 +200,56 @@ class TestVariableManager(TestCase): def test_load_specs(self): self.var_manager.load_variables('specs') + + def test_load_preface(self): + self.var_manager.load_variables('preface') + + def test_bad_load(self): + with self.assertRaises(Exception): + self.var_manager.load_variables('badconvention') + + def test_get_variable(self): + var1 = self._get_var_mock('var1', ['var1_alias']) + self.var_manager.register_variable(var1) + self.var_manager.create_aliases_dict() + + self.assertIs(self.var_manager.get_variable('var1'), var1) + self.assertIs(self.var_manager.get_variable('var1_alias'), var1) + + self.assertIsNone(self.var_manager.get_variable('var2')) + self.assertIsNone(self.var_manager.get_variable('var2', True)) + + def test_get_variable_and_alias(self): + var1 = self._get_var_mock('var1', ['var1_alias']) + self.var_manager.register_variable(var1) + self.var_manager.create_aliases_dict() + + self.assertEqual(self.var_manager.get_variable_and_alias('var1'), (VariableAlias('var1'), var1)) + self.assertEqual(self.var_manager.get_variable_and_alias('var1_alias'), (VariableAlias('var1_alias'), var1)) + + self.assertEqual(self.var_manager.get_variable_and_alias('var2'), (None, None)) + self.assertEqual(self.var_manager.get_variable_and_alias('var2', True), (None, None)) + + def _get_var_mock(self, name, aliases): + var1 = Mock() + var1.short_name = name + + def get_alias_mock(alias): + alias_mock = Mock() + alias_mock.alias = alias + alias_mock.grid = None + alias_mock.basin = None + return alias_mock + + var1.known_aliases = [get_alias_mock(alias) for alias in [name] + aliases] + return var1 + + def test_get_all_variables(self): + var1 = self._get_var_mock('var1', ['var1_alias']) + var2 = self._get_var_mock('var2', ['var2_alias']) + + self.var_manager.register_variable(var1) + self.var_manager.register_variable(var2) + self.var_manager.create_aliases_dict() + + self.assertEqual(self.var_manager.get_all_variables(), [var1, var2]) -- GitLab From d951340a24cc825ae2d113ca88f769d96c128b49 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 10 Jan 2018 12:12:14 +0100 Subject: [PATCH 152/168] Improved tests for cmormanager --- earthdiagnostics/cmorizer.py | 2 +- earthdiagnostics/cmormanager.py | 17 +-- earthdiagnostics/variable.py | 2 +- test/unit/test_cmormanager.py | 211 +++++++++++++++++++++++++++++++- 4 files changed, 212 insertions(+), 20 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index affeceb3..1e2d718b 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -545,7 +545,7 @@ class Cmorizer(object): var_codes = self.config.cmor.get_variables(frequency) for var_code in var_codes: if not os.path.exists('{0}_{1}.128.nc'.format(gribfile, var_code)): - continue + continue new_units = None cdo_operator = '-selmon,{0}'.format(month) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index c39234ea..c982c16e 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -281,11 +281,7 @@ class CMORManager(DataManager): cmor_table = cmor_var.get_table(frequency, self.config.data_convention) time_bound = self._get_time_component(chunk, date_str, frequency, startdate, year) - - if time_bound: - time_bound = '_{0}.nc'.format(time_bound) - else: - time_bound = '.nc' + time_bound = '_{0}.nc'.format(time_bound) if self.config.data_convention in ('specs', 'preface'): @@ -425,9 +421,6 @@ class CMORManager(DataManager): try: if grid == 'original': link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) - if os.path.islink(link_path): - link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) - Utils.create_folder_tree(link_path) else: link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) @@ -445,7 +438,7 @@ class CMORManager(DataManager): if move_old and link_path not in self._checked_vars: self._checked_vars.append(link_path) old_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, - 'old_{0}_f{1}h'.format(var, self.experiment.atmos_timestep)) + 'old_{0}'.format(os.path.basename(link_path))) regex = re.compile(var + '_[0-9]{6,8}\.nc') for filename in os.listdir(link_path): if regex.match(filename): @@ -458,8 +451,6 @@ class CMORManager(DataManager): os.remove(link_path) if not os.path.exists(filepath): raise ValueError('Original file {0} does not exists'.format(filepath)) - if not os.path.isdir(os.path.dirname(link_path)): - Utils.create_folder_tree(os.path.dirname(link_path)) relative_path = os.path.relpath(filepath, os.path.dirname(link_path)) os.symlink(relative_path, link_path) except Exception: @@ -684,7 +675,7 @@ class CMORManager(DataManager): for domain in os.listdir(os.path.join(path, freq)): for var in os.listdir(os.path.join(path, freq, domain)): for member in os.listdir(os.path.join(path, freq, domain, var)): - if member_str != member: + if member_str is not None and member_str != member: continue for name in os.listdir(os.path.join(path, freq, domain, var, member)): filepath = os.path.join(path, freq, domain, var, member, name) @@ -703,7 +694,7 @@ class CMORManager(DataManager): domain = None for var in os.listdir(os.path.join(path, member, table)): for grid in os.listdir(os.path.join(path, member, table, var)): - if member_str != member: + if member_str is not None and member_str != member: continue for name in os.listdir(os.path.join(path, member, table, var, grid)): filepath = os.path.join(path, member, table, var, grid, name) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 52ab3262..a2a3bc88 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -538,7 +538,7 @@ class CMORTable(object): def __init__(self, name, frequency, date): self.name = name - self.frequency = Frequency.parse(frequency) + self.frequency = frequency self.date = date def __str__(self): diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 90fb7c59..ac129d75 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -7,7 +7,7 @@ from unittest import TestCase import mock from mock import Mock -from earthdiagnostics.cmormanager import CMORManager +from earthdiagnostics.cmormanager import CMORManager, MergeYear from earthdiagnostics.modelingrealm import ModelingRealms @@ -101,6 +101,28 @@ class TestCMORManager(TestCase): 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + def test_get_file_path_specs_non_cmor(self): + cmor_manager = CMORManager(self.config) + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'mon' + frequency.frequency = 'mon' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + def test_get_file_path_specs_empty_time_info(self): + cmor_manager = CMORManager(self.config) + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'mon' + frequency.frequency = 'mon' + with self.assertRaises(ValueError): + cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, None, + frequency) + def test_get_file_path_preface(self): self._configure_preface() cmor_manager = CMORManager(self.config) @@ -271,6 +293,23 @@ class TestCMORManager(TestCase): 'Omon/var/grid/version/' 'var_Omon_model_expname_r2i1p1f1_grid_198901-198912.nc')) + def test_get_file_path_primavera_atmos(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.atmos, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' + 'r2i1p1f1/Omon/var/atmos_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_atmos_grid_198901-198912.nc')) + def test_get_file_path_primavera_year(self): self._configure_primavera() cmor_manager = CMORManager(self.config) @@ -317,16 +356,15 @@ class TestCMORManager(TestCase): with mock.patch('os.path.isfile') as isfile: cmor_manager = CMORManager(self.config) isfile.return_value = True - self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1)) + self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1, 1)) isfile.return_value = False - self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1)) - + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1, 1)) def test_file_exists_multiple_versions(self): with mock.patch('os.path.isfile') as isfile: cmor_manager = CMORManager(self.config) isfile.return_value = True - self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1, 1, possible_versions=('version1', 'version2'))) isfile.return_value = False self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, @@ -352,6 +390,20 @@ class TestCMORManager(TestCase): self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_01.nc')) + def test_get_file_path_bad_convention(self): + self.config.data_convention = 'badconvention' + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'day' + with self.assertRaises(ValueError): + cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 1, + frequency) + def test_create_link(self): cmor_manager = CMORManager(self.config) file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) @@ -362,6 +414,59 @@ class TestCMORManager(TestCase): filename = os.path.basename(path) self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + def test_create_link_no_source(self): + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.remove(path) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + with self.assertRaises(ValueError): + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) + + def test_create_link_exists(self): + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + filename = os.path.basename(path) + + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + os.symlink(path, os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename)) + + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) + self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + + def test_create_link_default_is_link(self): + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h')) + os.symlink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h'), + os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) + filename = os.path.basename(path) + self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + + def test_create_link_move_old(self): + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + fd = open(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', 'var_20001101.nc'), 'w') + fd.close() + + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, True, None) + filename = os.path.basename(path) + self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'old_var_f6h', + 'var_20001101.nc'))) + def test_create_link_with_grid(self): cmor_manager = CMORManager(self.config) file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) @@ -376,6 +481,54 @@ class TestCMORManager(TestCase): self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) + def test_create_link_with_grid_default_is_link(self): + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h')) + os.symlink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h'), + os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) + filename = os.path.basename(path) + var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') + var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') + self.assertTrue(os.path.islink(var_mainfolder)) + self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) + self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) + + def test_create_link_with_grid_default_is_folder(self): + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) + filename = os.path.basename(path) + var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') + var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') + self.assertTrue(os.path.islink(var_mainfolder)) + self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) + self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) + + def test_create_link_meteofrance(self): + self._configure_meteofrance() + cmor_manager = CMORManager(self.config) + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) + filename = os.path.basename(path) + var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') + var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') + self.assertFalse(os.path.islink(var_mainfolder)) + self.assertFalse(os.path.islink(os.path.join(var_grid_folder, filename))) + @mock.patch('earthdiagnostics.cmormanager.Cmorizer', autospec=True) def test_prepare_cmorize(self, mock_cmor): mock_instance = mock_cmor.return_value @@ -555,6 +708,15 @@ class TestCMORManager(TestCase): mock_get_file_path.assert_called_once() mock_create_link.assert_called_once() + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') + def test_link_file_default_freq(self, mock_get_file_path, mock_create_link): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + cmor_manager.link_file(ModelingRealms.ocean, 'var', cmor_var, '20010101', 1, 1) + mock_get_file_path.assert_called_once() + mock_create_link.assert_called_once() + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') def test_create_links_specs(self, mock_create_link): member_path = os.path.join(self.tmp_dir, @@ -609,6 +771,18 @@ class TestCMORManager(TestCase): cmor_manager.create_links('20010101', 1) mock_create_link.assert_called() + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') + def test_create_links_with_version_primavera_no_member(self, mock_create_link): + self._configure_primavera() + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var/gn/version') + os.makedirs(member_path) + self.config.var_manager.tables = {'Omon': Mock()} + tempfile.mkstemp(dir=member_path) + cmor_manager = CMORManager(self.config) + cmor_manager.create_links('20010101') + mock_create_link.assert_called() + @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') def test_create_links_member_not_found_primavera(self, mock_create_link): self._configure_primavera() @@ -641,6 +815,17 @@ class TestCMORManager(TestCase): datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) self.assertEqual(datafile.remote_file, '/path/to/file') + mock_frequency = Mock() + datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1, + frequency=mock_frequency) + self.assertEqual(datafile.remote_file, '/path/to/file') + + self.config.var_manager.get_variable.return_value = None + datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) + self.assertEqual(datafile.remote_file, '/path/to/file') + + + @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') def test_request_year(self, mock_get_file_path): self.config.experiment.get_year_chunks.return_value = (1, 2) @@ -657,8 +842,24 @@ class TestCMORManager(TestCase): datafile = cmor_manager.declare_year(ModelingRealms.ocean, 'var', '20010101', 1, 2001) self.assertEqual(datafile.remote_file, '/path/to/file') + self.config.var_manager.get_variable.return_value = None + datafile = cmor_manager.declare_year(ModelingRealms.ocean, 'var', '20010101', 1, 2001) + self.assertEqual(datafile.remote_file, '/path/to/file') + + +class TestMergeYear(TestCase): + + def setUp(self): + self.data_manager = Mock() + def test_generate_jobs(self): + self.assertIsNone(MergeYear.generate_jobs(None, None)) + def test_eq(self): + self.assertEqual(MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998), + MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998)) + self.assertEqual(MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998, 'grid'), + MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998, 'grid')) -- GitLab From efbb30313ef92d7354bcdef8c29407745bc95ed2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 10 Jan 2018 13:25:00 +0100 Subject: [PATCH 153/168] Updated tests --- earthdiagnostics/cmormanager.py | 12 +++--------- earthdiagnostics/variable.py | 10 ++++------ test/unit/test_cmormanager.py | 18 +++++++++++++++++- test/unit/test_frequency.py | 5 +++++ test/unit/test_variable.py | 23 +++++++++++++++++++++++ 5 files changed, 52 insertions(+), 16 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index c982c16e..ff69e385 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -298,11 +298,10 @@ class CMORManager(DataManager): self.experiment.experiment_name, self._get_member_str(member), grid, time_bound) - elif self.config.data_convention in ('meteofrance',): + # Meteofrance + else: time_bound = self._get_chunk_time_bounds(startdate, chunk) file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self._get_member_str(member)) - else: - raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name def _get_time_component(self, chunk, date_str, frequency, startdate, year): @@ -312,10 +311,8 @@ class CMORManager(DataManager): if frequency != Frequencies.yearly: raise ValueError('Year may be provided instead of chunk only if frequency is "yr"') time_bound = str(year) - elif date_str: - time_bound = date_str else: - raise ValueError('Chunk, year and date_str can not be None at the same time') + time_bound = date_str return time_bound def _get_full_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): @@ -575,9 +572,6 @@ class CMORManager(DataManager): Log.info('Unzipping cmorized data for {0} {1} {2}...', startdate, member, chunk) Utils.unzip(filepaths, True) - if not os.path.exists(self.cmor_path): - os.mkdir(self.cmor_path) - filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar') if len(filepaths) > 0: Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index a2a3bc88..41fc7e75 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -177,8 +177,6 @@ class VariableManager(object): def _load_alias_csv(self, filename): file_path = self._get_aliases_csv_path(filename) - if not os.path.isfile(file_path): - return with open(file_path, 'r') as csvfile: reader = csv.reader(csvfile, dialect='excel') @@ -503,10 +501,10 @@ class VariableAlias(object): """ - def __init__(self, alias): + def __init__(self, alias, basin=None, grid=None): self.alias = alias - self.basin = None - self.grid = None + self.basin = basin + self.grid = grid def __str__(self): string = self.alias @@ -545,7 +543,7 @@ class CMORTable(object): return self.name def __repr__(self): - return '{0.name} ({0.frequency}, {0.date}'.format(self) + return '{0.name} ({0.frequency}, {0.date})'.format(self) def __lt__(self, other): return self.name < other.name diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index ac129d75..5b673136 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -196,7 +196,7 @@ class TestCMORManager(TestCase): self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, 1, frequency, year='1998') - def test_get_file_path_raise_incomaptible_date_info(self): + def test_get_file_path_raise_incompatible_date_info(self): cmor_manager = CMORManager(self.config) cmor_var = Mock() omon = Mock() @@ -207,6 +207,8 @@ class TestCMORManager(TestCase): frequency.__str__ = Mock() frequency.__str__.return_value = 'frequency' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + None, frequency, year='1998') self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 1, frequency, year='1998') self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, @@ -252,6 +254,20 @@ class TestCMORManager(TestCase): 'r2i1p1f1/Omon/var/ocean_grid/version/' 'var_Omon_model_expname_r2i1p1f1_ocean_grid_198901-198912.nc')) + def test_get_file_path_primavera_no_cmor(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'mon' + frequency.frequency = 'mon' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' + 'r2i1p1f1/Omon/var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_198901-198912.nc')) + def test_get_file_path_no_version_primavera(self): self._configure_primavera() self.config.cmor.version = '' diff --git a/test/unit/test_frequency.py b/test/unit/test_frequency.py index d556e36f..b92069e3 100644 --- a/test/unit/test_frequency.py +++ b/test/unit/test_frequency.py @@ -7,6 +7,11 @@ from earthdiagnostics.variable import VariableType class TestFrequency(TestCase): + def test_parse(self): + freq = Frequency('m') + self.assertEqual(Frequency.parse('m'), freq) + self.assertIs(Frequency.parse(freq), freq) + def test_not_supported(self): with self.assertRaises(ValueError): Frequency('badfreq') diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index 81e17115..6fd11e7b 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -16,6 +16,12 @@ class TestCMORTable(TestCase): def test_str(self): self.assertEqual(str(CMORTable('name', 'm', 'Month YEAR')), 'name') + def test_repr(self): + self.assertEqual(repr(CMORTable('name', 'm', 'Month YEAR')), 'name (m, Month YEAR)') + + def test_lt(self): + self.assertLess(CMORTable('a', 'm', 'Month YEAR'), CMORTable('b', 'm', 'Month YEAR')) + class TestVariableAlias(TestCase): @@ -27,6 +33,23 @@ class TestVariableAlias(TestCase): alias.grid = 'grid' self.assertEqual(str(alias), 'alias Basin: basin Grid: grid') + def test_eq(self): + alias = VariableAlias('alias') + self.assertEqual(alias, VariableAlias('alias')) + alias.basin = 'basin' + self.assertEqual(alias, VariableAlias('alias', 'basin')) + alias.grid = 'grid' + self.assertEqual(alias, VariableAlias('alias', 'basin', 'grid')) + self.assertNotEqual(alias, None) + + def test_ne(self): + alias = VariableAlias('alias') + self.assertNotEqual(alias, VariableAlias('alias2')) + alias.basin = 'basin' + self.assertNotEqual(alias, VariableAlias('alias', 'basin2')) + alias.grid = 'grid' + self.assertNotEqual(alias, VariableAlias('alias', 'basin', 'grid2')) + class TestVariable(TestCase): -- GitLab From ff53b215c7943fa5d008f162766ec376ee9e0d0b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 10 Jan 2018 15:51:46 +0100 Subject: [PATCH 154/168] Added tests for obsreconmanager --- test/run_test.py | 8 +- test/unit/test_earthdiags.py | 17 ++-- test/unit/test_obsreconmanager.py | 142 ++++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 test/unit/test_obsreconmanager.py diff --git a/test/run_test.py b/test/run_test.py index 765057a5..12614344 100644 --- a/test/run_test.py +++ b/test/run_test.py @@ -5,6 +5,7 @@ Script to run the tests for EarthDiagnostics and generate the code coverage repo import os +import sys work_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) os.chdir(work_path) print(work_path) @@ -13,7 +14,12 @@ import unittest cov = coverage.Coverage() cov.start() -suite = unittest.TestLoader().discover('.') +if len(sys.argv) == 1: + suite = unittest.TestLoader().discover('.') +else: + suite = unittest.TestLoader().discover('.', pattern=sys.argv[1]) + + unittest.TextTestRunner(verbosity=2).run(suite) cov.stop() diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py index 122793ec..95468b71 100644 --- a/test/unit/test_earthdiags.py +++ b/test/unit/test_earthdiags.py @@ -1,8 +1,15 @@ -# # coding=utf-8 -# from unittest import TestCase +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.earthdiags import EarthDiags + # -# from earthdiagnostics.earthdiags import EarthDiags +# class TestEarthDiags(TestCase): # +# def setUp(self): +# self.earthdiags = EarthDiags('path/to/conf') # -# class TestEarthDiags(TestCase): -# pass +# self.earthdiags.config() +# +# def test_clean(self): +# self.earthdiags.parse_args() diff --git a/test/unit/test_obsreconmanager.py b/test/unit/test_obsreconmanager.py new file mode 100644 index 00000000..05d98f35 --- /dev/null +++ b/test/unit/test_obsreconmanager.py @@ -0,0 +1,142 @@ +# coding=utf-8 +import os +import shutil +import tempfile +from unittest import TestCase +import mock +from mock import Mock + +from earthdiagnostics.obsreconmanager import ObsReconManager +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestObsReconManager(TestCase): + + def setUp(self): + self.config = Mock() + self.config.data_convention = 'specs' + self.config.data_type = 'exp' + self.config.experiment.expid = 'expid' + self.config.experiment.model = 'model' + self.config.experiment.experiment_name = 'expname' + self.config.experiment.institute = 'institute' + self.config.experiment.member_count_start = 0 + self.config.experiment.chunk_size = 12 + self.config.experiment.num_chunks = 1 + self.config.experiment.calendar = 'standard' + self.config.experiment.atmos_timestep = 3 + self.config.experiment.ocean_timestep = 6 + + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'mon' + frequency.folder_name.return_value = 'monthly_mean' + + self.config.frequency = frequency + + self.config.cmor.initialization_number = 1 + self.config.cmor.version = '' + self.config.cmor.default_ocean_grid = 'ocean_grid' + self.config.cmor.default_atmos_grid = 'atmos_grid' + self.config.cmor.activity = 'activity' + self.config.cmor.force = False + self.config.cmor.force_untar = False + + self.tmp_dir = tempfile.mkdtemp() + self._make_data_folder() + self.config.data_dir = self.tmp_dir + + def _make_data_folder(self): + os.makedirs(os.path.join(self.tmp_dir, self.config.data_type, self.config.experiment.institute, + self.config.experiment.model, self.config.experiment.expid)) + + def tearDown(self): + if os.path.exists(self.tmp_dir): + shutil.rmtree(self.tmp_dir) + + def test_not_data_dir(self): + shutil.rmtree(self.tmp_dir) + with self.assertRaises(Exception): + ObsReconManager(self.config) + + def test_bad_num_chunks(self): + self.config.experiment.chunk_size = 3 + self.config.data_type = 'obs' + self._make_data_folder() + with self.assertRaises(Exception): + ObsReconManager(self.config) + + def test_get_file_path(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + frequency.folder_name.return_value = 'folder_name' + + manager = ObsReconManager(self.config) + self.assertEqual(manager.get_file_path('19900101', ModelingRealms.ocean, 'var', frequency, None), + os.path.join(self.tmp_dir, 'exp/institute/model/folder_name/var/var_19900101.nc')) + + def test_get_file_path_default_freq(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + + manager = ObsReconManager(self.config) + self.assertEqual(manager.get_file_path('19900101', ModelingRealms.ocean, 'var', None, None), + os.path.join(self.tmp_dir, 'exp/institute/model/monthly_mean/var/var_19900101.nc')) + + def test_get_file_path_no_startdate(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + + manager = ObsReconManager(self.config) + self.assertEqual(manager.get_file_path(None, ModelingRealms.ocean, 'var', None, None), + os.path.join(self.tmp_dir, 'exp/institute/model/monthly_mean/var/var.nc')) + + def test_get_file_path_recon_weekly(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + + self.config.data_type = 'recon' + self.config.experiment.chunk_size = 1 + self._make_data_folder() + + manager = ObsReconManager(self.config) + self.assertEqual(manager.get_file_path('19900101', ModelingRealms.ocean, 'var', None, None), + os.path.join(self.tmp_dir, 'recon/institute/model/monthly_mean/var/var_199001.nc')) + + @mock.patch('earthdiagnostics.obsreconmanager.ObsReconManager.get_file_path') + def test_declare_chunk(self, mock_get_file_path): + mock_get_file_path.return_value = '/path/to/file' + cmor_manager = ObsReconManager(self.config) + datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) + self.assertEqual(datafile.remote_file, '/path/to/file') + + mock_frequency = Mock() + datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1, + frequency=mock_frequency) + self.assertEqual(datafile.remote_file, '/path/to/file') + + self.config.var_manager.get_variable.return_value = None + datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) + self.assertEqual(datafile.remote_file, '/path/to/file') + + @mock.patch('earthdiagnostics.obsreconmanager.ObsReconManager.get_file_path') + def test_request_chunk(self, mock_get_file_path): + mock_get_file_path.return_value = '/path/to/file' + cmor_manager = ObsReconManager(self.config) + datafile = cmor_manager.request_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) + self.assertEqual(datafile.remote_file, '/path/to/file') + + + + -- GitLab From de3f6005e56bbc73e934745a0457186d7172d2c8 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Mon, 15 Jan 2018 18:20:18 +0100 Subject: [PATCH 155/168] Added more tests --- earthdiagnostics/variable.py | 68 ++++++++++++++++++++---------------- test/unit/test_datafile.py | 48 ++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 31 deletions(-) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 41fc7e75..e0808288 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -11,6 +11,7 @@ from bscearth.utils.log import Log from earthdiagnostics.constants import Basins from earthdiagnostics.frequency import Frequency from earthdiagnostics.modelingrealm import ModelingRealms +from concurrent.futures import ThreadPoolExecutor class VariableJsonException(Exception): @@ -139,21 +140,27 @@ class VariableManager(object): self._dict_variables[var.short_name.lower()] = var def _load_json(self, json_folder): + executor = ThreadPoolExecutor() for file_name in os.listdir(json_folder): if file_name in ('CMIP6_grids.json', 'CMIP6_formula_terms.json'): continue - with open(os.path.join(json_folder, file_name)) as json_file: - json_data = json_file.read() - try: - data = json.loads(json_data) - except ValueError: - continue - if 'variable_entry' in data: - Log.debug('Parsing file {0}'.format(file_name)) - table_id = data['Header']['table_id'][6:] - table = CMORTable(table_id, Frequency(data['Header']['frequency']), data['Header']['table_date']) - self.tables[table_id] = table - self._load_json_variables(data['variable_entry'], table) + + executor.submit(self._load_json_file, os.path.join(json_folder, file_name)) + executor.shutdown(True) + + def _load_json_file(self, json_path): + with open(json_path) as json_file: + json_data = json_file.read() + try: + data = json.loads(json_data) + except ValueError: + return + if 'variable_entry' in data: + Log.debug('Parsing file {0}'.format(json_path)) + table_id = data['Header']['table_id'][6:] + table = CMORTable(table_id, Frequency(data['Header']['frequency']), data['Header']['table_date']) + self.tables[table_id] = table + self._load_json_variables(data['variable_entry'], table) def _load_json_variables(self, json_data, table): for short_name in json_data.keys(): @@ -262,26 +269,27 @@ class VariableManager(object): for row in data_sheet.rows: if row[1].value in excel.sheetnames: table_data[row[1].value] = (Frequency(row[2].value), 'Date missing') - for sheet_name in excel.sheetnames: - try: - sheet = excel.get_sheet_by_name(sheet_name) - if sheet.title == 'Primday': - pass - if sheet['A1'].value != 'Priority': - continue - table_frequency, table_date = table_data[sheet.title] - table = CMORTable(sheet.title, table_frequency, table_date) - self.tables[sheet.title] = table - for row in sheet.rows: - if row[0].value == 'Priority' or not row[5].value: - continue - self._parse_xlsx_var_row(row, table) + sheet = excel.get_sheet_by_name(sheet_name) + if sheet.title == 'Primday': + pass + if sheet['A1'].value != 'Priority': + continue + self._load_xlsx_table(sheet, table_data) - except Exception as ex: - Log.error('Table {0} can not be loaded: {1}', sheet_name, ex) - import traceback - traceback.print_exc() + def _load_xlsx_table(self, sheet, table_data): + try: + table_frequency, table_date = table_data[sheet.title] + table = CMORTable(sheet.title, table_frequency, table_date) + self.tables[sheet.title] = table + for row in sheet.rows: + if row[0].value == 'Priority' or not row[5].value: + continue + self._parse_xlsx_var_row(row, table) + except Exception as ex: + Log.error('Table {0} can not be loaded: {1}', sheet.title, ex) + import traceback + traceback.print_exc() def _parse_xlsx_var_row(self, row, table): cmor_name = row[11].value diff --git a/test/unit/test_datafile.py b/test/unit/test_datafile.py index 4a17c691..0b47650e 100644 --- a/test/unit/test_datafile.py +++ b/test/unit/test_datafile.py @@ -1,7 +1,8 @@ # coding=utf-8 from unittest import TestCase +from mock import Mock -from earthdiagnostics.datafile import UnitConversion +from earthdiagnostics.datafile import UnitConversion, DataFile, LocalStatus, StorageStatus class TestConversion(TestCase): @@ -34,3 +35,48 @@ class TestConversion(TestCase): self.assertEqual(UnitConversion.get_conversion_factor_offset('m²', 'km'), (None, None)) UnitConversion._dict_conversions = dict() + + +class TestDatafile(TestCase): + + def setUp(self): + self.data_file = DataFile() + + def test_download_required(self): + self.assertFalse(self.data_file.download_required()) + + self.data_file.local_status = LocalStatus.READY + self.assertFalse(self.data_file.download_required()) + + self.data_file.local_status = LocalStatus.COMPUTING + self.assertFalse(self.data_file.download_required()) + + self.data_file.local_status = LocalStatus.PENDING + self.data_file.storage_status = StorageStatus.READY + self.assertTrue(self.data_file.download_required()) + + self.data_file.local_status = LocalStatus.PENDING + self.data_file.storage_status = StorageStatus.UPLOADING + self.assertFalse(self.data_file.download_required()) + + def test_upload_required(self): + self.assertFalse(self.data_file.upload_required()) + + self.data_file.local_status = LocalStatus.READY + self.assertFalse(self.data_file.upload_required()) + + self.data_file.local_status = LocalStatus.COMPUTING + self.assertFalse(self.data_file.upload_required()) + + self.data_file.local_status = LocalStatus.READY + self.data_file.storage_status = StorageStatus.PENDING + self.assertTrue(self.data_file.upload_required()) + + self.data_file.local_status = LocalStatus.PENDING + self.data_file.storage_status = StorageStatus.READY + self.assertFalse(self.data_file.upload_required()) + + def test_add_modifier(self): + self.assertFalse(self.data_file.has_modifiers()) + self.data_file.add_modifier(Mock()) + self.assertTrue(self.data_file.has_modifiers()) -- GitLab From 27271258b395156e14b694f4be348fbc3421caa9 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 16 Jan 2018 10:19:23 +0100 Subject: [PATCH 156/168] First version of new regionmean (not working) --- VERSION | 2 +- diags.conf | 60 ++++++----- earthdiagnostics/earthdiags.py | 1 + earthdiagnostics/ocean/regionmean.py | 150 +++++++++++++++------------ 4 files changed, 116 insertions(+), 97 deletions(-) diff --git a/VERSION b/VERSION index d1a88439..2638df16 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc2 +3.0.0rc3 diff --git a/diags.conf b/diags.conf index d0123ff8..6ea8d8c7 100644 --- a/diags.conf +++ b/diags.conf @@ -1,44 +1,44 @@ [DIAGNOSTICS] # Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) -DATA_ADAPTOR = OBSRECON +DATA_ADAPTOR = CMOR # Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER # Root path for the cmorized data to use DATA_DIR = /esnas:/esarchive # Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) -DATA_TYPE = recon +DATA_TYPE = exp # CMORization type to use. Important also for THREDDS as it affects variable name conventions. # Options: SPECS (default), PRIMAVERA, CMIP6 -DATA_CONVENTION = SPECS +DATA_CONVENTION = PRIMAVERA # Path to NEMO's mask and grid files needed for CDFTools CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -#DIAGS = discretize,atmos,sfcWind,,0,40 -DIAGS = interpcdo,ocean,tas,r240x121,bilinear,False,,False -# DIAGS = monmean,ocean,uovmean0.0-30.0m,day monmean,ocean,vovmean0.0-30.0m,day +DIAGS = regmean,ocean,thetao +# DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. -FREQUENCY = weekly +FREQUENCY = mon # Path to CDFTOOLS binaries CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir RESTORE_MESHES = False -# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available)z +# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available) MAX_CORES = 1 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False FORCE = False # If true, CMORizes ocean files. Default = True -OCEAN_FILES = True +OCEAN_FILES = False FILTER_FILES = # If true, CMORizes atmosphere files. Default = True ATMOSPHERE_FILES = False # You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var VARIABLE_LIST = +CHUNK_LIST = 1 # Variables to be CMORized from the grib atmospheric files, separated by comma. # You can also specify the levels to extract using the following syntax @@ -63,21 +63,25 @@ ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, # PHYSICS_VERSION = 1 # PHYSICS_DESCRIPTION = # ASSOCIATED_MODEL = -# SOURCE = 'EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 +# SOURCE = EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 +VERSION = v20170705 +ACTIVITY = CMIP +#ACTIVITY = CMIP_ece2cmor [THREDDS] SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = ecmwf -MODEL = erainterim +INSTITUTE = EC-Earth-Consortium +MODEL = EC-Earth3-HR +NAME = historical # Model version: Available versions -MODEL_VERSION = +MODEL_VERSION =Ec3.2_O25L75 # Atmospheric output timestep in hours -ATMOS_TIMESTEP = 6 +ATMOS_TIMESTEP = 3 # Ocean output timestep in hours -OCEAN_TIMESTEP = 6 +OCEAN_TIMESTEP = 3 # For those who use Autosubmit, this will be easy # EXPID is the unique identifier of the experiment. @@ -87,16 +91,18 @@ OCEAN_TIMESTEP = 6 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = testing_erainterim -STARTDATES = 20000101 -#{20050101,20161231,D} -MEMBERS = 0 +EXPID = a0n8 +STARTDATES = 19900101 +MEMBERS = fc0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 CHUNKS = 1 -CALENDAR = 366 # CHUNKS = 1 +[REPORT] +MAXIMUM_PRIORITY = 7 +PATH = $HOME/reports/a0n8 + # This ALIAS section is a bit different # Inside this, you can provide alias for frequent diagnostics calls. @@ -110,17 +116,17 @@ STC = mocarea,0,25,0,200,Pac mocarea,-25,0,0,200,Pac mocarea,0,25,0,200,Atl moca HEAT_SAL_MXL = mlotstsc mlotsthc LMSALC = vertmeanmeters,so,300,5400 USALC = vertmeanmeters,so,0,300 -OHC = ohc,glob,0,1,10 +OHC = ohc,glob,0,0,2000 XOHC = ohc,glob,1,0,0 -LOHC = ohc,glob,0,23,46 -MOHC = ohc,glob,0,18,22 -UOHC = ohc,glob,0,1,17 +LOHC = ohc,glob,0,700,2000 +MOHC = ohc,glob,0,300,700 +UOHC = ohc,glob,0,0,300 OHC_SPECIFIED_LAYER = ohclayer,0,300 ohclayer,300,800 3DTEMP = interp,thetao 3DSAL = interp,so -TSEC_AVE190-220E =avgsection,ocean,thetao,190,220,-90,90,regular -SSEC_AVE190-220E =avgsection,ocean,so,190,220,-90,90,regular -VERT_SSECTIONS = cutsection,so,Z,0 cutsection,so,Z,45 cutsection,so,Z,-45 cutsection,so,M,-30 cutsection,so,M,80 +TSEC_AVE190-220E =avgsection,thetao,190,220,-90,90 +SSEC_AVE190-220E =avgsection,so,190,220,-90,90 +VERT_SSECTIONS = cutsection,so,Z,0 cutsection,so,Z,45 cutsection,so,Z,-45 cutsection,so,M,-30 cutsection,so,M,180 cutsection,so,M,80 VERT_TSECTIONS = cutsection,thetao,Z,0 cutsection,thetao,Z,45 cutsection,thetao,Z,-45 cutsection,thetao,M,-30 cutsection,thetao,M,180 cutsection,thetao,M,80 SIASIESIV = siasiesiv,glob diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index d4b91083..ac541224 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -366,6 +366,7 @@ class EarthDiags(object): Utils.give_group_write_permissions(new_maskglo_scratch_path) self._link_file(new_maskglo_scratch_path, 'new_maskglo.nc') mask_regions_scratch_path = os.path.join(self.config.scratch_masks, mask_regions) + return if self._copy_file(mask_regions_path, mask_regions_scratch_path, restore_meshes): Utils.give_group_write_permissions(mask_regions_scratch_path) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 6e4496bb..8139803b 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -1,14 +1,15 @@ # coding=utf-8 """Diagnostic to compute regional averages""" -import os +import iris +import iris.exceptions -from earthdiagnostics import cdftools from earthdiagnostics.box import Box from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticIntOption, DiagnosticDomainOption, \ DiagnosticBoolOption, DiagnosticBasinOption, DiagnosticVariableOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile +from bscearth.utils.log import Log class RegionMean(Diagnostic): @@ -39,20 +40,20 @@ class RegionMean(Diagnostic): alias = 'regmean' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid_point, box, save3d, basin, - variance, grid): + def __init__(self, data_manager, startdate, member, chunk, domain, variable, box, save3d, weights_file, + variance, basin): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member self.chunk = chunk self.domain = domain self.variable = variable - self.grid_point = grid_point.upper() self.box = box self.save3d = save3d - self.basin = basin + self.weights_file = weights_file self.variance = variance - self.grid = grid + self.basin = basin + self.declared = {} self.lat_name = 'lat' @@ -64,8 +65,10 @@ class RegionMean(Diagnostic): def __str__(self): return 'Region mean Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Variable: {0.variable} ' \ - 'Grid point: {0.grid_point} Box: {0.box} Save 3D: {0.save3d} Save variance: {0.variance} ' \ - 'Original grid: {0.grid} Basin: {0.basin}'.format(self) + 'Box: {0.box} Save 3D: {0.save3d} Save variance: {0.variance}'.format(self) + + def __hash__(self): + return hash(str(self)) @classmethod def generate_jobs(cls, diags, options): @@ -93,17 +96,22 @@ class RegionMean(Diagnostic): box.min_depth = options['min_depth'] box.max_depth = options['max_depth'] + weights_file = TempFile.get() + weight_diagnostics = ComputeWeights(diags.data_manager, options['grid_point'], options['basin'], weights_file) + job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(RegionMean(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['grid_point'], box, - options['save3D'], options['basin'], options['variance'], options['grid'])) + job = RegionMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], box, + options['save3D'], weights_file, options['variance'], options['basin']) + job.add_subjob(weight_diagnostics) + job_list.append(job) + return job_list def request_data(self): """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk) def declare_data_generated(self): """Declare data to be generated by the diagnostic""" @@ -122,61 +130,10 @@ class RegionMean(Diagnostic): def compute(self): """Run the diagnostic""" - mean_file = TempFile.get() - - variable_file = self.variable_file.local_file - - handler = Utils.open_cdf(variable_file) - self.save3d &= 'lev' in handler.dimensions - if "latitude" in handler.variables: - self.lat_name = 'latitude' - if "longitude" in handler.variables: - self.lon_name = 'longitude' - handler.close() - - cdfmean_options = [self.variable, self.grid_point, 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] - if self.variance: - cdfmean_options += ['-var'] - if self.basin != Basins().Global: - cdfmean_options.append('-M') - cdfmean_options.append('mask_regions.3d.nc') - cdfmean_options.append(self.basin.name) - - cdftools.run('cdfmean', input_file=variable_file, output_file=mean_file, options=cdfmean_options) - Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) + data = iris.load_cube(self.variable_file, self.variable) + weights = iris.load_cube(self.weights_file, 'weights') + mean = data.collapsed(['latitude', 'longitude'], iris.analysis.MEAN, weights=weights) - self._send_var('mean', False, mean_file) - self._send_var('mean', True, mean_file) - - if self.variance: - self._send_var('var', False, mean_file) - self._send_var('var', True, mean_file) - - os.remove(mean_file) - - def _send_var(self, var, threed, mean_file): - if threed: - if not self.save3d: - return False - original_name = '{0}_{1}'.format(var, self.variable) - final_name = '{1}3d{0}'.format(var, self.variable) - levels = ',lev' - else: - original_name = '{0}_3D{1}'.format(var, self.variable) - final_name = '{1}{0}'.format(var, self.variable) - levels = '' - - temp2 = TempFile.get() - Utils.nco.ncks(input=mean_file, output=temp2, - options=('-v {0},{2.lat_name},{2.lon_name}{1}'.format(original_name, levels, self),)) - handler = Utils.open_cdf(temp2) - var_handler = handler.variables[original_name] - if hasattr(var_handler, 'valid_min'): - del var_handler.valid_min - if hasattr(var_handler, 'valid_max'): - del var_handler.valid_max - handler.close() - self.declared[final_name].set_local_file(temp2, diagnostic=self, rename_var=original_name, region=self.basin) def _declare_var(self, var, threed, box_save): if threed: @@ -187,4 +144,59 @@ class RegionMean(Diagnostic): final_name = '{1}{0}'.format(var, self.variable) self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, - self.chunk, box=box_save, region=self.basin, grid=self.grid) + self.chunk, box=box_save, region=self.basin) + + +class ComputeWeights(Diagnostic): + """ + Diagnostic used to compute regional mean and sum weights + + Parameters + ---------- + data_manager: DataManager + startdate: str + member: int + chunk: int + weights_file: str + """ + + alias = 'computeinterpcdoweights' + "Diagnostic alias for the configuration file" + + @classmethod + def generate_jobs(cls, diags, options): + pass + + def __init__(self, data_manager, grid_point, basin, weights_file): + Diagnostic.__init__(self, data_manager) + self.weights_file = weights_file + self.basin = basin + self.grid_point = grid_point.lower() + + def __str__(self): + return 'Computing weights for region averaging: Point {0.grid_point} Basin: {0.basin}'.format(self) + + def __hash__(self): + return hash(str(self)) + + def compute(self): + """Compute weights""" + mask = Utils.get_mask(self.basin) + e1 = iris.load_cube('mesh_hgr.nc', 'e1{0}'.format(self.grid_point)) + e2 = iris.load_cube('mesh_hgr.nc', 'e2{0}'.format(self.grid_point)) + try: + e3 = iris.load_cube('mesh_hgr.nc', 'e3{0}'.format(self.grid_point)) + except iris.exceptions.ConstraintMismatchError: + e3 = iris.load_cube('mesh_hgr.nc', 'e3{0}_0'.format(self.grid_point)) + weights = e3 * e1 * e2 * mask + weights.var_name = 'weights' + Log.info(str(weights)) + iris.save(weights, self.weights_file) + + def request_data(self): + """Request data required by the diagnostic""" + None + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + pass -- GitLab From 221f099da3498d4698eb6de8171399a5d20f172e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 29 Jan 2018 16:43:42 +0100 Subject: [PATCH 157/168] Region mean now working --- earthdiagnostics/cmormanager.py | 3 + earthdiagnostics/earthdiags.py | 1 - earthdiagnostics/ocean/regionmean.py | 112 ++++++++++++++++++++++----- earthdiagnostics/work_manager.py | 3 +- 4 files changed, 99 insertions(+), 20 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index ff69e385..8fb5eade 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -123,6 +123,8 @@ class CMORManager(DataManager): DataFile """ + if frequency is None: + frequency = self.config.frequency cmor_var = self.variable_list.get_variable(var) var = self._get_final_var_name(box, var) filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) @@ -259,6 +261,7 @@ class CMORManager(DataManager): If you provide two or more parameters from chunk, year or date_str or none at all """ + frequency = Frequency.parse(frequency) options = sum(x is not None for x in (chunk, year, date_str)) if options == 0: raise ValueError('You must provide chunk, year or date_str') diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index ac541224..d4b91083 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -366,7 +366,6 @@ class EarthDiags(object): Utils.give_group_write_permissions(new_maskglo_scratch_path) self._link_file(new_maskglo_scratch_path, 'new_maskglo.nc') mask_regions_scratch_path = os.path.join(self.config.scratch_masks, mask_regions) - return if self._copy_file(mask_regions_path, mask_regions_scratch_path, restore_meshes): Utils.give_group_write_permissions(mask_regions_scratch_path) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 8139803b..a4ed524b 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -1,8 +1,12 @@ # coding=utf-8 """Diagnostic to compute regional averages""" import iris +import iris.util +import iris.analysis import iris.exceptions +import numpy as np + from earthdiagnostics.box import Box from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticIntOption, DiagnosticDomainOption, \ @@ -60,6 +64,8 @@ class RegionMean(Diagnostic): self.lon_name = 'lon' def __eq__(self, other): + if type(self) is not type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable @@ -130,22 +136,74 @@ class RegionMean(Diagnostic): def compute(self): """Run the diagnostic""" - data = iris.load_cube(self.variable_file, self.variable) - weights = iris.load_cube(self.weights_file, 'weights') - mean = data.collapsed(['latitude', 'longitude'], iris.analysis.MEAN, weights=weights) + handler = Utils.open_cdf(self.variable_file.local_file) + var = handler.variables[self.variable] + coordinates = '' + has_levels = False + for dimension in handler.variables.keys(): + if dimension in ['time', 'lev', 'lat', 'latitude', 'lon', 'longitude']: + coordinates += ' {0}'.format(dimension) + if dimension == 'lev': + has_levels = True + var.coordinates = coordinates + handler.close() + + data = iris.load_cube(self.variable_file.local_file, iris.AttributeConstraint(short_name=self.variable)) + Log.info(str(data)) + weights = iris.load_cube(self.weights_file, 'weights').data + if has_levels: + mean = iris.cube.CubeList() + mean3d = iris.cube.CubeList() + var = iris.cube.CubeList() + var3d = iris.cube.CubeList() + for time_slice in data.slices_over('time'): + mean.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], + iris.analysis.MEAN, weights=weights)) + mean3d.append(time_slice.collapsed(['latitude', 'longitude'], + iris.analysis.MEAN, weights=weights)) + # var.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], + # iris.analysis.VARIANCE, weights=weights)) + # var3d.append(time_slice.collapsed(['latitude', 'longitude'], + # iris.analysis.VARIANCE, weights=weights)) + self._send_var('mean', True, mean3d.merge_cube()) + self._send_var('mean', False, mean.merge_cube()) + if self.variance: + self._send_var('var', True, var3d.merge_cube()) + + self._send_var('var', False, var.merge_cube()) + else: + mean = iris.cube.CubeList() + var = iris.cube.CubeList() + for time_slice in data.slices_over('time'): + mean.append(time_slice.collapsed(['latitude', 'longitude'], iris.analysis.MEAN, weights=weights)) + var.append(time_slice.collapsed(['latitude', 'longitude'], iris.analysis.VARIANCE, weights=weights)) + + self._send_var('mean', False, mean.merge_cube()) + if self.variance: + self._send_var('var', False, var.merge_cube()) def _declare_var(self, var, threed, box_save): if threed: if not self.save3d: return False - final_name = '{1}3d{0}'.format(var, self.variable) + final_name = '{1}3d{0}iris'.format(var, self.variable) else: - final_name = '{1}{0}'.format(var, self.variable) + final_name = '{1}{0}iris'.format(var, self.variable) self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, box=box_save, region=self.basin) + def _send_var(self, var, threed, cube): + if threed: + final_name = '{1}3d{0}iris'.format(var, self.variable) + else: + final_name = '{1}{0}iris'.format(var, self.variable) + cube.var_name = 'result' + temp = TempFile.get() + iris.save(cube, temp) + self.declared[final_name].set_local_file(temp, diagnostic=self, rename_var='result', region=self.basin) + class ComputeWeights(Diagnostic): """ @@ -154,9 +212,8 @@ class ComputeWeights(Diagnostic): Parameters ---------- data_manager: DataManager - startdate: str - member: int - chunk: int + grid_point: str + basin: int weights_file: str """ @@ -173,6 +230,12 @@ class ComputeWeights(Diagnostic): self.basin = basin self.grid_point = grid_point.lower() + def __eq__(self, other): + if type(self) is not type(other): + return False + return self.weights_file == other.weights_file and self.basin == other.basin and \ + self.grid_point == other.grid_point + def __str__(self): return 'Computing weights for region averaging: Point {0.grid_point} Basin: {0.basin}'.format(self) @@ -181,21 +244,34 @@ class ComputeWeights(Diagnostic): def compute(self): """Compute weights""" - mask = Utils.get_mask(self.basin) - e1 = iris.load_cube('mesh_hgr.nc', 'e1{0}'.format(self.grid_point)) - e2 = iris.load_cube('mesh_hgr.nc', 'e2{0}'.format(self.grid_point)) - try: - e3 = iris.load_cube('mesh_hgr.nc', 'e3{0}'.format(self.grid_point)) - except iris.exceptions.ConstraintMismatchError: - e3 = iris.load_cube('mesh_hgr.nc', 'e3{0}_0'.format(self.grid_point)) - weights = e3 * e1 * e2 * mask + iris.FUTURE.netcdf_promote = True + iris.FUTURE.netcdf_no_unlimited = True + + + mask = np.squeeze(Utils.get_mask(self.basin)) + e1 = self.try_load_cube(1) + e2 = self.try_load_cube(2) + e3 = self.try_load_cube(3) + mask = e1 * e2 * mask + iris.util.demote_dim_coord_to_aux_coord(e3, 'lev') + for coord in e3.coords(): + e3.remove_coord(coord) + for coord in mask.coords(): + mask.remove_coord(coord) + weights = e3 * mask weights.var_name = 'weights' - Log.info(str(weights)) iris.save(weights, self.weights_file) + def try_load_cube(self, number): + try: + cube = iris.load_cube('mesh_hgr.nc', 'e{0}{1}'.format(number, self.grid_point)) + except iris.exceptions.ConstraintMismatchError: + cube = iris.load_cube('mesh_hgr.nc', 'e{0}{1}_0'.format(number, self.grid_point)) + return iris.util.squeeze(cube) + def request_data(self): """Request data required by the diagnostic""" - None + pass def declare_data_generated(self): """Declare data to be generated by the diagnostic""" diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 5fb8eb35..385a5525 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -48,7 +48,8 @@ class WorkManager(object): for job in diag_class.generate_jobs(self, diag_options): list_jobs.append(job) for subjob in job.subjobs: - list_jobs.append(subjob) + if subjob not in list_jobs: + list_jobs.append(subjob) continue except DiagnosticOptionError as ex: Log.error('Can not configure diagnostic {0}: {1}', diag_options[0], ex) -- GitLab From 632a3b29e9a2596114bec29a1493d4711744e586 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 29 Jan 2018 16:53:39 +0100 Subject: [PATCH 158/168] Fixed bugs --- earthdiagnostics/cmormanager.py | 2 ++ earthdiagnostics/ocean/interpolate.py | 3 +++ earthdiagnostics/utils.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index ff69e385..273bdc6a 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -266,6 +266,8 @@ class CMORManager(DataManager): raise ValueError('You must provide only one parameter in chunk, year or date_str') if frequency is None: frequency = self.config.frequency + else: + frequency = Frequency.parse(frequency) folder_path = self._get_full_cmor_folder_path(startdate, member, domain, var, frequency, grid, cmor_var) file_name = self._get_cmor_file_name(startdate, member, domain, var, cmor_var, frequency, diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index acb114e6..56f47844 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -74,6 +74,9 @@ class Interpolate(Diagnostic): self.variable, self.grid, self.invert_latitude, self.model_version, self.original_grid) + def __hash__(self): + return hash(str(self)) + @classmethod def generate_jobs(cls, diags, options): """ diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 5b343d38..29a5d0ed 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -211,7 +211,7 @@ class Utils(object): cubes = iris.load(filepath) if len(cubes) == 0: return False - except (iris.exceptions.IrisError, RuntimeError) as ex: + except (iris.exceptions.IrisError, RuntimeError, OSError) as ex: Log.debug('netCDF checks failed: {0}', ex) return False return True -- GitLab From cb2e02158a7690fe0871d2f8bd78986a919805d5 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 31 Jan 2018 13:01:48 +0100 Subject: [PATCH 159/168] Regmean now only loads the minimum required data --- earthdiagnostics/diagnostic.py | 2 +- earthdiagnostics/earthdiags.py | 2 +- earthdiagnostics/ocean/regionmean.py | 104 ++++++++++++++++++++------- launch_diags.sh | 8 +-- 4 files changed, 86 insertions(+), 30 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 34125249..a145a01a 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -492,7 +492,7 @@ class DiagnosticIntOption(DiagnosticOption): Parameters ---------- name: str - default_value: list, optional + default_value: int, optional min_limit: int, optional If setted, any value below this will not be accepted max_limit: int, optional diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index d4b91083..0af2c75c 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -159,7 +159,7 @@ class EarthDiags(object): self._prepare_scratch_dir() - self._prepare_mesh_files() + # self._prepare_mesh_files() self._initialize_basins() self._prepare_data_manager() diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index a4ed524b..11fa4051 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -2,6 +2,7 @@ """Diagnostic to compute regional averages""" import iris import iris.util +import iris.coords import iris.analysis import iris.exceptions @@ -91,8 +92,8 @@ class RegionMean(Diagnostic): DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticOption('grid_point', 'T'), DiagnosticBasinOption('basin', Basins().Global), - DiagnosticIntOption('min_depth', 0), - DiagnosticIntOption('max_depth', 0), + DiagnosticIntOption('min_depth', -1), + DiagnosticIntOption('max_depth', -1), DiagnosticBoolOption('save3D', True), DiagnosticBoolOption('variance', False), DiagnosticOption('grid', '')) @@ -136,21 +137,50 @@ class RegionMean(Diagnostic): def compute(self): """Run the diagnostic""" + iris.FUTURE.netcdf_promote = True + iris.FUTURE.netcdf_no_unlimited = True + handler = Utils.open_cdf(self.variable_file.local_file) var = handler.variables[self.variable] coordinates = '' has_levels = False for dimension in handler.variables.keys(): - if dimension in ['time', 'lev', 'lat', 'latitude', 'lon', 'longitude']: + if dimension in ['time', 'lev', 'lat', 'latitude', 'lon', 'longitude', 'i', 'j']: coordinates += ' {0}'.format(dimension) if dimension == 'lev': has_levels = True var.coordinates = coordinates handler.close() - data = iris.load_cube(self.variable_file.local_file, iris.AttributeConstraint(short_name=self.variable)) - Log.info(str(data)) + def add_i_j(cube, field, filename): + if cube.var_name != self.variable: + return + if not cube.coords('i'): + index = field.dimensions.index('i') + i = np.arange(1, field.shape[index]+1) + i_coord = iris.coords.DimCoord(i, var_name='i') + cube.add_dim_coord(i_coord, index) + if not cube.coords('j'): + index = field.dimensions.index('j') + i = np.arange(1, field.shape[index] + 1) + i_coord = iris.coords.DimCoord(i, var_name='j') + cube.add_dim_coord(i_coord, index) + + data = iris.load_cube(self.variable_file.local_file, + iris.AttributeConstraint(short_name=self.variable), + callback=add_i_j) + weights = iris.load_cube(self.weights_file, 'weights').data + i_indexes = iris.load_cube(self.weights_file, 'i_indexes').data + j_indexes = iris.load_cube(self.weights_file, 'j_indexes').data + + def selected_i(cell): + return cell.point - 1 in i_indexes + + def selected_j(cell): + return cell.point - 1 in j_indexes + + data = data.extract(iris.Constraint(i=selected_i, j=selected_j)) if has_levels: mean = iris.cube.CubeList() mean3d = iris.cube.CubeList() @@ -159,18 +189,21 @@ class RegionMean(Diagnostic): for time_slice in data.slices_over('time'): mean.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], iris.analysis.MEAN, weights=weights)) - mean3d.append(time_slice.collapsed(['latitude', 'longitude'], - iris.analysis.MEAN, weights=weights)) - # var.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], - # iris.analysis.VARIANCE, weights=weights)) - # var3d.append(time_slice.collapsed(['latitude', 'longitude'], - # iris.analysis.VARIANCE, weights=weights)) - self._send_var('mean', True, mean3d.merge_cube()) - self._send_var('mean', False, mean.merge_cube()) + if self.save3d: + mean3d.append(time_slice.collapsed(['latitude', 'longitude'], + iris.analysis.MEAN, weights=weights)) + if self.variance: + var.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], + iris.analysis.VARIANCE, weights=weights)) + if self.save3d: + var3d.append(time_slice.collapsed(['latitude', 'longitude'], + iris.analysis.VARIANCE, weights=weights)) + self._send_var('mean', True, mean3d) + self._send_var('mean', False, mean) if self.variance: - self._send_var('var', True, var3d.merge_cube()) + self._send_var('var', True, var3d) - self._send_var('var', False, var.merge_cube()) + self._send_var('var', False, var) else: mean = iris.cube.CubeList() var = iris.cube.CubeList() @@ -194,12 +227,19 @@ class RegionMean(Diagnostic): self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, box=box_save, region=self.basin) - def _send_var(self, var, threed, cube): + def _send_var(self, var, threed, cube_list): if threed: + if not self.save3d and threed: + return False final_name = '{1}3d{0}iris'.format(var, self.variable) else: final_name = '{1}{0}iris'.format(var, self.variable) + cube = cube_list.merge_cube() cube.var_name = 'result' + cube.remove_coord('latitude') + cube.remove_coord('longitude') + cube.remove_coord('depth') + Log.result(str(cube.data)) temp = TempFile.get() iris.save(cube, temp) self.declared[final_name].set_local_file(temp, diagnostic=self, rename_var='result', region=self.basin) @@ -247,20 +287,36 @@ class ComputeWeights(Diagnostic): iris.FUTURE.netcdf_promote = True iris.FUTURE.netcdf_no_unlimited = True - mask = np.squeeze(Utils.get_mask(self.basin)) + i_indexes = np.where(np.any(mask != 0, 0))[0] + j_indexes = np.where(np.any(mask != 0, 1))[0] + mask_small = np.take(np.take(mask, i_indexes, 1), j_indexes, 0) + e1 = self.try_load_cube(1) e2 = self.try_load_cube(2) e3 = self.try_load_cube(3) - mask = e1 * e2 * mask + + def selected_i(cell): + return cell.point - 1 in i_indexes + + def selected_j(cell): + return cell.point - 1 in j_indexes + + e1_small = e1.extract(iris.Constraint(i=selected_i, j=selected_j)) + e2_small = e2.extract(iris.Constraint(i=selected_i, j=selected_j)) + e3_small = e3.extract(iris.Constraint(i=selected_i, j=selected_j)) + + mask_small = e1_small * e2_small * mask_small iris.util.demote_dim_coord_to_aux_coord(e3, 'lev') - for coord in e3.coords(): - e3.remove_coord(coord) - for coord in mask.coords(): - mask.remove_coord(coord) - weights = e3 * mask + for coord in e3_small.coords(): + e3_small.remove_coord(coord) + for coord in mask_small.coords(): + mask_small.remove_coord(coord) + weights = e3_small * mask_small weights.var_name = 'weights' - iris.save(weights, self.weights_file) + i_indexes = iris.cube.Cube(i_indexes, var_name='i_indexes') + j_indexes = iris.cube.Cube(j_indexes, var_name='j_indexes') + iris.save((weights, i_indexes, j_indexes), self.weights_file) def try_load_cube(self, number): try: diff --git a/launch_diags.sh b/launch_diags.sh index 1c87fec2..59dd9e14 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash -#SBATCH -n 4 +#SBATCH -n 1 #SBATCH --time 7-00:00:00 #SBATCH --error=job.%J.err #SBATCH --output=job.%J.out -PATH_TO_CONF_FILE=~jvegas/earthdiagnostics/diags.conf -PATH_TO_DIAGNOSTICS=~jvegas/earthdiagnostics -PATH_TO_CONDAENV=diags +PATH_TO_CONF_FILE=~jvegas/earthdiags_confs/diags_roberto.conf +PATH_TO_DIAGNOSTICS=~jvegas/PycharmProjects/earthdiagnostics +PATH_TO_CONDAENV=/home/Earth/jvegas/.conda/envs/earthdiagnostics3/ module purge module load CDFTOOLS/3.0a8-foss-2015a -- GitLab From 5abaa67d9a901c32d74dba4d7cee21b1b8962cba Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 31 Jan 2018 15:34:13 +0100 Subject: [PATCH 160/168] Added level support --- earthdiagnostics/ocean/regionmean.py | 46 ++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 11fa4051..d4d40c49 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -104,7 +104,8 @@ class RegionMean(Diagnostic): box.max_depth = options['max_depth'] weights_file = TempFile.get() - weight_diagnostics = ComputeWeights(diags.data_manager, options['grid_point'], options['basin'], weights_file) + weight_diagnostics = ComputeWeights(diags.data_manager, options['grid_point'], options['basin'], box, + weights_file) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): @@ -165,6 +166,11 @@ class RegionMean(Diagnostic): i = np.arange(1, field.shape[index] + 1) i_coord = iris.coords.DimCoord(i, var_name='j') cube.add_dim_coord(i_coord, index) + if not cube.coords('lev'): + index = field.dimensions.index('lev') + i = np.arange(1, field.shape[index] + 1) + lev = iris.coords.AuxCoord(i, var_name='lev') + cube.add_aux_coord(lev, index) data = iris.load_cube(self.variable_file.local_file, iris.AttributeConstraint(short_name=self.variable), @@ -173,6 +179,7 @@ class RegionMean(Diagnostic): weights = iris.load_cube(self.weights_file, 'weights').data i_indexes = iris.load_cube(self.weights_file, 'i_indexes').data j_indexes = iris.load_cube(self.weights_file, 'j_indexes').data + lev_limits = iris.load_cube(self.weights_file, 'lev_limits').data def selected_i(cell): return cell.point - 1 in i_indexes @@ -180,7 +187,10 @@ class RegionMean(Diagnostic): def selected_j(cell): return cell.point - 1 in j_indexes - data = data.extract(iris.Constraint(i=selected_i, j=selected_j)) + def selected_level(cell): + return lev_limits[0] <= cell.point <= lev_limits[1] + + data = data.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) if has_levels: mean = iris.cube.CubeList() mean3d = iris.cube.CubeList() @@ -239,7 +249,7 @@ class RegionMean(Diagnostic): cube.remove_coord('latitude') cube.remove_coord('longitude') cube.remove_coord('depth') - Log.result(str(cube.data)) + cube.remove_coord('lev') temp = TempFile.get() iris.save(cube, temp) self.declared[final_name].set_local_file(temp, diagnostic=self, rename_var='result', region=self.basin) @@ -264,20 +274,22 @@ class ComputeWeights(Diagnostic): def generate_jobs(cls, diags, options): pass - def __init__(self, data_manager, grid_point, basin, weights_file): + def __init__(self, data_manager, grid_point, basin, box, weights_file): Diagnostic.__init__(self, data_manager) self.weights_file = weights_file self.basin = basin self.grid_point = grid_point.lower() + self.box = box def __eq__(self, other): if type(self) is not type(other): return False return self.weights_file == other.weights_file and self.basin == other.basin and \ - self.grid_point == other.grid_point + self.grid_point == other.grid_point and self.box != other.box def __str__(self): - return 'Computing weights for region averaging: Point {0.grid_point} Basin: {0.basin}'.format(self) + return 'Computing weights for region averaging: Point {0.grid_point} Basin: {0.basin} Box: {0.box}'\ + .format(self) def __hash__(self): return hash(str(self)) @@ -295,6 +307,18 @@ class ComputeWeights(Diagnostic): e1 = self.try_load_cube(1) e2 = self.try_load_cube(2) e3 = self.try_load_cube(3) + depth = iris.util.squeeze(iris.load_cube('mesh_hgr.nc', 'gdept_0')) + if self.box.min_depth == -1: + min_level = 0 + else: + distance = abs((depth - self.box.min_depth).data) + min_level = np.argmin(distance) + + if self.box.max_depth == -1: + max_level = depth.shape[0] + else: + distance = abs((depth - self.box.max_depth).data) + max_level = np.argmin(distance) def selected_i(cell): return cell.point - 1 in i_indexes @@ -302,12 +326,14 @@ class ComputeWeights(Diagnostic): def selected_j(cell): return cell.point - 1 in j_indexes + def selected_level(cell): + return min_level <= cell.point <= max_level + e1_small = e1.extract(iris.Constraint(i=selected_i, j=selected_j)) e2_small = e2.extract(iris.Constraint(i=selected_i, j=selected_j)) - e3_small = e3.extract(iris.Constraint(i=selected_i, j=selected_j)) + e3_small = e3.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) mask_small = e1_small * e2_small * mask_small - iris.util.demote_dim_coord_to_aux_coord(e3, 'lev') for coord in e3_small.coords(): e3_small.remove_coord(coord) for coord in mask_small.coords(): @@ -316,7 +342,8 @@ class ComputeWeights(Diagnostic): weights.var_name = 'weights' i_indexes = iris.cube.Cube(i_indexes, var_name='i_indexes') j_indexes = iris.cube.Cube(j_indexes, var_name='j_indexes') - iris.save((weights, i_indexes, j_indexes), self.weights_file) + lev_limits = iris.cube.Cube([min_level, max_level], var_name='lev_limits') + iris.save((weights, i_indexes, j_indexes, lev_limits), self.weights_file) def try_load_cube(self, number): try: @@ -325,6 +352,7 @@ class ComputeWeights(Diagnostic): cube = iris.load_cube('mesh_hgr.nc', 'e{0}{1}_0'.format(number, self.grid_point)) return iris.util.squeeze(cube) + def request_data(self): """Request data required by the diagnostic""" pass -- GitLab From 58a043504124e65e461986c6a80051a7d6e0670f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 31 Jan 2018 17:33:13 +0100 Subject: [PATCH 161/168] Fixed prospector warnings --- earthdiagnostics/earthdiags.py | 7 +- earthdiagnostics/ocean/regionmean.py | 143 ++++++++++++++------------- earthdiagnostics/ocean/rotation.py | 1 - launch_diags.sh | 4 +- 4 files changed, 83 insertions(+), 72 deletions(-) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 0af2c75c..23107feb 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -159,7 +159,7 @@ class EarthDiags(object): self._prepare_scratch_dir() - # self._prepare_mesh_files() + self._prepare_mesh_files() self._initialize_basins() self._prepare_data_manager() @@ -383,8 +383,9 @@ class EarthDiags(object): if not force and os.path.exists(destiny): # Small size differences can be due to the renaming of variables - delta_size = abs(os.stat(source).st_size - os.stat(destiny).st_size) - if delta_size < 512: + reference_size = os.stat(source).st_size + delta_size = abs(reference_size - os.stat(destiny).st_size) + if delta_size < 2048 or delta_size / reference_size < 1/1000: Log.info('File {0} already exists', destiny) return True diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index d4d40c49..1755adda 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -14,7 +14,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic DiagnosticBoolOption, DiagnosticBasinOption, DiagnosticVariableOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile -from bscearth.utils.log import Log class RegionMean(Diagnostic): @@ -141,24 +140,71 @@ class RegionMean(Diagnostic): iris.FUTURE.netcdf_promote = True iris.FUTURE.netcdf_no_unlimited = True - handler = Utils.open_cdf(self.variable_file.local_file) - var = handler.variables[self.variable] - coordinates = '' - has_levels = False - for dimension in handler.variables.keys(): - if dimension in ['time', 'lev', 'lat', 'latitude', 'lon', 'longitude', 'i', 'j']: - coordinates += ' {0}'.format(dimension) - if dimension == 'lev': - has_levels = True - var.coordinates = coordinates - handler.close() + has_levels = self._fix_file_metadata() + + data = self._load_data() + + weights = iris.load_cube(self.weights_file, 'weights').data + i_indexes = iris.load_cube(self.weights_file, 'i_indexes').data + j_indexes = iris.load_cube(self.weights_file, 'j_indexes').data + lev_limits = iris.load_cube(self.weights_file, 'lev_limits').data + + def selected_i(cell): + return cell.point - 1 in i_indexes + + def selected_j(cell): + return cell.point - 1 in j_indexes + + def selected_level(cell): + return lev_limits[0] <= cell.point <= lev_limits[1] + + data = data.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) + if has_levels: + self._meand_3d_variable(data, weights) + else: + self._mean_2d_var(data, weights) + + def _mean_2d_var(self, data, weights): + mean = iris.cube.CubeList() + var = iris.cube.CubeList() + for time_slice in data.slices_over('time'): + mean.append(time_slice.collapsed(['latitude', 'longitude'], iris.analysis.MEAN, weights=weights)) + var.append(time_slice.collapsed(['latitude', 'longitude'], iris.analysis.VARIANCE, weights=weights)) + self._send_var('mean', False, mean.merge_cube()) + if self.variance: + self._send_var('var', False, var.merge_cube()) + + def _meand_3d_variable(self, data, weights): + mean = iris.cube.CubeList() + mean3d = iris.cube.CubeList() + var = iris.cube.CubeList() + var3d = iris.cube.CubeList() + for time_slice in data.slices_over('time'): + mean.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], + iris.analysis.MEAN, weights=weights)) + if self.save3d: + mean3d.append(time_slice.collapsed(['latitude', 'longitude'], + iris.analysis.MEAN, weights=weights)) + if self.variance: + var.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], + iris.analysis.VARIANCE, weights=weights)) + if self.save3d: + var3d.append(time_slice.collapsed(['latitude', 'longitude'], + iris.analysis.VARIANCE, weights=weights)) + self._send_var('mean', True, mean3d) + self._send_var('mean', False, mean) + if self.variance: + self._send_var('var', True, var3d) + + self._send_var('var', False, var) + def _load_data(self): def add_i_j(cube, field, filename): if cube.var_name != self.variable: return if not cube.coords('i'): index = field.dimensions.index('i') - i = np.arange(1, field.shape[index]+1) + i = np.arange(1, field.shape[index] + 1) i_coord = iris.coords.DimCoord(i, var_name='i') cube.add_dim_coord(i_coord, index) if not cube.coords('j'): @@ -175,64 +221,29 @@ class RegionMean(Diagnostic): data = iris.load_cube(self.variable_file.local_file, iris.AttributeConstraint(short_name=self.variable), callback=add_i_j) + return data - weights = iris.load_cube(self.weights_file, 'weights').data - i_indexes = iris.load_cube(self.weights_file, 'i_indexes').data - j_indexes = iris.load_cube(self.weights_file, 'j_indexes').data - lev_limits = iris.load_cube(self.weights_file, 'lev_limits').data - - def selected_i(cell): - return cell.point - 1 in i_indexes - - def selected_j(cell): - return cell.point - 1 in j_indexes - - def selected_level(cell): - return lev_limits[0] <= cell.point <= lev_limits[1] - - data = data.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) - if has_levels: - mean = iris.cube.CubeList() - mean3d = iris.cube.CubeList() - var = iris.cube.CubeList() - var3d = iris.cube.CubeList() - for time_slice in data.slices_over('time'): - mean.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], - iris.analysis.MEAN, weights=weights)) - if self.save3d: - mean3d.append(time_slice.collapsed(['latitude', 'longitude'], - iris.analysis.MEAN, weights=weights)) - if self.variance: - var.append(time_slice.collapsed(['latitude', 'longitude', 'depth'], - iris.analysis.VARIANCE, weights=weights)) - if self.save3d: - var3d.append(time_slice.collapsed(['latitude', 'longitude'], - iris.analysis.VARIANCE, weights=weights)) - self._send_var('mean', True, mean3d) - self._send_var('mean', False, mean) - if self.variance: - self._send_var('var', True, var3d) - - self._send_var('var', False, var) - else: - mean = iris.cube.CubeList() - var = iris.cube.CubeList() - for time_slice in data.slices_over('time'): - mean.append(time_slice.collapsed(['latitude', 'longitude'], iris.analysis.MEAN, weights=weights)) - var.append(time_slice.collapsed(['latitude', 'longitude'], iris.analysis.VARIANCE, weights=weights)) - - self._send_var('mean', False, mean.merge_cube()) - if self.variance: - self._send_var('var', False, var.merge_cube()) - + def _fix_file_metadata(self): + handler = Utils.open_cdf(self.variable_file.local_file) + var = handler.variables[self.variable] + coordinates = '' + has_levels = False + for dimension in handler.variables.keys(): + if dimension in ['time', 'lev', 'lat', 'latitude', 'lon', 'longitude', 'i', 'j']: + coordinates += ' {0}'.format(dimension) + if dimension == 'lev': + has_levels = True + var.coordinates = coordinates + handler.close() + return has_levels def _declare_var(self, var, threed, box_save): if threed: if not self.save3d: return False - final_name = '{1}3d{0}iris'.format(var, self.variable) + final_name = '{1}3d{0}'.format(var, self.variable) else: - final_name = '{1}{0}iris'.format(var, self.variable) + final_name = '{1}{0}'.format(var, self.variable) self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, box=box_save, region=self.basin) @@ -241,9 +252,9 @@ class RegionMean(Diagnostic): if threed: if not self.save3d and threed: return False - final_name = '{1}3d{0}iris'.format(var, self.variable) + final_name = '{1}3d{0}'.format(var, self.variable) else: - final_name = '{1}{0}iris'.format(var, self.variable) + final_name = '{1}{0}'.format(var, self.variable) cube = cube_list.merge_cube() cube.var_name = 'result' cube.remove_coord('latitude') diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 12654b3b..b00a30b9 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -5,7 +5,6 @@ import shutil from bscearth.utils.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption -from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile diff --git a/launch_diags.sh b/launch_diags.sh index 59dd9e14..a333a239 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -7,7 +7,7 @@ -PATH_TO_CONF_FILE=~jvegas/earthdiags_confs/diags_roberto.conf +PATH_TO_CONF_FILE=~rfernand/run_earthdiagnostics/m04p/diags.conf PATH_TO_DIAGNOSTICS=~jvegas/PycharmProjects/earthdiagnostics PATH_TO_CONDAENV=/home/Earth/jvegas/.conda/envs/earthdiagnostics3/ @@ -21,4 +21,4 @@ source activate ${PATH_TO_CONDAENV} export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ -./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} +./earthdiags.py -f ${PATH_TO_CONF_FILE} -- GitLab From d075ef3cf9ef732f94b5742177cbd28307035c36 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 1 Feb 2018 12:04:35 +0100 Subject: [PATCH 162/168] Fixed tests --- earthdiagnostics/cmormanager.py | 4 +- test/unit/ocean/test_region_mean.py | 56 +++++++------- test/unit/test_cmormanager.py | 111 +++++++--------------------- 3 files changed, 57 insertions(+), 114 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 28fa5adc..80e323db 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -246,7 +246,7 @@ class CMORManager(DataManager): var: str cmor_var: Variable chunk: int or None - frequency: Frequency + frequency: Frequency or str grid: str or None year: int or None date_str: str or None @@ -261,6 +261,8 @@ class CMORManager(DataManager): If you provide two or more parameters from chunk, year or date_str or none at all """ + if frequency is None: + frequency = self.config.frequency frequency = Frequency.parse(frequency) options = sum(x is not None for x in (chunk, year, date_str)) if options == 0: diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index 5152c2fc..2ee865fa 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -26,29 +26,29 @@ class TestRegionMean(TestCase): def test_generate_jobs(self): box = Box() - box.min_depth = 0 - box.max_depth = 0 + box.min_depth = -1 + box.max_depth = -1 jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'T', - box, True, Basins().Global, False, '')) - self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'T', - box, True, Basins().Global, False, '')) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + box, True, 'weights', False, Basins().Global)) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + box, True, 'weights', False, Basins().Global)) jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', - box, True, Basins().Global, False, '')) - self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', - box, True, Basins().Global, False, '')) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + box, True, 'weights', False, Basins().Global)) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + box, True, 'weights', False, Basins().Global)) jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', - box, True, Basins().Global, False, '')) - self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', - box, True, Basins().Global, False, '')) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + box, True, 'weights', False, Basins().Global)) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + box, True, 'weights', False, Basins().Global)) box = Box() box.min_depth = 1.0 @@ -56,32 +56,32 @@ class TestRegionMean(TestCase): jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', - box, True, Basins().Global, False, '')) - self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', - box, True, Basins().Global, False, '')) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + box, True, 'weights', False, Basins().Global)) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + box, True, 'weights', False, Basins().Global)) jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box, False, Basins().Global, False, '')) - self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', box, False, Basins().Global, False, '')) jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', 'True']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box, False, Basins().Global, True, '')) - self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', box, False, Basins().Global, True, '')) jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', 'True', 'grid']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box, False, Basins().Global, True, 'grid')) - self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', box, False, Basins().Global, True, 'grid')) with self.assertRaises(DiagnosticOptionError): @@ -96,7 +96,7 @@ class TestRegionMean(TestCase): box.min_depth = 1 box.max_depth = 10 - diag = RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', box, False, - Basins().Global, True, 'grid') - self.assertEqual(str(diag), 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Grid point: U ' - 'Box: 1-10 Save 3D: False Save variance: True Original grid: grid Basin: Global') + diag = RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box, False, 'file', + True, Basins().Global) + self.assertEqual(str(diag), 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var ' + 'Box: 1-10 Save 3D: False Save variance: True') diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 5b673136..841182f4 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -16,6 +16,7 @@ class TestCMORManager(TestCase): def setUp(self): self.config = Mock() self.config.data_convention = 'specs' + self.config.frequency = '6hr' self.config.data_type = 'exp' self.config.experiment.expid = 'expid' self.config.experiment.model = 'model' @@ -79,11 +80,8 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' with self.assertRaises(ValueError): - cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, frequency) + cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, 'mon') def test_get_file_path_specs(self): cmor_manager = CMORManager(self.config) @@ -91,37 +89,26 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - frequency) + 'mon') self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) def test_get_file_path_specs_non_cmor(self): cmor_manager = CMORManager(self.config) - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'mon' - frequency.frequency = 'mon' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, 0, - frequency) + 'mon') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) def test_get_file_path_specs_empty_time_info(self): cmor_manager = CMORManager(self.config) - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'mon' - frequency.frequency = 'mon' with self.assertRaises(ValueError): cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, None, - frequency) + 'mon') def test_get_file_path_preface(self): self._configure_preface() @@ -130,13 +117,10 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - frequency) + 'mon') self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/expname/S19900101/frequency/' + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/expname/S19900101/mon/' 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_198901_198912.nc')) @@ -147,13 +131,10 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - frequency) + 'mon') self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' 'ocean/var/r2i1p1/version/' 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) @@ -163,13 +144,10 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - frequency, 'grid') + 'mon', 'grid') self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' 'ocean/var/grid/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) @@ -179,22 +157,15 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.frequency = 'year' - frequency.__str__ = Mock() - frequency.__str__.return_value = 'year' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, - frequency, year='1998') + 'year', year='1998') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/year/' 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_1998.nc')) - frequency.frequency = 'other' - frequency.__str__ = Mock() - frequency.__str__.return_value = 'other' self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, - 1, frequency, year='1998') + 1, 'mon', year='1998') def test_get_file_path_raise_incompatible_date_info(self): cmor_manager = CMORManager(self.config) @@ -226,14 +197,10 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' - frequency.frequency = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, - frequency, date_str='date_str') + 'mon', date_str='date_str') self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_date_str.nc')) @@ -244,11 +211,8 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - frequency) + 'mon') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' 'r2i1p1f1/Omon/var/ocean_grid/version/' @@ -262,7 +226,7 @@ class TestCMORManager(TestCase): frequency.__str__.return_value = 'mon' frequency.frequency = 'mon' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, 0, - frequency) + 'mon') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' 'r2i1p1f1/Omon/var/ocean_grid/version/' @@ -299,11 +263,8 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - frequency, 'grid') + 'mon', 'grid') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' 'Omon/var/grid/version/' @@ -316,11 +277,8 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.atmos, 'var', cmor_var, 0, - frequency) + 'mon') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' 'r2i1p1f1/Omon/var/atmos_grid/version/' @@ -333,22 +291,15 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.frequency = 'year' - frequency.__str__ = Mock() - frequency.__str__.return_value = 'year' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, - frequency, year='1998') + 'year', year='1998') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/' 'var/ocean_grid/version/' 'var_Omon_model_expname_r2i1p1f1_ocean_grid_1998.nc')) - frequency.frequency = 'other' - frequency.__str__ = Mock() - frequency.__str__.return_value = 'other' self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, - 1, frequency, year='1998') + 1, 'mon', year='1998') def test_get_file_path_primavera_date_str(self): self._configure_primavera() @@ -357,12 +308,8 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' - frequency.frequency = 'frequency' file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, - frequency, date_str='date_str') + 'mon', date_str='date_str') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' 'Omon/var/ocean_grid/version/' @@ -393,16 +340,13 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'day' file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 1, - frequency) + 'day') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_01.nc')) file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 2, - frequency) + 'day') self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_01.nc')) @@ -413,12 +357,9 @@ class TestCMORManager(TestCase): omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'day' with self.assertRaises(ValueError): cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 1, - frequency) + 'day') def test_create_link(self): cmor_manager = CMORManager(self.config) -- GitLab From e9eacc0859f0e6f510f3a62488726a2edf89b342 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 1 Feb 2018 12:25:51 +0100 Subject: [PATCH 163/168] Improved doc --- earthdiagnostics/diagnostic.py | 80 ++++++++++++++++++++++++++++ earthdiagnostics/ocean/regionmean.py | 8 +-- earthdiagnostics/variable.py | 1 + 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index a145a01a..b5b87bd5 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -589,6 +589,14 @@ class DiagnosticVariableOption(DiagnosticOption): self.var_manager = var_manager def parse(self, option_value): + """ + Parse option value + + Returns + ------- + Variable + + """ option_value = self._check_default(option_value) real_name = self.var_manager.get_variable(option_value, False) if real_name is None: @@ -613,6 +621,14 @@ class DiagnosticVariableListOption(DiagnosticOption): self.var_manager = var_manager def parse(self, option_value): + """ + Parse option value + + Returns + ------- + List[Variable] + + """ option_value = self._check_default(option_value) var_names = [] for value in option_value.split(':'): @@ -639,6 +655,13 @@ class DiagnosticDomainOption(DiagnosticOption): super(DiagnosticDomainOption, self).__init__(name, default_value) def parse(self, option_value): + """ + Parse option value + + Returns + ------- + ModelingRealm + """ return ModelingRealms.parse(self._check_default(option_value)) @@ -657,6 +680,18 @@ class DiagnosticFrequencyOption(DiagnosticOption): super(DiagnosticFrequencyOption, self).__init__(name, default_value) def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value: str + + Returns + ------- + Frequency + + """ return Frequency.parse(self._check_default(option_value)) @@ -664,6 +699,18 @@ class DiagnosticBasinOption(DiagnosticOption): """Class to parse basin options""" def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value: str + + Returns + ------- + Basin + + """ value = self._check_default(option_value) basin = Basins().parse(value) if basin is None: @@ -680,6 +727,17 @@ class DiagnosticComplexStrOption(DiagnosticOption): """ def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value:str + + Returns + ------- + str + """ return self._check_default(option_value).replace('&;', ',').replace('&.', ' ') @@ -687,6 +745,17 @@ class DiagnosticBoolOption(DiagnosticOption): """Class to parse boolean options""" def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value:str + + Returns + ------- + Bool + """ option_value = self._check_default(option_value) if isinstance(option_value, bool): return option_value @@ -719,6 +788,17 @@ class DiagnosticChoiceOption(DiagnosticOption): self.parse(default_value) def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value:str + + Returns + ------- + str + """ option_value = self._check_default(option_value) if self.ignore_case: option_value = option_value.lower() diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 1755adda..77198caa 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -315,9 +315,9 @@ class ComputeWeights(Diagnostic): j_indexes = np.where(np.any(mask != 0, 1))[0] mask_small = np.take(np.take(mask, i_indexes, 1), j_indexes, 0) - e1 = self.try_load_cube(1) - e2 = self.try_load_cube(2) - e3 = self.try_load_cube(3) + e1 = self._try_load_cube(1) + e2 = self._try_load_cube(2) + e3 = self._try_load_cube(3) depth = iris.util.squeeze(iris.load_cube('mesh_hgr.nc', 'gdept_0')) if self.box.min_depth == -1: min_level = 0 @@ -356,7 +356,7 @@ class ComputeWeights(Diagnostic): lev_limits = iris.cube.Cube([min_level, max_level], var_name='lev_limits') iris.save((weights, i_indexes, j_indexes, lev_limits), self.weights_file) - def try_load_cube(self, number): + def _try_load_cube(self, number): try: cube = iris.load_cube('mesh_hgr.nc', 'e{0}{1}'.format(number, self.grid_point)) except iris.exceptions.ConstraintMismatchError: diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index e0808288..3cea888f 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -29,6 +29,7 @@ class VariableManager(object): self.clean() def clean(self): + """Clean all information contained in the variable manager""" self._dict_variables = {} self._dict_aliases = {} self.tables = {} -- GitLab From 36869e0b8b17d7a142e2868869f361003e2ce9cd Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 1 Feb 2018 18:07:56 +0100 Subject: [PATCH 164/168] Improved doc --- earthdiagnostics/diagnostic.py | 43 +++++++++++++++++++ .../statistics/climatologicalpercentile.py | 8 ++++ .../statistics/monthlypercentile.py | 28 ++++++++++++ launch_diags.sh | 2 +- 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index b5b87bd5..4090dcd6 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -482,6 +482,17 @@ class DiagnosticFloatOption(DiagnosticOption): """Class for parsing float options""" def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value:str + + Returns + ------- + float + """ return float(self._check_default(option_value)) @@ -506,6 +517,22 @@ class DiagnosticIntOption(DiagnosticOption): self.max_limit = max_limit def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value:str + + Returns + ------- + int + + Raises + ------ + DiagnosticOptionError + If parsed values is outside limits + """ value = int(self._check_default(option_value)) self._check_limits(value) return value @@ -540,6 +567,22 @@ class DiagnosticListIntOption(DiagnosticIntOption): """ Upper limit """ def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value:str + + Returns + ------- + list(int) + + Raises + ------ + DiagnosticOptionError + If parsed values is outside limits + """ option_value = self._check_default(option_value) if isinstance(option_value, tuple) or isinstance(option_value, list): return option_value diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index d97e24c9..8050f7ef 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -82,6 +82,14 @@ class ClimatologicalPercentile(Diagnostic): return job_list def requested_startdates(self): + """ + Required startdates to compute the percentile + + Returns + ------- + list of str + + """ return ['{0}{1:02}01'.format(year, self.forecast_month) for year in range(self.start_year, self.end_year + 1)] def request_data(self): diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index af1c8572..9e3575bd 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -92,13 +92,41 @@ class MonthlyPercentile(Diagnostic): @property def variable_max(self): + """ + Variable name for the maximum + + Returns + ------- + str + + """ return '{0}max'.format(self.variable) @property def variable_min(self): + """ + Variable name for the minimum + + Returns + ------- + str + + """ return '{0}min'.format(self.variable) def percentile(self, percentile): + """ + Variable name for the given percentile + + Parameters + ---------- + percentile: int + + Returns + ------- + str + + """ return '{0}_q{1}'.format(self.variable, percentile) def compute(self): diff --git a/launch_diags.sh b/launch_diags.sh index a333a239..df9a004b 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -21,4 +21,4 @@ source activate ${PATH_TO_CONDAENV} export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ -./earthdiags.py -f ${PATH_TO_CONF_FILE} +./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} -- GitLab From 4f7a5117a6ba09716fac8698337a86e2b8cac723 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 1 Feb 2018 18:13:26 +0100 Subject: [PATCH 165/168] Improved doc --- earthdiagnostics/ocean/regionmean.py | 2 +- earthdiagnostics/variable.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 77198caa..e0704a39 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -278,7 +278,7 @@ class ComputeWeights(Diagnostic): weights_file: str """ - alias = 'computeinterpcdoweights' + alias = 'computeregmeanweights' "Diagnostic alias for the configuration file" @classmethod diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 3cea888f..fee14f52 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -138,6 +138,14 @@ class VariableManager(object): self.register_variable(var) def register_variable(self, var): + """ + Register variable info + + Parameters + ---------- + var: Variable + + """ self._dict_variables[var.short_name.lower()] = var def _load_json(self, json_folder): @@ -242,6 +250,7 @@ class VariableManager(object): return csv_table_path def create_aliases_dict(self): + """Create aliases dictionary for the registered variables""" self._dict_aliases = {} for cmor_var_name in self._dict_variables: cmor_var = self._dict_variables[cmor_var_name] -- GitLab From 6e2c55239d6e0bf0fb5443912a4ac3865f241a84 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 1 Feb 2018 18:16:51 +0100 Subject: [PATCH 166/168] Improved doc and cleaned code --- earthdiagnostics/ocean/interpolatecdo.py | 5 +++++ earthdiagnostics/ocean/regionmean.py | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 8bdb634c..5a09eaf3 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -292,6 +292,11 @@ class ComputeWeights(Diagnostic): @classmethod def generate_jobs(cls, diags, options): + """ + Generate the instances of the diagnostics that will be run by the manager + + This method does not does anything as this diagnostic is not expected to be called by the users + """ pass def __init__(self, data_manager, startdate, member, chunk, domain, variable, target_grid, diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index e0704a39..d57324a0 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -283,6 +283,11 @@ class ComputeWeights(Diagnostic): @classmethod def generate_jobs(cls, diags, options): + """ + Generate the instances of the diagnostics that will be run by the manager + + This method does not does anything as this diagnostic is not expected to be called by the users + """ pass def __init__(self, data_manager, grid_point, basin, box, weights_file): @@ -296,7 +301,7 @@ class ComputeWeights(Diagnostic): if type(self) is not type(other): return False return self.weights_file == other.weights_file and self.basin == other.basin and \ - self.grid_point == other.grid_point and self.box != other.box + self.grid_point == other.grid_point and self.box != other.box def __str__(self): return 'Computing weights for region averaging: Point {0.grid_point} Basin: {0.basin} Box: {0.box}'\ @@ -363,7 +368,6 @@ class ComputeWeights(Diagnostic): cube = iris.load_cube('mesh_hgr.nc', 'e{0}{1}_0'.format(number, self.grid_point)) return iris.util.squeeze(cube) - def request_data(self): """Request data required by the diagnostic""" pass -- GitLab From b187f8a8c1b8fad3475c56b7bc8536c772f4ac3d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 12 Feb 2018 12:13:18 +0100 Subject: [PATCH 167/168] Fixed bug in regionmean --- earthdiagnostics/ocean/regionmean.py | 21 +++++++++++++-------- earthdiagnostics/utils.py | 10 +++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index d57324a0..fa41fcf1 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -256,6 +256,8 @@ class RegionMean(Diagnostic): else: final_name = '{1}{0}'.format(var, self.variable) cube = cube_list.merge_cube() + print(cube) + print(cube.data) cube.var_name = 'result' cube.remove_coord('latitude') cube.remove_coord('longitude') @@ -315,10 +317,11 @@ class ComputeWeights(Diagnostic): iris.FUTURE.netcdf_promote = True iris.FUTURE.netcdf_no_unlimited = True - mask = np.squeeze(Utils.get_mask(self.basin)) - i_indexes = np.where(np.any(mask != 0, 0))[0] - j_indexes = np.where(np.any(mask != 0, 1))[0] - mask_small = np.take(np.take(mask, i_indexes, 1), j_indexes, 0) + mask = np.squeeze(Utils.get_mask(self.basin, True)) + surface_mask = mask[0, ...] + i_indexes = np.where(np.any(surface_mask != 0, 0))[0] + j_indexes = np.where(np.any(surface_mask != 0, 1))[0] + mask_small = np.take(np.take(mask, i_indexes, 2), j_indexes, 1) e1 = self._try_load_cube(1) e2 = self._try_load_cube(2) @@ -348,13 +351,15 @@ class ComputeWeights(Diagnostic): e1_small = e1.extract(iris.Constraint(i=selected_i, j=selected_j)) e2_small = e2.extract(iris.Constraint(i=selected_i, j=selected_j)) e3_small = e3.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) + mask_small = mask_small[min_level:max_level, ...] - mask_small = e1_small * e2_small * mask_small - for coord in e3_small.coords(): - e3_small.remove_coord(coord) + mask_small = e3_small * mask_small + e_small = e1_small * e2_small + for coord in e_small.coords(): + e_small.remove_coord(coord) for coord in mask_small.coords(): mask_small.remove_coord(coord) - weights = e3_small * mask_small + weights = mask_small * e_small weights.var_name = 'weights' i_indexes = iris.cube.Cube(i_indexes, var_name='i_indexes') j_indexes = iris.cube.Cube(j_indexes, var_name='j_indexes') diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 29a5d0ed..bf44e4b4 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -46,7 +46,7 @@ class Utils(object): """An instance of Cdo class ready to be used""" @staticmethod - def get_mask(basin): + def get_mask(basin, with_levels=False): """ Return the mask for the given basin @@ -66,8 +66,12 @@ class Utils(object): basin = Basins().parse(basin) if basin != Basins().Global: try: - mask_handler = Utils.open_cdf('mask_regions.nc') - mask = mask_handler.variables[basin.name][:, 0, :] + if with_levels: + mask_handler = Utils.open_cdf('mask_regions.3d.nc') + mask = mask_handler.variables[basin.name][0, ...] + else: + mask_handler = Utils.open_cdf('mask_regions.nc') + mask = mask_handler.variables[basin.name][:, 0, :] mask_handler.close() except IOError: raise Exception('File mask.regions.nc is required for basin {0}'.format(basin)) -- GitLab From 13be83ed9d8343b714999bd976102225ef091e2d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 14 Feb 2018 12:49:36 +0100 Subject: [PATCH 168/168] Add support for STARTDATES in PRIMAVERA and CMIP6 conventions --- earthdiagnostics/cmormanager.py | 38 ++++++++++++++++++++++++--------- earthdiagnostics/config.py | 1 + test/unit/test_cmormanager.py | 16 ++++++++++++++ test/unit/test_config.py | 1 + 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 80e323db..931f4ec7 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -35,6 +35,24 @@ class CMORManager(DataManager): self.find_model_data() self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid) + def experiment_name(self, startdate): + """ + Get experiment name, appending startdate if needed + + Parameters + ---------- + startdate: str + + Returns + ------- + str + + """ + if self.config.cmor.append_startdate: + return '{}S{}'.format(self.config.experiment.experiment_name, startdate) + else: + return self.config.experiment.experiment_name + def find_model_data(self): """ Seek the configured data folders for the experiment data @@ -293,7 +311,7 @@ class CMORManager(DataManager): if self.config.data_convention in ('specs', 'preface'): file_name = '{0}_{1}_{2}_{3}_S{4}_{5}{6}'.format(var, cmor_table.name, self.experiment.model, - self.experiment.experiment_name, startdate, + self.experiment_name(startdate), startdate, self._get_member_str(member), time_bound) elif self.config.data_convention in ('primavera', 'cmip6'): if not grid: @@ -302,7 +320,7 @@ class CMORManager(DataManager): else: grid = self.config.cmor.default_atmos_grid file_name = '{0}_{1}_{2}_{3}_{4}_{5}{6}'.format(var, cmor_table.name, self.experiment.model, - self.experiment.experiment_name, + self.experiment_name(startdate), self._get_member_str(member), grid, time_bound) # Meteofrance @@ -347,7 +365,7 @@ class CMORManager(DataManager): table_name, var, grid, self.config.cmor.version) elif self.config.data_convention == 'meteofrance': - folder_path = os.path.join(self.config.data_dir, self.experiment.experiment_name, + folder_path = os.path.join(self.config.data_dir, self.experiment_name(startdate), 'H{0}'.format(chr(64 + int(startdate[4:6]))), startdate[0:4]) else: @@ -609,7 +627,7 @@ class CMORManager(DataManager): self._fix_model_as_experiment_error(startdate) def _fix_model_as_experiment_error(self, startdate): - if self.experiment.experiment_name != self.experiment.model: + if self.experiment_name(startdate) != self.experiment.model: bad_path = os.path.join(self.cmor_path, self.experiment.institute, self.experiment.model, self.experiment.model) Log.debug('Correcting double model appearance') @@ -621,15 +639,15 @@ class CMORManager(DataManager): good = filepath good = good.replace('_{0}_output_'.format(self.experiment.model), '_{0}_{1}_S{2}_'.format(self.experiment.model, - self.experiment.experiment_name, + self.experiment_name(startdate), startdate)) good = good.replace('/{0}/{0}'.format(self.experiment.model), '/{0}/{1}'.format(self.experiment.model, - self.experiment.experiment_name)) + self.experiment_name(startdate))) Utils.move_file(filepath, good) - if self.experiment.model != self.experiment.experiment_name: + if self.experiment.model != self.experiment_name(startdate): os.rmdir(dirpath) Log.debug('Done') @@ -722,13 +740,13 @@ class CMORManager(DataManager): """ if self.config.data_convention == 'specs': return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.experiment.institute, - self.experiment.model, self.experiment.experiment_name, 'S' + startdate) + self.experiment.model, self.experiment_name(startdate), 'S' + startdate) elif self.config.data_convention == 'preface': return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.experiment.institute, - self.experiment.experiment_name, 'S' + startdate) + self.experiment_name(startdate), 'S' + startdate) else: return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.config.cmor.activity, - self.experiment.institute, self.experiment.model, self.experiment.experiment_name) + self.experiment.institute, self.experiment.model, self.experiment_name(startdate)) def _get_member_str(self, member): if self.config.data_convention in ('specs', 'preface'): diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 68ca08b2..7ef15246 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -230,6 +230,7 @@ class CMORConfig(object): self.default_atmos_grid = parser.get_option('CMOR', 'DEFAULT_ATMOS_GRID', 'gr') self.activity = parser.get_option('CMOR', 'ACTIVITY', 'CMIP') self.min_cmorized_vars = parser.get_int_option('CMOR', 'MIN_CMORIZED_VARS', 10) + self.append_startdate = parser.get_bool_option('CMOR', 'APPEND_STARTDATE', False) vars_string = parser.get_option('CMOR', 'VARIABLE_LIST', '') self.var_manager = var_manager diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 841182f4..6a32c367 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -36,6 +36,7 @@ class TestCMORManager(TestCase): self.config.cmor.activity = 'activity' self.config.cmor.force = False self.config.cmor.force_untar = False + self.config.cmor.append_startdate = False self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) @@ -218,6 +219,21 @@ class TestCMORManager(TestCase): 'r2i1p1f1/Omon/var/ocean_grid/version/' 'var_Omon_model_expname_r2i1p1f1_ocean_grid_198901-198912.nc')) + def test_get_file_path_primavera_with_startdate(self): + self._configure_primavera() + self.config.cmor.append_startdate = True + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + 'mon') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expnameS19900101/' + 'r2i1p1f1/Omon/var/ocean_grid/version/' + 'var_Omon_model_expnameS19900101_r2i1p1f1_ocean_grid_198901-198912.nc')) + def test_get_file_path_primavera_no_cmor(self): self._configure_primavera() cmor_manager = CMORManager(self.config) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 6b38a194..b1ee9671 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -108,6 +108,7 @@ class TestCMORConfig(TestCase): self.assertEqual(config.default_atmos_grid, 'gr') self.assertEqual(config.default_ocean_grid, 'gn') self.assertEqual(config.min_cmorized_vars, 10) + self.assertEqual(config.append_startdate, False) def test_cmorize(self): config = CMORConfig(self.mock_parser, self.var_manager) -- GitLab