From f378a58084bffa7c10f9ba7d7c84f866e91922a9 Mon Sep 17 00:00:00 2001
From: Javier Vegas-Regidor <javier.vegas@bsc.es>
Date: Mon, 17 Feb 2020 17:38:07 +0100
Subject: [PATCH 1/2] Extract vars before MMA cmorization

---
 earthdiagnostics/cmorizer.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py
index cc254ead..a316c680 100644
--- a/earthdiagnostics/cmorizer.py
+++ b/earthdiagnostics/cmorizer.py
@@ -318,7 +318,22 @@ class Cmorizer(object):
                         options="-O", input=filename, output=temp
                     )
                     shutil.move(temp, filename)
-            Utils.cdo().mergetime(input=files, output=merged)
+            cmorize_vars = set()
+            for filename in files:
+                handler = Utils.open_cdf(filename)
+                for variable in handler.variables.keys():
+                    _, var_cmor = self.config.var_manager.get_variable_and_alias(
+                        variable, silent=True,
+                    )
+                    if self.cmor.cmorize(var_cmor):
+                        cmorize_vars.add(variable)
+            if not cmorize_vars:
+                continue
+            var_str = ",".join([str(var) for var in cmorize_vars])
+            Utils.cdo().mergetime(
+                input=[f"-selvar,{var_str} {filepath}" for filepath in files],
+                output=merged
+            )
             for filename in files:
                 self._remove(filename)
             tar_startdate = (
-- 
GitLab


From e3eb1f2c25870da9f4f5985c6f486815bc535fb5 Mon Sep 17 00:00:00 2001
From: Javier Vegas-Regidor <javier.vegas@bsc.es>
Date: Mon, 17 Feb 2020 18:28:17 +0100
Subject: [PATCH 2/2] Fix tests

---
 earthdiagnostics/cmorizer.py      |  5 +++-
 test/integration/test_cmorizer.py | 38 +++++++++++++++++--------------
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py
index a316c680..6cb10a3d 100644
--- a/earthdiagnostics/cmorizer.py
+++ b/earthdiagnostics/cmorizer.py
@@ -319,14 +319,17 @@ class Cmorizer(object):
                     )
                     shutil.move(temp, filename)
             cmorize_vars = set()
+            var_manager = self.config.var_manager
             for filename in files:
                 handler = Utils.open_cdf(filename)
                 for variable in handler.variables.keys():
-                    _, var_cmor = self.config.var_manager.get_variable_and_alias(
+
+                    _, var_cmor = var_manager.get_variable_and_alias(
                         variable, silent=True,
                     )
                     if self.cmor.cmorize(var_cmor):
                         cmorize_vars.add(variable)
+                handler.close()
             if not cmorize_vars:
                 continue
             var_str = ",".join([str(var) for var in cmorize_vars])
diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py
index 14ee5db2..29fab22b 100644
--- a/test/integration/test_cmorizer.py
+++ b/test/integration/test_cmorizer.py
@@ -21,7 +21,7 @@ import calendar
 class TestCmorizer(TestCase):
     """Tests for Cmorizer class"""
 
-    def _get_variable_and_alias(self, variable):
+    def _get_variable_and_alias(self, variable, silent=False):
         mock_alias = Mock()
         mock_alias.basin = None
         mock_alias.grid = None
@@ -349,22 +349,26 @@ class TestCmorizer(TestCase):
                         log.Log.WARNING: warnings,
                     }
                 ):
-                    if value:
-                        self.assertTrue(
-                            [
-                                record
-                                for record in cmd.records
-                                if record.levelno == level
-                            ]
-                        )
-                    else:
-                        self.assertFalse(
-                            [
-                                record
-                                for record in cmd.records
-                                if record.levelno == level
-                            ]
-                        )
+                    try:
+                        if value:
+                            self.assertTrue(
+                                [
+                                    record
+                                    for record in cmd.records
+                                    if record.levelno == level
+                                ]
+                            )
+                        else:
+                            self.assertFalse(
+                                [
+                                    record
+                                    for record in cmd.records
+                                    if record.levelno == level
+                                ]
+                            )
+                    except AssertionError:
+                        print(cmd.records)
+                        raise
         else:
             cmorizer = Cmorizer(self.data_manager, "19900101", 0)
             if ocean:
-- 
GitLab