From 55bfd691cbd8f7d1e1318b653bf9a64ae1e2fa60 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 7 Sep 2018 17:49:07 +0200 Subject: [PATCH 01/28] First test installed done --- .codacy.yml | 4 ++-- environment.yml | 1 + hermesv3_gr/config/config.py | 6 +++--- hermesv3_gr/hermes.py | 12 +++++------- setup.py | 8 ++++++-- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.codacy.yml b/.codacy.yml index 3e730e5..bf12fc2 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -13,9 +13,9 @@ engines: enabled: true pylint: enabled: true - python_version: 3 + python_version: 2 exclude_paths: [ 'doc/**', - 'earthdiagnostics/cmor_tables/**', + 'data/**', ] diff --git a/environment.yml b/environment.yml index 168dd4e..9a70344 100644 --- a/environment.yml +++ b/environment.yml @@ -6,6 +6,7 @@ channels: - conda-forge dependencies: + - python = 2 - numpy - netcdf4 >= 1.3.1 - python-cdo >= 1.3.4 diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index c2b1ad2..196ddd9 100644 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -52,7 +52,7 @@ class Config(ArgParser): help="Name of the output file. You can add the string '' that will be substitute by the " + "starting date of the simulation day.") p.add_argument('--start_date', required=True, help='Starting Date to simulate (UTC)') - p.add_argument('--end_date', required=False, + p.add_argument('--end_date', required=False, default=None, help='If you want to simulate more than one day you have to specify the ending date of ' + 'simulation in this parameter. If it is not set end_date = start_date.') @@ -166,8 +166,8 @@ class Config(ArgParser): exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( item, options.inc_x, options.inc_y)) - options.start_date = self._parse_start_date(str(options.start_date)) - options.end_date = self._parse_end_date(str(options.end_date), options.start_date) + options.start_date = self._parse_start_date(options.start_date) + options.end_date = self._parse_end_date(options.end_date, options.start_date) self.create_dir(options.output_dir) self.create_dir(options.auxiliar_files_path) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 2ca7ab6..84ac1b4 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -90,13 +90,7 @@ class Hermes(object): """ Main functionality of the model. """ - from multiprocessing import Process, Queue, cpu_count - from threading import Thread - import copy - import gc - import numpy as np from datetime import timedelta - from cf_units import Unit st_time = timeit.default_timer() settings.write_log('') @@ -142,8 +136,12 @@ class Hermes(object): return None -if __name__ == '__main__': +def run(): date = Hermes(Config()).main() while date is not None: date = Hermes(Config(), new_date=date).main() sys.exit(0) + + +if __name__ == '__main__': + run() diff --git a/setup.py b/setup.py index ede6bbd..adc7133 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ setup( 'pyproj', 'configargparse', 'cf_units>=1.1.3', - 'ESMPy>=7.1.0', + 'ESMPy>=7.1.0.dev0', 'holidays', 'pytz', 'timezonefinder', @@ -78,5 +78,9 @@ setup( ] }, - scripts=['bin/hermes_gr'] + entry_points={ + 'console_scripts': [ + 'hermesv3_gr = hermesv3_gr.hermes:run', + ], + }, ) -- GitLab From 252f3b5b3396f7aae7fd3b6ac0ff8f6add298fb0 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 10 Sep 2018 11:52:18 +0200 Subject: [PATCH 02/28] Copy example files done --- environment.yml | 2 +- hermesv3_gr/hermes.py | 1 - hermesv3_gr/tools/sample_files.py | 171 ++++++++++++++++++++++++++++++ setup.py | 5 +- 4 files changed, 175 insertions(+), 4 deletions(-) create mode 100644 hermesv3_gr/tools/sample_files.py diff --git a/environment.yml b/environment.yml index 9a70344..d36c8f2 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - python = 2 - numpy - netcdf4 >= 1.3.1 - - python-cdo >= 1.3.4 + - python-cdo >= 1.3.3 - geopandas - pyproj - configargparse diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 84ac1b4..cf1f0ed 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -19,7 +19,6 @@ import timeit - from hermesv3_gr.config import settings from hermesv3_gr.config.config import Config from hermesv3_gr.modules.emision_inventories.emission_inventory import EmissionInventory diff --git a/hermesv3_gr/tools/sample_files.py b/hermesv3_gr/tools/sample_files.py new file mode 100644 index 0000000..78c762a --- /dev/null +++ b/hermesv3_gr/tools/sample_files.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +# Copyright 2018 Earth Sciences Department, BSC-CNS +# +# This file is part of HERMESv3_GR. +# +# HERMESv3_GR is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# HERMESv3_GR is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with HERMESv3_GR. If not, see . + + +import sys +import os + + +def make_conf_file_list(): + main_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir) + + file_list = [ + {'conf': [ + os.path.join(main_dir, 'conf', 'hermes.conf'), + os.path.join(main_dir, 'conf', 'EI_configuration.csv'), + ]}, + ] + + return file_list + + +def make_profiles_file_list(): + main_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir) + + file_list = [ + {'data': [ + os.path.join(main_dir, 'data', 'global_attributes.csv'), + {'profiles': [{ + 'speciation': [ + os.path.join(main_dir, 'data', 'profiles', 'speciation', 'Speciation_profile_cb05_aero5_CMAQ.csv'), + os.path.join(main_dir, 'data', 'profiles', 'speciation', + 'Speciation_profile_cb05_aero5_MONARCH_aerosols.csv'), + os.path.join(main_dir, 'data', 'profiles', 'speciation', + 'Speciation_profile_cb05_aero5_MONARCH_fullchem.csv'), + os.path.join(main_dir, 'data', 'profiles', 'speciation', + 'Speciation_profile_radm2_madesorgam_WRF_CHEM.csv'), + ]}, + {'temporal': [ + os.path.join(main_dir, 'data', 'profiles', 'temporal', 'TemporalProfile_Daily.csv'), + os.path.join(main_dir, 'data', 'profiles', 'temporal', 'TemporalProfile_Hourly.csv'), + os.path.join(main_dir, 'data', 'profiles', 'temporal', 'TemporalProfile_Monthly.csv'), + os.path.join(main_dir, 'data', 'profiles', 'temporal', 'tz_world_country_iso3166.csv'), + ]}, + {'vertical': [ + os.path.join(main_dir, 'data', 'profiles', 'vertical', '1layer_vertical_description.csv'), + os.path.join(main_dir, 'data', 'profiles', 'vertical', 'CMAQ_15layers_vertical_description.csv'), + os.path.join(main_dir, 'data', 'profiles', 'vertical', + 'MONARCH_Global_48layers_vertical_description.csv'), + os.path.join(main_dir, 'data', 'profiles', 'vertical', + 'MONARCH_regional_48layers_vertical_description.csv'), + os.path.join(main_dir, 'data', 'profiles', 'vertical', 'Vertical_profile.csv'), + ]}, + ]}, + ]}, + ] + + return file_list + + +def make_preproc_file_list(): + main_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir) + + file_list = [ + os.path.join(main_dir, 'preproc', 'ceds_preproc.py'), + os.path.join(main_dir, 'preproc', 'eclipsev5a_preproc.py'), + os.path.join(main_dir, 'preproc', 'edgarv432_ap_preproc.py'), + os.path.join(main_dir, 'preproc', 'edgarv432_voc_preproc.py'), + os.path.join(main_dir, 'preproc', 'emep_preproc.py'), + os.path.join(main_dir, 'preproc', 'gfas12_preproc.py'), + os.path.join(main_dir, 'preproc', 'htapv2_preproc.py'), + os.path.join(main_dir, 'preproc', 'tno_mac_iii_preproc.py'), + os.path.join(main_dir, 'preproc', 'tno_mac_iii_preproc_voc_ratios.py'), + os.path.join(main_dir, 'preproc', 'wiedinmyer_preproc.py'), + ] + + return file_list + + +def query_yes_no(question, default="yes"): + valid = {"yes": True, "y": True, "1": True, 1: True, + "no": False, "n": False, "0": False, 0: False} + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while True: + sys.stdout.write(question + prompt) + choice = raw_input().lower() + if default is not None and choice == '': + return valid[default] + elif choice in valid: + return valid[choice] + else: + sys.stdout.write("Please respond with 'yes' or 'no' (or 'y' or 'n').\n") + + +def check_args(args, exe_str): + if len(args) == 0: + print("Missing destination path after '{0}'. e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES/sample_files".format(exe_str)) + sys.exit(1) + elif len(args) > 1: + print("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES/sample_files".format(exe_str)) + sys.exit(1) + else: + dir_path = args[0] + + if not os.path.exists(dir_path): + if query_yes_no("'{0}' does not exist. Do you want to create it? ".format(dir_path)): + os.makedirs(dir_path) + else: + sys.exit(0) + + return dir_path + + +def copy_files(file_list, directory): + from shutil import copy2 + + if not os.path.exists(directory): + os.makedirs(directory) + + for element in file_list: + if dict == type(element): + copy_files(element.values()[0], os.path.join(directory, element.keys()[0])) + else: + copy2(element, directory) + return True + + +def copy_sample_files(): + argv = sys.argv[1:] + + parent_dir = check_args(argv, 'copy_sample_files') + + copy_files(make_conf_file_list(), parent_dir) + copy_files(make_profiles_file_list(), parent_dir) + + +def copy_preproc_files(): + argv = sys.argv[1:] + + parent_dir = check_args(argv, 'copy_preproc_files') + + copy_files(make_preproc_file_list(), parent_dir) + + +if __name__ == '__main__': + copy_sample_files() diff --git a/setup.py b/setup.py index adc7133..da05cf8 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( install_requires=[ 'numpy', 'netCDF4>=1.3.1', - 'cdo>=1.3.4', + 'cdo>=1.3.3', 'pandas', 'geopandas', 'pyproj', @@ -70,7 +70,6 @@ setup( ], include_package_data=True, package_data={'hermesv3_gr': [ - 'data/*', 'README', 'CHANGELOG', 'VERSION', @@ -81,6 +80,8 @@ setup( entry_points={ 'console_scripts': [ 'hermesv3_gr = hermesv3_gr.hermes:run', + 'hermesv3_copy_sample_files = hermesv3_gr.tools.sample_files:copy_sample_files', + 'hermesv3_copy_preproc_files = hermesv3_gr.tools.sample_files:copy_preproc_files', ], }, ) -- GitLab From 6871e9e88fc1369938ca9415a7a5bc3301da1a82 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 10 Sep 2018 12:17:31 +0200 Subject: [PATCH 03/28] change hermesv3_copy_sample_files by hermesv3_copy_config_files --- hermesv3_gr/tools/sample_files.py | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hermesv3_gr/tools/sample_files.py b/hermesv3_gr/tools/sample_files.py index 78c762a..987b806 100644 --- a/hermesv3_gr/tools/sample_files.py +++ b/hermesv3_gr/tools/sample_files.py @@ -150,10 +150,10 @@ def copy_files(file_list, directory): return True -def copy_sample_files(): +def copy_config_files(): argv = sys.argv[1:] - parent_dir = check_args(argv, 'copy_sample_files') + parent_dir = check_args(argv, 'hermesv3_copy_config_files') copy_files(make_conf_file_list(), parent_dir) copy_files(make_profiles_file_list(), parent_dir) @@ -162,10 +162,10 @@ def copy_sample_files(): def copy_preproc_files(): argv = sys.argv[1:] - parent_dir = check_args(argv, 'copy_preproc_files') + parent_dir = check_args(argv, 'hermesv3_copy_preproc_files') copy_files(make_preproc_file_list(), parent_dir) if __name__ == '__main__': - copy_sample_files() + copy_config_files() diff --git a/setup.py b/setup.py index da05cf8..69e262c 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ setup( entry_points={ 'console_scripts': [ 'hermesv3_gr = hermesv3_gr.hermes:run', - 'hermesv3_copy_sample_files = hermesv3_gr.tools.sample_files:copy_sample_files', + 'hermesv3_copy_config_files = hermesv3_gr.tools.sample_files:copy_config_files', 'hermesv3_copy_preproc_files = hermesv3_gr.tools.sample_files:copy_preproc_files', ], }, -- GitLab From 28f74f1c4f8ec1a5042da1f90ede3ec4e2eb3924 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 10 Sep 2018 12:37:43 +0200 Subject: [PATCH 04/28] Added run_test script --- environment.yml | 1 + run_test.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 run_test.py diff --git a/environment.yml b/environment.yml index d36c8f2..0cb41ee 100644 --- a/environment.yml +++ b/environment.yml @@ -18,6 +18,7 @@ dependencies: - pytz - timezonefinder - mpi4py + - pytest - pip: - holidays diff --git a/run_test.py b/run_test.py new file mode 100644 index 0000000..cec7ec8 --- /dev/null +++ b/run_test.py @@ -0,0 +1,24 @@ +# coding=utf-8 +"""Script to run the tests for EarthDiagnostics and generate the code coverage report""" + +import os +import sys +import pytest +work_path = os.path.abspath(os.path.join(os.path.dirname(__file__))) +os.chdir(work_path) +print(work_path) + + +version = sys.version_info[0] +report_dir = 'test/report/python{}'.format(version) +errno = pytest.main([ + 'test', + '--ignore=test/report', + '--cov=earthdiagnostics', + '--cov-report=term', + '--cov-report=html:{}/coverage_html'.format(report_dir), + '--cov-report=xml:{}/coverage.xml'.format(report_dir), + '--profile', + '--profile-svg', +]) +sys.exit(errno) -- GitLab From 0dae89e46994f0cf732b1d12918a9cd7272a1ba4 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 10 Sep 2018 12:41:47 +0200 Subject: [PATCH 05/28] Added test dependencies to environment --- environment.yml | 2 ++ preproc/ceds_preproc.py | 0 preproc/eclipsev5a_preproc.py | 0 preproc/emep_preproc.py | 0 preproc/tno_mac_iii_preproc.py | 0 preproc/tno_mac_iii_preproc_voc_ratios.py | 0 run_test.py | 4 +--- 7 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 preproc/ceds_preproc.py mode change 100644 => 100755 preproc/eclipsev5a_preproc.py mode change 100644 => 100755 preproc/emep_preproc.py mode change 100644 => 100755 preproc/tno_mac_iii_preproc.py mode change 100644 => 100755 preproc/tno_mac_iii_preproc_voc_ratios.py diff --git a/environment.yml b/environment.yml index 0cb41ee..7cab561 100644 --- a/environment.yml +++ b/environment.yml @@ -18,7 +18,9 @@ dependencies: - pytz - timezonefinder - mpi4py + # Testing - pytest + - pytest-cov - pip: - holidays diff --git a/preproc/ceds_preproc.py b/preproc/ceds_preproc.py old mode 100644 new mode 100755 diff --git a/preproc/eclipsev5a_preproc.py b/preproc/eclipsev5a_preproc.py old mode 100644 new mode 100755 diff --git a/preproc/emep_preproc.py b/preproc/emep_preproc.py old mode 100644 new mode 100755 diff --git a/preproc/tno_mac_iii_preproc.py b/preproc/tno_mac_iii_preproc.py old mode 100644 new mode 100755 diff --git a/preproc/tno_mac_iii_preproc_voc_ratios.py b/preproc/tno_mac_iii_preproc_voc_ratios.py old mode 100644 new mode 100755 diff --git a/run_test.py b/run_test.py index cec7ec8..9fcabec 100644 --- a/run_test.py +++ b/run_test.py @@ -14,11 +14,9 @@ report_dir = 'test/report/python{}'.format(version) errno = pytest.main([ 'test', '--ignore=test/report', - '--cov=earthdiagnostics', + '--cov=hermesv3_gr', '--cov-report=term', '--cov-report=html:{}/coverage_html'.format(report_dir), '--cov-report=xml:{}/coverage.xml'.format(report_dir), - '--profile', - '--profile-svg', ]) sys.exit(errno) -- GitLab From 295d3fab1b1de75ac6f762aae7e536d0039d172c Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 10 Sep 2018 12:44:14 +0200 Subject: [PATCH 06/28] Fix test script --- run_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_test.py b/run_test.py index 9fcabec..663775c 100644 --- a/run_test.py +++ b/run_test.py @@ -10,10 +10,10 @@ print(work_path) version = sys.version_info[0] -report_dir = 'test/report/python{}'.format(version) +report_dir = 'tests/report/python{}'.format(version) errno = pytest.main([ - 'test', - '--ignore=test/report', + 'tests', + '--ignore=tests/report', '--cov=hermesv3_gr', '--cov-report=term', '--cov-report=html:{}/coverage_html'.format(report_dir), -- GitLab From 73a3d524e07b522735a512c8175cbc4afa285813 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 10 Sep 2018 13:49:04 +0200 Subject: [PATCH 07/28] Correcting code to python conventions --- environment.yml | 1 + hermesv3_gr/config/settings.py | 5 +- hermesv3_gr/hermes.py | 9 +- .../emision_inventories/emission_inventory.py | 8 +- hermesv3_gr/modules/regrid/regrid.py | 2 +- hermesv3_gr/modules/writing/writer_cmaq.py | 8 +- hermesv3_gr/modules/writing/writer_monarch.py | 1 - .../modules/writing/writer_wrf_chem.py | 3 - hermesv3_gr/tools/coordinates_tools.py | 22 +- hermesv3_gr/tools/lcc_LatLon_to_m.py | 22 - tests/unit/test_lint.py | 33 + tests/unit/test_temporal.py | 765 +++++++++--------- 12 files changed, 440 insertions(+), 439 deletions(-) delete mode 100644 hermesv3_gr/tools/lcc_LatLon_to_m.py create mode 100644 tests/unit/test_lint.py diff --git a/environment.yml b/environment.yml index 7cab561..2916618 100644 --- a/environment.yml +++ b/environment.yml @@ -21,6 +21,7 @@ dependencies: # Testing - pytest - pytest-cov + - pycodestyle - pip: - holidays diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index d16ac27..52c5c39 100644 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -53,7 +53,7 @@ def define_global_vars(in_log_level): global comm global rank global size - + icomm = MPI.COMM_WORLD comm = icomm.Split(color=0, key=0) rank = comm.Get_rank() @@ -115,7 +115,8 @@ def finish_logs(output_dir, date): date.strftime('%Y%m%d%H'), str(size).zfill(4))) if os.path.exists(times_path): os.remove(times_path) - df_merged = reduce(lambda left, right: pd.merge(left, right, on=['Class', 'Function'], how='outer'), data_frames) + df_merged = reduce(lambda left, right: pd.merge(left, right, on=['Class', 'Function'], how='outer'), + data_frames) df_merged['min'] = df_merged.loc[:, range(size)].min(axis=1) df_merged['max'] = df_merged.loc[:, range(size)].max(axis=1) df_merged['mean'] = df_merged.loc[:, range(size)].mean(axis=1) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index cf1f0ed..f094111 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -57,7 +57,8 @@ class Hermes(object): if self.options.output_model in ['CMAQ', 'WRF_CHEM'] and self.options.domain_type == 'global': settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: - raise AttributeError('ERROR: Global domain is not aviable for {0} output model.'.format(self.options.output_model)) + raise AttributeError('ERROR: Global domain is not aviable for {0} output model.'.format( + self.options.output_model)) sys.exit(1) self.levels = VerticalDistribution.get_vertical_output_profile(self.options.vertical_description) @@ -70,7 +71,8 @@ class Hermes(object): self.options.nx, self.options.ny, self.options.inc_x, self.options.inc_y, self.options.x_0, self.options.y_0, self.options.lat_ts) - self.emission_list = EmissionInventory.make_emission_list(self.options, self.grid, self.levels, self.options.start_date) + self.emission_list = EmissionInventory.make_emission_list(self.options, self.grid, self.levels, + self.options.start_date) self.delta_hours = TemporalDistribution.calculate_delta_hours( self.options.start_date, self.options.output_timestep_type, self.options.output_timestep_num, @@ -78,7 +80,8 @@ class Hermes(object): self.writer = Writer.get_writer( self.options.output_model, self.config.get_output_name(self.options.start_date), self.grid, - self.levels, self.options.start_date, self.delta_hours, self.options.output_attributes, compress=settings.compressed_netcdf, + self.levels, self.options.start_date, self.delta_hours, self.options.output_attributes, + compress=settings.compressed_netcdf, parallel=not settings.writing_serial) settings.write_log('End of HERMESv3 initialization.') diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index 3259949..75573dd 100644 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -157,10 +157,10 @@ class EmissionInventory(object): def create_pollutants_dicts(self, pollutants): """ Creates a list of dictionaries with the information of the name, paht and Dataset of each pollutant - + :param pollutants: List of pollutants names :type pollutants: list - + :return: List of dictionaries :rtype: list """ @@ -266,10 +266,10 @@ class EmissionInventory(object): :param options: Full list of parameters given by passing argument or in the configuration file. :type options: Namespace - + :param grid: Grid to use. :type grid: Grid - + :param vertical_output_profile: Path to eht file that contains the vertical profile. :type vertical_output_profile: str diff --git a/hermesv3_gr/modules/regrid/regrid.py b/hermesv3_gr/modules/regrid/regrid.py index 7304084..d533678 100644 --- a/hermesv3_gr/modules/regrid/regrid.py +++ b/hermesv3_gr/modules/regrid/regrid.py @@ -33,7 +33,7 @@ class Regrid(object): self.pollutant_dicts = pollutant_dicts self.weight_matrix_file = weight_matrix_file self.masking = masking - + if not self.is_created_weight_matrix(erase=False): settings.write_log("\t\t\tWeight matrix {0} is not created. ".format(weight_matrix_file) + "Trying to create it", level=1) diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 7b23615..439a6f9 100644 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -167,8 +167,8 @@ class WriterCmaq(Writer): atts_dict[att] = np.array(df.loc[df['attribute'] == att, 'value'].item().split(), dtype=np.float32) except ValueError: - settings.write_log('WARNING: The global attribute {0} is not defined; Using default value {1}'.format( - att, atts_dict[att])) + settings.write_log('WARNING: The global attribute {0} is not defined;'.format(att) + + ' Using default value {0}'.format(atts_dict[att])) if settings.rank == 0: warning('WARNING: The global attribute {0} is not defined; Using default value {1}'.format( att, atts_dict[att])) @@ -597,8 +597,8 @@ class WriterCmaq(Writer): elif i == settings.size - 1: var[:, :, :, full_position[i][2]:] = recvbuf[i, :, :, :, :-1] else: - var[:, :, :, full_position[i][2]:full_position[i][3]] = recvbuf[i, :, :, :, - : full_shape[i][-1]] + var[:, :, :, full_position[i][2]:full_position[i][3]] = \ + recvbuf[i, :, :, :, : full_shape[i][-1]] except: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index f823928..a612eaf 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -748,4 +748,3 @@ class WriterMonarch(Writer): if settings.rank == 0: netcdf.close() settings.write_time('WriterMonarch', 'write_serial_netcdf', timeit.default_timer() - st_time, level=3) - diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index b42a0db..0e479d0 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -439,6 +439,3 @@ class WriterWrfChem(Writer): netcdf.close() settings.write_time('WriterWrfChem', 'write_serial_netcdf', timeit.default_timer() - st_time, level=3) return True - - - diff --git a/hermesv3_gr/tools/coordinates_tools.py b/hermesv3_gr/tools/coordinates_tools.py index 4c1b767..20a4d12 100644 --- a/hermesv3_gr/tools/coordinates_tools.py +++ b/hermesv3_gr/tools/coordinates_tools.py @@ -206,7 +206,8 @@ def rotated2latlon_single(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min= # # # # sph=ctph0*stph+stph0*ctph*ctlm - # sin_rotated_lat = (cos_lat_pole_rad*sin_lat_rad) + (sin_distance_from_center_lon*cos_lat_rad*cos_distance_from_center_lon) + # sin_rotated_lat = (cos_lat_pole_rad*sin_lat_rad) + + # (sin_distance_from_center_lon*cos_lat_rad*cos_distance_from_center_lon) # # sph=min(sph,1.) # # sph=max(sph,-1.) # if sin_rotated_lat > 1.: @@ -215,7 +216,8 @@ def rotated2latlon_single(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min= # sin_rotated_lat = -1. # # aph=asin(sph) # real_latitude = math.asin(sin_rotated_lat) - # real_longitude = math.atan2(cos_lat_rad*sin_distance_from_center_lon, (cos_distance_from_center_lon*cos_lat_rad - sin_lat_pole_rad*sin_rotated_lat)/cos_lat_pole_rad) - math.pi + # real_longitude = math.atan2(cos_lat_rad*sin_distance_from_center_lon, + # (cos_distance_from_center_lon*cos_lat_rad - sin_lat_pole_rad*sin_rotated_lat)/cos_lat_pole_rad) - math.pi # Positive east to negative east lon_pole_deg -= 180 @@ -467,23 +469,9 @@ if __name__ == '__main__': import numpy as np new_pole_longitude_degrees = 20.0 # lonpole tlm0d new_pole_latitude_degrees = 35.0 # latpole tph0d - # print latlon2rotated(new_pole_longitude_degrees, new_pole_latitude_degrees, 20.0, 35.0) print latlon2rotated(new_pole_longitude_degrees, new_pole_latitude_degrees, -20.2485, -9.9036) - # print rotated2latlon_single(new_pole_longitude_degrees, new_pole_latitude_degrees, 0, 0) print rotated2latlon_single(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -35.) - # # print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -34.9) - # # print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -34.8) - # # print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -34.7) print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, np.array([-51., -51., -51., -51.]), - np.array([-35., -34.9, -34.8, -34.7])) - # - # lat, lon = rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, np.array([0]), np.array([0])) - # print lat - - # lat, lon, b_lat, b_lon = create_regular_grid(0, 0, -180, -90, 1., 1.) - # print lat - # print lon - # print b_lat - # print b_lon + np.array([-35., -34.9, -34.8, -34.7])) diff --git a/hermesv3_gr/tools/lcc_LatLon_to_m.py b/hermesv3_gr/tools/lcc_LatLon_to_m.py deleted file mode 100644 index 5fbe9d9..0000000 --- a/hermesv3_gr/tools/lcc_LatLon_to_m.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -from pyproj import Proj -if __name__ == '__main__': - projection = Proj( - proj='lcc', - ellps='WGS84', - R=6370000.000, - lat_1=37, - lat_2=43, - lon_0=-3, - lat_0=40, - to_meter=1, - x_0=0, - y_0=0, - a=6370000.000, - k_0=1.0) - lon_array = [-11.5488, -11.5066, 7.1104] - lat_array = [32.5108, 32.5142, 46.6579] - UTMx, UTMy = projection(lon_array, lat_array) - - print UTMx, UTMy \ No newline at end of file diff --git a/tests/unit/test_lint.py b/tests/unit/test_lint.py new file mode 100644 index 0000000..5a5fd76 --- /dev/null +++ b/tests/unit/test_lint.py @@ -0,0 +1,33 @@ +""" Lint tests """ +import os +import unittest + +import pycodestyle # formerly known as pep8 + + +class TestLint(unittest.TestCase): + + def test_pep8_conformance(self): + """Test that we conform to PEP-8.""" + + check_paths = [ + 'hermesv3_gr', + 'tests', + ] + exclude_paths = [ + + ] + + print("PEP8 check of directories: {}\n".format(', '.join(check_paths))) + + # Get paths wrt package root + package_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + for paths in (check_paths, exclude_paths): + for i, path in enumerate(paths): + paths[i] = os.path.join(package_root, path) + + style = pycodestyle.StyleGuide() + style.options.exclude.extend(exclude_paths) + style.options.max_line_length = 120 + + self.assertEqual(style.check_files(check_paths).total_errors, 0) diff --git a/tests/unit/test_temporal.py b/tests/unit/test_temporal.py index 3ffed3c..729c07a 100644 --- a/tests/unit/test_temporal.py +++ b/tests/unit/test_temporal.py @@ -28,388 +28,390 @@ class TestTemporalDistribution(unittest.TestCase): def setUp(self): pass - def testing_calculate_ending_date_1hour(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 1, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_ending_date(), - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0)) - - def testing_calculate_ending_date_24hours(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 24, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_ending_date(), - datetime(year=2016, month=01, day=01, hour=23, minute=0, second=0)) - - def testing_calculate_ending_date_3hour_each2(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 3, 2, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_ending_date(), - datetime(year=2016, month=01, day=01, hour=4, minute=0, second=0)) - - def testing_def_calculate_timedelta_3hour_each2(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 3, 2, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_timedelta(datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0)), - timedelta(hours=2)) - - def testing_def_calculate_timedelta_month(self): - temporal = TemporalDistribution( - datetime(year=2017, month=02, day=01, hour=0, minute=0, second=0), 'monthly', 1, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_timedelta(datetime(year=2017, month=02, day=01, hour=0, minute=0, second=0)), - timedelta(hours=24*28)) - - def testing_def_calculate_timedelta_month_leapyear(self): - temporal = TemporalDistribution( - datetime(year=2016, month=02, day=01, hour=0, minute=0, second=0), 'monthly', 1, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_timedelta(datetime(year=2016, month=02, day=01, hour=0, minute=0, second=0)), - timedelta(hours=24*29)) - - def testing_get_tz_from_id(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.get_tz_from_id(309), - "Europe/Andorra") - - def testing_get_id_from_tz(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.get_id_from_tz("Europe/Andorra"), - 309) - - def testing_parse_tz(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.parse_tz("America/Fort_Nelson"), - 'America/Vancouver') - - def testing_find_closest_timezone_BCN(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.find_closest_timezone(41.390205, 2.154007), - 'Europe/Madrid') - - def testing_find_closest_timezone_MEX(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.find_closest_timezone(19.451054, -99.125519), - "America/Mexico_City") - - def testing_find_closest_timezone_Kuwait(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.find_closest_timezone(29.378586, 47.990341), - "Asia/Kuwait") - - def testing_find_closest_timezone_Shanghai(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - + # def testing_calculate_ending_date_1hour(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 1, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_ending_date(), + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0)) + # + # def testing_calculate_ending_date_24hours(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 24, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_ending_date(), + # datetime(year=2016, month=01, day=01, hour=23, minute=0, second=0)) + # + # def testing_calculate_ending_date_3hour_each2(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 3, 2, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_ending_date(), + # datetime(year=2016, month=01, day=01, hour=4, minute=0, second=0)) + # + # def testing_def_calculate_timedelta_3hour_each2(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 3, 2, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_timedelta(datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0)), + # timedelta(hours=2)) + # + # def testing_def_calculate_timedelta_month(self): + # temporal = TemporalDistribution( + # datetime(year=2017, month=02, day=01, hour=0, minute=0, second=0), 'monthly', 1, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_timedelta(datetime(year=2017, month=02, day=01, hour=0, minute=0, second=0)), + # timedelta(hours=24*28)) + # + # def testing_def_calculate_timedelta_month_leapyear(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=02, day=01, hour=0, minute=0, second=0), 'monthly', 1, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_timedelta(datetime(year=2016, month=02, day=01, hour=0, minute=0, second=0)), + # timedelta(hours=24*29)) + # + # def testing_get_tz_from_id(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.get_tz_from_id(309), + # "Europe/Andorra") + # + # def testing_get_id_from_tz(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.get_id_from_tz("Europe/Andorra"), + # 309) + # + # def testing_parse_tz(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.parse_tz("America/Fort_Nelson"), + # 'America/Vancouver') + # + # def testing_find_closest_timezone_BCN(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.find_closest_timezone(41.390205, 2.154007), + # 'Europe/Madrid') + # + # def testing_find_closest_timezone_MEX(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.find_closest_timezone(19.451054, -99.125519), + # "America/Mexico_City") + # + # def testing_find_closest_timezone_Kuwait(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.find_closest_timezone(29.378586, 47.990341), + # "Asia/Kuwait") + # + # def testing_find_closest_timezone_Shanghai(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.find_closest_timezone(31.267401, 121.522179), + # "Asia/Shanghai") + # + # def testing_create_netcdf_timezones(self): + # import numpy as np + # from hermesv3_gr.modules.grids.grid import Grid + # from hermesv3_gr.tools.netcdf_tools import extract_vars + # + # aux_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing' + # if not os.path.exists(aux_path): + # os.makedirs(aux_path) + # + # grid = Grid('global', aux_path) + # grid.center_latitudes = np.array([[41.390205, 19.451054], [29.378586, 31.267401]]) + # grid.center_longitudes = np.array([[2.154007, -99.125519], [47.990341, 121.522179]]) + # + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # aux_path) + # + # self.assertTrue(temporal.create_netcdf_timezones(grid)) + # + # [timezones] = extract_vars(temporal.netcdf_timezones, ['timezone_id']) + # timezones = list(timezones['data'][0, :].astype(int).flatten()) + # + # self.assertEqual(timezones, + # [335, 147, 247, 268]) + # + # def testing_calculate_timezones(self): + # self.testing_create_netcdf_timezones() + # + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # self.assertEqual(temporal.calculate_timezones().tolist(), + # [['Europe/Madrid', "America/Mexico_City"], ["Asia/Kuwait", "Asia/Shanghai"]]) + # + # def testing_calculate_2d_temporal_factors(self): + # self.testing_create_netcdf_timezones() + # + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # timezones = temporal.calculate_timezones() + # + # temporal.monthly_profile = {1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1., + # 7: 1., + # 8: 1., + # 9: 1., + # 10: 1., + # 11: 1., + # 12: 1.} + # temporal.daily_profile_id = {0: 1., + # 1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1.} + # temporal.hourly_profile = {0: 1., + # 1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1., + # 7: 1., + # 8: 1., + # 9: 1., + # 10: 1., + # 11: 1., + # 12: 1., + # 13: 20., + # 14: 1., + # 15: 1., + # 16: 1., + # 17: 1., + # 18: 1., + # 19: 1., + # 20: 1., + # 21: 1., + # 22: 1., + # 23: 1.} + # self.assertEqual( - temporal.find_closest_timezone(31.267401, 121.522179), - "Asia/Shanghai") - - def testing_create_netcdf_timezones(self): - import numpy as np - from hermesv3_gr.modules.grids.grid import Grid - from hermesv3_gr.tools.netcdf_tools import extract_vars - - aux_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing' - if not os.path.exists(aux_path): - os.makedirs(aux_path) - - grid = Grid('global', aux_path) - grid.center_latitudes = np.array([[41.390205, 19.451054], [29.378586, 31.267401]]) - grid.center_longitudes = np.array([[2.154007, -99.125519], [47.990341, 121.522179]]) - - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - aux_path) - - self.assertTrue(temporal.create_netcdf_timezones(grid)) - - [timezones] = extract_vars(temporal.netcdf_timezones, ['timezone_id']) - timezones = list(timezones['data'][0, :].astype(int).flatten()) - - self.assertEqual(timezones, - [335, 147, 247, 268]) - - def testing_calculate_timezones(self): - self.testing_create_netcdf_timezones() - - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - self.assertEqual(temporal.calculate_timezones().tolist(), - [['Europe/Madrid', "America/Mexico_City"], ["Asia/Kuwait", "Asia/Shanghai"]]) - - def testing_calculate_2d_temporal_factors(self): - self.testing_create_netcdf_timezones() - - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - timezones = temporal.calculate_timezones() - - temporal.monthly_profile = {1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1., - 7: 1., - 8: 1., - 9: 1., - 10: 1., - 11: 1., - 12: 1.} - temporal.daily_profile_id = {0: 1., - 1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1.} - temporal.hourly_profile = {0: 1., - 1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1., - 7: 1., - 8: 1., - 9: 1., - 10: 1., - 11: 1., - 12: 1., - 13: 20., - 14: 1., - 15: 1., - 16: 1., - 17: 1., - 18: 1., - 19: 1., - 20: 1., - 21: 1., - 22: 1., - 23: 1.} - - self.assertEqual(temporal.calculate_2d_temporal_factors(datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), timezones).tolist(), - [[20., 1.], [1., 1.]]) - - def testing_do_temporal(self): - import numpy as np - from hermesv3_gr.modules.grids.grid import Grid - self.testing_create_netcdf_timezones() - - aux_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing' - - temporal = TemporalDistribution( - datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), 'hourly', 1, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - aux_path) - temporal.monthly_profile = {1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1., - 7: 1., - 8: 1., - 9: 1., - 10: 1., - 11: 1., - 12: 1.} - temporal.daily_profile_id = {0: 1., - 1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1.} - temporal.hourly_profile = {0: 1., - 1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1., - 7: 1., - 8: 1., - 9: 1., - 10: 1., - 11: 1., - 12: 1., - 13: 20., - 14: 1., - 15: 1., - 16: 1., - 17: 1., - 18: 1., - 19: 1., - 20: 1., - 21: 1., - 22: 1., - 23: 1.} - - grid = Grid('global', aux_path) - grid.center_latitudes = np.array([[41.390205, 19.451054], [29.378586, 31.267401]]) - grid.center_longitudes = np.array([[2.154007, -99.125519], [47.990341, 121.522179]]) - data_in = [{'data': np.array([[10., 10.], [10., 10.]])}] - # data_out = [{'data': np.array([[200., 10.], [10., 10.]])}] - data_out = temporal.do_temporal(data_in, grid) - - self.assertEqual(data_out[0]['data'].tolist(), [[[200., 10.], [10., 10.]]]) - - def testing_calculate_weekdays_no_leap_year(self): - from datetime import datetime - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - self.assertEqual(temporal.calculate_weekdays(datetime(year=2017, month=02, day=1)), - {0: 4, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}) - - def testing_calculate_weekdays_leap_year(self): - from datetime import datetime - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - self.assertEqual(temporal.calculate_weekdays(datetime(year=2016, month=02, day=1)), - {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}) - - def testing_calculate_weekdays_factors_full_month(self): - from datetime import datetime - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - - self.assertEqual(round(temporal.calculate_weekday_factor_full_month( - {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}), 5), - round(0.2/29, 5)) - - def testing_calculate_rebalance_factor(self): - from datetime import datetime - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - - self.assertEqual(round(temporal.calculate_rebalance_factor( - {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, datetime(year=2016, month=02, day=1)), 5), - round(0.2/29, 5)) + temporal.calculate_2d_temporal_factors( + datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), timezones).tolist(), + [[20., 1.], [1., 1.]]) + + # def testing_do_temporal(self): + # import numpy as np + # from hermesv3_gr.modules.grids.grid import Grid + # self.testing_create_netcdf_timezones() + # + # aux_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing' + # + # temporal = TemporalDistribution( + # datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), 'hourly', 1, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # aux_path) + # temporal.monthly_profile = {1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1., + # 7: 1., + # 8: 1., + # 9: 1., + # 10: 1., + # 11: 1., + # 12: 1.} + # temporal.daily_profile_id = {0: 1., + # 1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1.} + # temporal.hourly_profile = {0: 1., + # 1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1., + # 7: 1., + # 8: 1., + # 9: 1., + # 10: 1., + # 11: 1., + # 12: 1., + # 13: 20., + # 14: 1., + # 15: 1., + # 16: 1., + # 17: 1., + # 18: 1., + # 19: 1., + # 20: 1., + # 21: 1., + # 22: 1., + # 23: 1.} + # + # grid = Grid('global', aux_path) + # grid.center_latitudes = np.array([[41.390205, 19.451054], [29.378586, 31.267401]]) + # grid.center_longitudes = np.array([[2.154007, -99.125519], [47.990341, 121.522179]]) + # data_in = [{'data': np.array([[10., 10.], [10., 10.]])}] + # # data_out = [{'data': np.array([[200., 10.], [10., 10.]])}] + # data_out = temporal.do_temporal(data_in, grid) + # + # self.assertEqual(data_out[0]['data'].tolist(), [[[200., 10.], [10., 10.]]]) + # + # def testing_calculate_weekdays_no_leap_year(self): + # from datetime import datetime + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # self.assertEqual(temporal.calculate_weekdays(datetime(year=2017, month=02, day=1)), + # {0: 4, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}) + # + # def testing_calculate_weekdays_leap_year(self): + # from datetime import datetime + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # self.assertEqual(temporal.calculate_weekdays(datetime(year=2016, month=02, day=1)), + # {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}) + # + # def testing_calculate_weekdays_factors_full_month(self): + # from datetime import datetime + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # + # self.assertEqual(round(temporal.calculate_weekday_factor_full_month( + # {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}), 5), + # round(0.2/29, 5)) + # + # def testing_calculate_rebalance_factor(self): + # from datetime import datetime + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # + # self.assertEqual(round(temporal.calculate_rebalance_factor( + # {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, datetime(year=2016, month=02, day=1)), 5), + # round(0.2/29, 5)) # def testing_get_temporal_daily_profile(self): # from datetime import datetime @@ -424,4 +426,3 @@ class TestTemporalDistribution(unittest.TestCase): # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') # # print temporal.get_temporal_daily_profile(date) - -- GitLab From 8a3216fba1165030ace8dd92a87db045c6d6dba7 Mon Sep 17 00:00:00 2001 From: Carles Tena Medina Date: Mon, 10 Sep 2018 19:38:22 +0200 Subject: [PATCH 08/28] corrected python conventions --- conf/hermes.conf | 4 +- hermesv3_gr/modules/writing/writer.py | 2 +- hermesv3_gr/modules/writing/writer_cmaq.py | 14 ++-- hermesv3_gr/modules/writing/writer_monarch.py | 62 +++++++++++++---- .../modules/writing/writer_wrf_chem.py | 68 ++++++++++++++++--- hermesv3_gr/tools/netcdf_tools.py | 54 +++++++-------- 6 files changed, 146 insertions(+), 58 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 36f5cc2..82ed49f 100644 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -5,8 +5,8 @@ input_dir = /home/Earth/ctena/Models/HERMESv3/IN # data_path = /gpfs/scratch/bsc32/bsc32538/HERMES_data data_path = /esarchive/recon #output_dir = /gpfs/projects/bsc32/bsc32538/HERMESv3_GR_rotated/OUT -output_dir = /home/Earth/ctena/Models/HERMESv3/OUT -output_name = HERMES_paralel_.nc +output_dir = /home/carles/HERMES_out +output_name = HERMESv3_.nc start_date = 2014/09/02 00:00:00 # ***** end_date = start_date [DEFAULT] ***** # end_date = 2014/09/03 00:00:00 diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index a110b6c..36d7185 100644 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -420,7 +420,7 @@ class Writer(object): var.grid_mapping = 'mercator' try: var[:] = variable['data'] - except: + except ValueError: print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) # Grid mapping diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 439a6f9..4921417 100644 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -470,7 +470,8 @@ class WriterCmaq(Writer): # Correcting NAN if data is None: data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] = data + var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] = data settings.write_log("\t\t\t'{0}' variable filled".format(variable['name'])) netcdf.close() @@ -484,7 +485,8 @@ class WriterCmaq(Writer): # Gathering the index if mpi_numpy or mpi_vector: - rank_position = np.array([self.grid.x_lower_bound, self.grid.x_upper_bound, self.grid.y_lower_bound, self.grid.y_upper_bound], dtype='i') + rank_position = np.array([self.grid.x_lower_bound, self.grid.x_upper_bound, self.grid.y_lower_bound, + self.grid.y_upper_bound], dtype='i') full_position = None if settings.rank == 0: full_position = np.empty([settings.size, 4], dtype='i') @@ -617,9 +619,13 @@ class WriterCmaq(Writer): for i in xrange(settings.size): # print 'Resizeing {0}'.format(i) if not i == settings.size - 1: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) else: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) else: data = 0 var[:] = data diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index a612eaf..62cc8fb 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -38,6 +38,13 @@ class WriterMonarch(Writer): # } def unit_change(self, variable, data): + # TODO Documentation + """ + + :param variable: + :param data: + :return: + """ from cf_units import Unit st_time = timeit.default_timer() @@ -62,6 +69,11 @@ class WriterMonarch(Writer): return data def create_parallel_netcdf(self): + # TODO Documentation + """ + + :return: + """ from cf_units import Unit, encode_time st_time = timeit.default_timer() @@ -96,7 +108,8 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) lat_dim = ('lon', 'lat', ) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(self.grid.center_latitudes.shape)) + print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format( + len(self.grid.center_latitudes.shape)) sys.exit(1) # Longitude @@ -109,7 +122,8 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[1]), level=3) lon_dim = ('lon', 'lat', ) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(self.grid.center_longitudes.shape)) + print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( + len(self.grid.center_longitudes.shape)) sys.exit(1) elif Rotated: var_dim = ('rlat', 'rlon',) @@ -335,6 +349,12 @@ class WriterMonarch(Writer): settings.write_time('WriterMonarch', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) def write_parallel_netcdf(self, emission_list): + # TODO Documentation + """ + + :param emission_list: + :return: + """ st_time = timeit.default_timer() @@ -358,18 +378,26 @@ class WriterMonarch(Writer): # Correcting NAN if data is None: data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] = data + var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] = data settings.write_log("\t\t\t'{0}' variable filled".format(variable['name'])) if self.grid.cell_area is not None: c_area = netcdf.variables['cell_area'] - c_area[self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] = self.grid.cell_area + c_area[self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] = self.grid.cell_area netcdf.close() settings.write_time('WriterMonarch', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) def write_serial_netcdf(self, emission_list,): + # TODO Documentation + """ + + :param emission_list: + :return: + """ from cf_units import Unit, encode_time st_time = timeit.default_timer() @@ -379,7 +407,8 @@ class WriterMonarch(Writer): # Gathering the index if mpi_numpy or mpi_vector: - rank_position = np.array([self.grid.x_lower_bound, self.grid.x_upper_bound, self.grid.y_lower_bound, self.grid.y_upper_bound], dtype='i') + rank_position = np.array([self.grid.x_lower_bound, self.grid.x_upper_bound, self.grid.y_lower_bound, + self.grid.y_upper_bound], dtype='i') full_position = None if settings.rank == 0: full_position = np.empty([settings.size, 4], dtype='i') @@ -407,11 +436,13 @@ class WriterMonarch(Writer): # Latitude if len(self.grid.center_latitudes.shape) == 1: - settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) + settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) netcdf.createDimension('lat', self.grid.center_latitudes.shape[0]) lat_dim = ('lat',) elif len(self.grid.center_latitudes.shape) == 2: - settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) + settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) netcdf.createDimension('lat', self.grid.center_latitudes.shape[0]) lat_dim = ('lon', 'lat', ) else: @@ -424,11 +455,13 @@ class WriterMonarch(Writer): # Longitude if len(self.grid.center_longitudes.shape) == 1: - settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[0]), level=3) + settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[0]), + level=3) netcdf.createDimension('lon', self.grid.center_longitudes.shape[0]) lon_dim = ('lon',) elif len(self.grid.center_longitudes.shape) == 2: - settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[0]), level=3) + settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[0]), + level=3) netcdf.createDimension('lon', self.grid.center_longitudes.shape[1]) lon_dim = ('lon', 'lat', ) else: @@ -498,7 +531,8 @@ class WriterMonarch(Writer): time = netcdf.createVariable('time', 'd', ('time',)) u = Unit('hours') time.units = str(u.offset_by_time(encode_time( - self.date.year, self.date.month, self.date.day, self.date.hour, self.date.minute, self.date.second))) + self.date.year, self.date.month, self.date.day, self.date.hour, self.date.minute, + self.date.second))) time.standard_name = "time" time.calendar = "gregorian" time.long_name = "time" @@ -695,9 +729,13 @@ class WriterMonarch(Writer): data = np.empty(var[:].shape, dtype=settings.precision) for i in xrange(settings.size): if not i == settings.size - 1: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) else: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) else: data = 0 var[:] = data diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index 0e479d0..e64ed52 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -37,7 +37,7 @@ class WriterWrfChem(Writer): 'TITLE', 'START_DATE', 'WEST-EAST_GRID_DIMENSION', 'SOUTH-NORTH_GRID_DIMENSION', 'BOTTOM-TOP_GRID_DIMENSION', 'DX', 'DY', 'GRIDTYPE', 'DIFF_OPT', 'KM_OPT', 'DAMP_OPT', 'DAMPCOEF', 'KHDIF', 'KVDIF', 'MP_PHYSICS', 'RA_LW_PHYSICS', 'RA_SW_PHYSICS', 'SF_SFCLAY_PHYSICS', 'SF_SURFACE_PHYSICS', - 'BL_PBL_PHYSICS', 'CU_PHYSICS', 'SF_LAKE_PHYSICS', 'SURFACE_INPUT_SOURCE','SST_UPDATE', 'GRID_FDDA', + 'BL_PBL_PHYSICS', 'CU_PHYSICS', 'SF_LAKE_PHYSICS', 'SURFACE_INPUT_SOURCE', 'SST_UPDATE', 'GRID_FDDA', 'GFDDA_INTERVAL_M', 'GFDDA_END_H', 'GRID_SFDDA', 'SGFDDA_INTERVAL_M', 'SGFDDA_END_H', 'WEST-EAST_PATCH_START_UNSTAG', 'WEST-EAST_PATCH_END_UNSTAG', 'WEST-EAST_PATCH_START_STAG', 'WEST-EAST_PATCH_END_STAG', 'SOUTH-NORTH_PATCH_START_UNSTAG', 'SOUTH-NORTH_PATCH_END_UNSTAG', @@ -48,6 +48,13 @@ class WriterWrfChem(Writer): 'MAP_PROJ', 'MMINLU', 'NUM_LAND_CAT', 'ISWATER', 'ISLAKE', 'ISICE', 'ISURBAN', 'ISOILWATER'] def unit_change(self, variable, data): + # TODO Documentation + """ + + :param variable: + :param data: + :return: + """ from cf_units import Unit if data is not None: @@ -74,6 +81,11 @@ class WriterWrfChem(Writer): return data def change_variable_attributes(self): + # TODO Documentation + """ + + :return: + """ from cf_units import Unit new_variable_dict = {} @@ -105,6 +117,11 @@ class WriterWrfChem(Writer): self.variables_attributes = new_variable_dict def read_global_attributes(self): + # TODO Documentation + """ + + :return: + """ import pandas as pd from warnings import warn as warning @@ -183,7 +200,8 @@ class WriterWrfChem(Writer): except ValueError: print 'A warning has occurred. Check the .err file to get more information.' if settings.rank == 0: - warning('The global attribute {0} is not defined; Using default value {1}'.format(att, atts_dict[att])) + warning('The global attribute {0} is not defined; Using default value {1}'.format( + att, atts_dict[att])) else: settings.write_log('WARNING: Check the .err file to get more information.') @@ -197,6 +215,7 @@ class WriterWrfChem(Writer): return atts_dict def create_global_attributes(self): + # TODO Documentation """ Creates the global attributes that have to be filled. """ @@ -247,6 +266,11 @@ class WriterWrfChem(Writer): return global_attributes def create_times_var(self): + # TODO Documentation + """ + + :return: + """ from datetime import timedelta import netCDF4 @@ -260,6 +284,11 @@ class WriterWrfChem(Writer): return str_out def create_parallel_netcdf(self): + # TODO Documentation + """ + + :return: + """ st_time = timeit.default_timer() settings.write_log("\tCreating parallel NetCDF file.", level=2) netcdf = Dataset(self.path, mode='w', format="NETCDF4") @@ -268,13 +297,15 @@ class WriterWrfChem(Writer): # ===== Dimensions ===== settings.write_log("\t\tCreating NetCDF dimensions.", level=2) netcdf.createDimension('Time', None) - settings.write_log("\t\t\t'Time' dimension: {0}".format('UNLIMITED ({0})'.format(len(self.hours))), level=3) + settings.write_log("\t\t\t'Time' dimension: {0}".format('UNLIMITED ({0})'.format(len(self.hours))), + level=3) netcdf.createDimension('DateStrLen', 19) settings.write_log("\t\t\t'DateStrLen' dimension: 19", level=3) netcdf.createDimension('west_east', self.grid.center_longitudes.shape[1]) settings.write_log("\t\t\t'west_east' dimension: {0}".format(len(self.hours)), level=3) netcdf.createDimension('south_north', self.grid.center_latitudes.shape[0]) - settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) + settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) netcdf.createDimension('emissions_zdim', len(self.levels)) settings.write_log("\t\t\t'emissions_zdim' dimension: {0}".format(len(self.levels)), level=3) @@ -305,6 +336,12 @@ class WriterWrfChem(Writer): settings.write_time('WriterCmaq', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) def write_parallel_netcdf(self, emission_list): + # TODO Documentation + """ + + :param emission_list: + :return: + """ st_time = timeit.default_timer() settings.write_log("\tAppending data to parallel NetCDF file.", level=2) @@ -325,13 +362,20 @@ class WriterWrfChem(Writer): # Correcting NAN if data is None: data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] = data + var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] = data settings.write_log("\t\t\t'{0}' variable filled".format(var_name)) netcdf.close() settings.write_time('WriterCmaq', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) def write_serial_netcdf(self, emission_list): + # TODO Documentation + """ + + :param emission_list: + :return: + """ st_time = timeit.default_timer() # Gathering the index @@ -356,7 +400,8 @@ class WriterWrfChem(Writer): netcdf.createDimension('west_east', self.grid.center_longitudes.shape[1]) settings.write_log("\t\t\t'west_east' dimension: {0}".format(len(self.hours)), level=3) netcdf.createDimension('south_north', self.grid.center_latitudes.shape[0]) - settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) + settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) netcdf.createDimension('emissions_zdim', len(self.levels)) settings.write_log("\t\t\t'emissions_zdim' dimension: {0}".format(len(self.levels)), level=3) @@ -409,7 +454,8 @@ class WriterWrfChem(Writer): st_time = timeit.default_timer() index += 1 - var = netcdf.createVariable(var_name, 'f', ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=self.compress) + var = netcdf.createVariable(var_name, 'f', ('Time', 'emissions_zdim', 'south_north', 'west_east',), + zlib=self.compress) var.setncatts(self.variables_attributes[var_name]) var_time = timeit.default_timer() @@ -421,9 +467,13 @@ class WriterWrfChem(Writer): for i in xrange(settings.size): # print 'Resizeing {0}'.format(i) if not i == settings.size - 1: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) else: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) else: data = 0 var[:] = data diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index d094233..6a50c73 100644 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -152,15 +152,11 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # Bounds if boundary_latitudes is not None: - # print boundary_latitudes.shape - # print len(boundary_latitudes[0, 0]) try: netcdf.createDimension('nv', len(boundary_latitudes[0, 0])) except TypeError: netcdf.createDimension('nv', boundary_latitudes.shape[1]) - # sys.exit() - # Time netcdf.createDimension('time', None) @@ -178,7 +174,8 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, u = Unit('hours') # print u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second)) # Unit('hour since 1970-01-01 00:00:00.0000000 UTC') - time.units = str(u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second))) + time.units = str(u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, + date.second))) time.standard_name = "time" time.calendar = "gregorian" time.long_name = "time" @@ -276,7 +273,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, var.grid_mapping = 'mercator' try: var[:] = variable['data'] - except: + except ValueError: print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) # Grid mapping @@ -300,7 +297,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, mapping.longitude_of_central_meridian = lon_0 mapping.latitude_of_projection_origin = lat_0 elif Mercator: - #Mercator + # Mercator mapping = netcdf.createVariable('mercator', 'i') mapping.grid_mapping_name = "mercator" mapping.longitude_of_projection_origin = lon_0 @@ -322,23 +319,23 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, - levels=None, date=None, hours=None, - boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, - RegularLatLon=False, - Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, - LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): + levels=None, date=None, hours=None, + boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, + regular_latlon=False, + rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, + lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): from cf_units import Unit, encode_time import sys from netCDF4 import Dataset import numpy as np - if not (RegularLatLon or LambertConformalConic or Rotated): - RegularLatLon = True + if not (regular_latlon or lcc or rotated): + regular_latlon = True netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -362,7 +359,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, else: print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) sys.exit(1) - elif Rotated: + elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude @@ -379,7 +376,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic: + elif lcc: var_dim = ('y', 'x',) netcdf.createDimension('y', len(lcc_y)) @@ -416,7 +413,8 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, u = Unit('hours') # print u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second)) # Unit('hour since 1970-01-01 00:00:00.0000000 UTC') - time.units = str(u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second))) + time.units = str(u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, + date.second))) time.standard_name = "time" time.calendar = "gregorian" time.long_name = "time" @@ -450,7 +448,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes - if Rotated: + if rotated: # Rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=True) rlat.long_name = "latitude in rotated pole grid" @@ -464,7 +462,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = rotated_lons - if LambertConformalConic: + if lcc: x = netcdf.createVariable('x', 'd', ('x',), zlib=True) x.units = Unit("km").symbol x.long_name = "x coordinate of projection" @@ -504,11 +502,11 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, var.coordinates = "lat lon" if cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif rotated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' # print 'HOURSSSSSSSSSSSSSSSSSSSSS:', hours # if variable['data'] is not 0: @@ -520,19 +518,19 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, var[:] = np.zeros(shape) # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: + elif rotated: # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = north_pole_lat mapping.grid_north_pole_longitude = north_pole_lon - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" @@ -576,7 +574,3 @@ def calculate_displacements(counts): if __name__ == '__main__': pass - - - - -- GitLab From 3e913e13184590a5aead2e03fd569d14916a7578 Mon Sep 17 00:00:00 2001 From: Carles Tena Medina Date: Mon, 10 Sep 2018 19:51:11 +0200 Subject: [PATCH 09/28] changed max-line-lenggth for Codacy --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 4094c1f..db7741b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -99,7 +99,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme [FORMAT] # Maximum number of characters on a single line. -max-line-length=79 +max-line-length=120 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ -- GitLab From f7e2073c773d286a17ac8317d048c3bf86930887 Mon Sep 17 00:00:00 2001 From: Carles Tena Medina Date: Mon, 10 Sep 2018 20:02:29 +0200 Subject: [PATCH 10/28] Correcting Code conventions --- hermesv3_gr/modules/grids/grid_rotated.py | 3 ++- hermesv3_gr/modules/masking/masking.py | 4 ++-- hermesv3_gr/modules/temporal/temporal.py | 8 ++++---- hermesv3_gr/modules/writing/writer.py | 2 +- hermesv3_gr/modules/writing/writer_cmaq.py | 4 ++-- hermesv3_gr/modules/writing/writer_monarch.py | 4 ++-- hermesv3_gr/modules/writing/writer_wrf_chem.py | 4 ++-- hermesv3_gr/tools/coordinates_tools.py | 2 -- 8 files changed, 15 insertions(+), 16 deletions(-) diff --git a/hermesv3_gr/modules/grids/grid_rotated.py b/hermesv3_gr/modules/grids/grid_rotated.py index 02dc6e4..a21fdbc 100644 --- a/hermesv3_gr/modules/grids/grid_rotated.py +++ b/hermesv3_gr/modules/grids/grid_rotated.py @@ -17,9 +17,10 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import timeit + import sys import os +import timeit import hermesv3_gr.config.settings as settings from grid import Grid diff --git a/hermesv3_gr/modules/masking/masking.py b/hermesv3_gr/modules/masking/masking.py index 98c113c..068c819 100644 --- a/hermesv3_gr/modules/masking/masking.py +++ b/hermesv3_gr/modules/masking/masking.py @@ -17,10 +17,10 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import timeit -import hermesv3_gr.config.settings as settings import os +import timeit +import hermesv3_gr.config.settings as settings from warnings import warn as warning diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 98fd6d3..4f6909f 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import timeit -import hermesv3_gr.config.settings as settings import os import sys +import timeit +import hermesv3_gr.config.settings as settings import numpy as np @@ -86,7 +86,7 @@ class TemporalDistribution(object): self.timestep_num = timestep_num self.timestep_freq = timestep_freq - self.ending_date = self.calculate_ending_date() + self.ending_date = self.calculates_ending_date() if month_profile_id is not None: if len(month_profile_id) > 4: if os.path.exists(month_profile_id): @@ -142,7 +142,7 @@ class TemporalDistribution(object): settings.write_time('TemporalDistribution', 'Init', timeit.default_timer() - st_time, level=2) - def calculate_ending_date(self): + def calculates_ending_date(self): """ Calculates the date of the last timestep. diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index 36d7185..7fd70d8 100644 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -18,9 +18,9 @@ # along with HERMESv3_GR. If not, see . +import sys import timeit from hermesv3_gr.config import settings -import sys class Writer(object): diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 4921417..7da5172 100644 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -18,11 +18,11 @@ # along with HERMESv3_GR. If not, see . +import os +import sys from hermesv3_gr.modules.writing.writer import Writer import timeit from hermesv3_gr.config import settings -import os -import sys import numpy as np from netCDF4 import Dataset from mpi4py import MPI diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index 62cc8fb..98831ad 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -18,11 +18,11 @@ # along with HERMESv3_GR. If not, see . +import os +import sys from hermesv3_gr.modules.writing.writer import Writer import timeit from hermesv3_gr.config import settings -import os -import sys import numpy as np from netCDF4 import Dataset from mpi4py import MPI diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index e64ed52..6711880 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -18,11 +18,11 @@ # along with HERMESv3_GR. If not, see . +import os +import sys from hermesv3_gr.modules.writing.writer import Writer import timeit from hermesv3_gr.config import settings -import os -import sys import numpy as np from netCDF4 import Dataset from mpi4py import MPI diff --git a/hermesv3_gr/tools/coordinates_tools.py b/hermesv3_gr/tools/coordinates_tools.py index 20a4d12..51d074a 100644 --- a/hermesv3_gr/tools/coordinates_tools.py +++ b/hermesv3_gr/tools/coordinates_tools.py @@ -19,8 +19,6 @@ import os import sys -# import numpy as np -# import math # Global variables -- GitLab From 47c8420c83c07f0443d6300466402a28b9f9fd4a Mon Sep 17 00:00:00 2001 From: Carles Tena Medina Date: Mon, 10 Sep 2018 21:08:16 +0200 Subject: [PATCH 11/28] Correcting Code conventions --- hermesv3_gr/modules/temporal/temporal.py | 137 ++++++----- preproc/ceds_preproc.py | 52 ++-- preproc/eclipsev5a_preproc.py | 48 ++-- preproc/edgarv432_ap_preproc.py | 54 ++--- preproc/edgarv432_voc_preproc.py | 52 ++-- preproc/emep_preproc.py | 20 +- preproc/gfas12_preproc.py | 22 +- preproc/htapv2_preproc.py | 277 +++++++++++++--------- preproc/tno_mac_iii_preproc.py | 28 +-- preproc/tno_mac_iii_preproc_voc_ratios.py | 140 ++++++++--- preproc/wiedinmyer_preproc.py | 20 +- 11 files changed, 493 insertions(+), 357 deletions(-) diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 4f6909f..5da069d 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -67,7 +67,6 @@ class TemporalDistribution(object): timezones. :type auxiliar_files_dir: str """ - def __init__(self, starting_date, timestep_type, timestep_num, timestep_freq, monthly_profile_path, month_profile_id, daily_profile_path, daily_profile_id, hourly_profile_path, hourly_profile_id, world_info_path, auxiliar_files_dir, grid): @@ -86,7 +85,7 @@ class TemporalDistribution(object): self.timestep_num = timestep_num self.timestep_freq = timestep_freq - self.ending_date = self.calculates_ending_date() + self.ending_date = self.calculate_ending_date() if month_profile_id is not None: if len(month_profile_id) > 4: if os.path.exists(month_profile_id): @@ -142,9 +141,9 @@ class TemporalDistribution(object): settings.write_time('TemporalDistribution', 'Init', timeit.default_timer() - st_time, level=2) - def calculates_ending_date(self): + def calculate_ending_date(self): """ - Calculates the date of the last timestep. + Calculate the date of the last timestep. :return: Date of the last timestep :rtype: datetime.datetime @@ -174,7 +173,7 @@ class TemporalDistribution(object): def calculate_timedelta(self, date): """ - Calculates the difference of time to the next timestep. + Calculate the difference of time to the next timestep. :param date: Date of the current timestep. :type date: datetime.datetime @@ -208,7 +207,7 @@ class TemporalDistribution(object): def get_tz_from_id(self, tz_id): """ - Extracts the timezone (string format) for the given id (int). + Extract the timezone (string format) for the given id (int). :param tz_id: ID of the timezone. :type tz_id: int @@ -216,14 +215,13 @@ class TemporalDistribution(object): :return: Timezone :rtype: str """ - tz = self.world_info_df.time_zone[self.world_info_df.time_zone_code == tz_id].values return tz[0] def get_id_from_tz(self, tz): """ - Extracts the id (int) for the given timezone (string format). + Extract the id (int) for the given timezone (string format). :param tz: Timezone of the ID. :type tz: str @@ -231,7 +229,6 @@ class TemporalDistribution(object): :return: ID :rtype: int """ - tz_id = self.world_info_df.time_zone_code[self.world_info_df.time_zone == tz].values try: @@ -247,9 +244,11 @@ class TemporalDistribution(object): return tz_id @staticmethod - def parse_tz(tz): + def parse_tz(timezone): """ - Parses the timezone (string format). It is needed because some libraries have more timezones than others and it + Parse the timezone (string format). + + It is needed because some libraries have more timezones than others and it tries to simplify setting the strange ones into the nearest common one. Examples: 'America/Punta_Arenas': 'America/Santiago', @@ -262,8 +261,8 @@ class TemporalDistribution(object): 'Asia/Tomsk': 'Asia/Novokuznetsk', 'America/Fort_Nelson': 'America/Vancouver' - :param tz: Not parsed timezone. - :type tz: str + :param timezone: Not parsed timezone. + :type timezone: str :return: Parsed timezone :rtype: str @@ -281,14 +280,14 @@ class TemporalDistribution(object): 'Asia/Famagusta': 'Asia/Nicosia', } - if tz in tz_dict.iterkeys(): - tz = tz_dict[tz] + if timezone in tz_dict.iterkeys(): + timezone = tz_dict[timezone] - return tz + return timezone def find_closest_timezone(self, latitude, longitude): """ - Finds the closest timezone for the given coordinates. + Find the closest timezone for the given coordinates. :param latitude: Latitude coordinate to find timezone. :type latitude: float @@ -299,22 +298,21 @@ class TemporalDistribution(object): :return: Nearest timezone of the given coordinates. :rtype: str """ - st_time = timeit.default_timer() - dg = 0 - tz = None - while tz is None: - tz = self.tf.closest_timezone_at(lng=longitude, lat=latitude, delta_degree=dg) - dg += 1 + degrees = 0 + timezone = None + while timezone is None: + timezone = self.tf.closest_timezone_at(lng=longitude, lat=latitude, delta_degree=degrees) + degrees += 1 settings.write_time('TemporalDistribution', 'find_closest_timezone', timeit.default_timer() - st_time, level=3) - return tz + return timezone def is_created_netcdf_timezones(self): """ - Checks if the NetCDF of timezones is created + Check if the NetCDF of timezones is created :return: True if it is already created. :rtype: bool @@ -323,7 +321,7 @@ class TemporalDistribution(object): def create_netcdf_timezones(self, grid): """ - Creates a NetCDF with the timezones in the resolution of the given grid. + Create a NetCDF with the timezones in the resolution of the given grid. :param grid: Grid object with the coordinates. :type grid: Grid @@ -344,20 +342,17 @@ class TemporalDistribution(object): num = 0 points = zip(lat.flatten(), lon.flatten()) - # points = points[534000:] - # print len(points) + for lat_aux, lon_aux in points: num += 1 settings.write_log("\t\t\tlat:{0}, lon:{1} ({2}/{3})".format(lat_aux, lon_aux, num, len(points)), level=3) - tz = self.find_closest_timezone(lat_aux, lon_aux) - tz_id = self.get_id_from_tz(tz) + timezone = self.find_closest_timezone(lat_aux, lon_aux) + tz_id = self.get_id_from_tz(timezone) dst_var.append(tz_id) dst_var = np.array(dst_var) dst_var = dst_var.reshape((1,) + lat.shape) dst_var = settings.comm.gather(dst_var, root=0) if settings.rank == 0: - for var in dst_var: - print var.shape total_lat = np.concatenate(total_lat, axis=1) total_lon = np.concatenate(total_lon, axis=1) dst_var = np.concatenate(dst_var, axis=2) @@ -372,6 +367,13 @@ class TemporalDistribution(object): return True def read_gridded_profile(self, path, value): + # TODO Documentation + """ + + :param path: + :param value: + :return: + """ from netCDF4 import Dataset st_time = timeit.default_timer() @@ -391,7 +393,7 @@ class TemporalDistribution(object): def calculate_timezones(self): """ - Extracts the timezones ID's from the NetCDF and convert them to the timezone (str). + Calculate the timezones ID's from the NetCDF and convert them to the timezone (str). :return: Array with the timezone of each cell. :rtype: numpy.chararray @@ -408,8 +410,8 @@ class TemporalDistribution(object): tz_list = np.chararray(timezones.shape, itemsize=32) for id_aux in xrange(timezones.min(), timezones.max() + 1): try: - tz = self.get_tz_from_id(id_aux) - tz_list[timezones == id_aux] = tz + timezone = self.get_tz_from_id(id_aux) + tz_list[timezones == id_aux] = timezone except: pass settings.write_time('TemporalDistribution', 'calculate_timezones', timeit.default_timer() - st_time, level=3) @@ -418,7 +420,7 @@ class TemporalDistribution(object): def calculate_2d_temporal_factors(self, date): """ - Calculates the temporal factor to correct the input data of the given date for each cell. + Calculate the temporal factor to correct the input data of the given date for each cell. :param date: Date of the current timestep. :type date: datetime.datetime @@ -449,7 +451,7 @@ class TemporalDistribution(object): df['hour'] = df.index.hour if self.hourly_profile is not None: - if type(self.hourly_profile) is dict: + if isinstance(self.hourly_profile, dict): df['hour_factor'] = df['hour'].map(self.hourly_profile) else: profile_ids = self.parse_hourly_profile_id() @@ -481,13 +483,13 @@ class TemporalDistribution(object): if self.monthly_profile is None: df['month_factor'] = 1 - elif type(self.monthly_profile) == dict: + elif isinstance(self.monthly_profile, dict): df['month_factor'] = df['month'].map(self.monthly_profile) - elif type(self.monthly_profile) == np.ndarray: + elif isinstance(self.monthly_profile, np.ndarray): for m, df_aux in df.groupby('month'): try: df.loc[df['month'] == m, 'month_factor'] = \ - self.monthly_profile[m-1, df.loc[df['month'] == m, 'i'].values] + self.monthly_profile[m - 1, df.loc[df['month'] == m, 'i'].values] except IndexError: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -511,7 +513,7 @@ class TemporalDistribution(object): def calculate_3d_temporal_factors(self): """ - Calculates the temporal factor to correct the input data of the given date for each cell. + Calculate the temporal factor to correct the input data of the given date for each cell. :return: 3D array with the factors to correct the input data to the date of this timestep. :rtype: numpy.array @@ -520,19 +522,19 @@ class TemporalDistribution(object): settings.write_log("\tCalculating temporal factors.", level=2) factors = [] - date_aux = self.starting_date + date = self.starting_date count = 0 - while date_aux <= self.ending_date: + while date <= self.ending_date: count += 1 settings.write_log("\t\t{0} temporal factor ({1}/{2}).".format( - date_aux.strftime('%Y/%m/%d %H:%M:%S'), count, self.timestep_num), level=3) + date.strftime('%Y/%m/%d %H:%M:%S'), count, self.timestep_num), level=3) - factors.append(self.calculate_2d_temporal_factors(date_aux)) + factors.append(self.calculate_2d_temporal_factors(date)) - d = date_aux - self.starting_date - self.hours_since.append(d.seconds / 3600 + d.days * 24) # 3600 seconds per hour - date_aux = date_aux + self.calculate_timedelta(date_aux) + date_aux = date - self.starting_date + self.hours_since.append(date_aux.seconds / 3600 + date_aux.days * 24) # 3600 seconds per hour + date = date + self.calculate_timedelta(date) factors = np.array(factors) @@ -542,7 +544,8 @@ class TemporalDistribution(object): def parse_hourly_profile_id(self): """ - Parses the hourly profile ID to get a dictionary with the ID for "weekday", "saturday" and "sunday" + Parse the hourly profile ID to get a dictionary with the ID for "weekday", "saturday" and "sunday" + :return: """ import re @@ -557,7 +560,8 @@ class TemporalDistribution(object): def get_temporal_hourly_profile(self, profile_id, date=None): """ - Extracts the hourly profile of the given ID in a dictionary format. + Extract the hourly profile of the given ID in a dictionary format. + The hour (0 to 23) is the key (int) and the value (float) is the factor. :param profile_id: ID of the hourly profile to use. @@ -572,8 +576,6 @@ class TemporalDistribution(object): import pandas as pd st_time = timeit.default_timer() - # settings.write_log("\t\t\tGetting temporal hourly profile '{0}' from {1} .".format( - # profile_id, self.hourly_profile_path), level=3) if date is None: df = pd.read_csv(self.hourly_profile_path) try: @@ -587,7 +589,6 @@ class TemporalDistribution(object): profile.pop('TP_H', None) profile = {int(k): float(v) for k, v in profile.items()} else: - # print self.hourly_profile profile = None settings.write_time('TemporalDistribution', 'get_temporal_hourly_profile', timeit.default_timer() - st_time, level=3) @@ -596,7 +597,8 @@ class TemporalDistribution(object): def get_temporal_daily_profile(self, date): """ - Extracts the daily profile of the given ID in a dictionary format. + Extract the daily profile of the given ID in a dictionary format. + The weekday (0 to 6) is the key (int) and the value (float) is the factor. :param date: Date of the timestep to simulate. @@ -633,8 +635,10 @@ class TemporalDistribution(object): def calculate_rebalance_factor(self, profile, date): """ - Calculates the necessary factor make consistent the full month data. This is needed for the months that if you - sum the daily factor of each day of the month it doesn't sum as the number of days of the month. + Calculate the necessary factor make consistent the full month data. + + This is needed for the months that if you sum the daily factor of each day of the month it doesn't sum as + the number of days of the month. :param profile: Daily profile. :type profile: dict @@ -657,8 +661,9 @@ class TemporalDistribution(object): @staticmethod def calculate_weekday_factor_full_month(profile, weekdays): + # TODO Documentation """ - Operates with all the days of the month to get the sum of daily factors of the full month. + Operate with all the days of the month to get the sum of daily factors of the full month. :param profile: :param weekdays: @@ -679,6 +684,12 @@ class TemporalDistribution(object): @staticmethod def calculate_weekdays(date): + # TODO Documentation + """ + + :param date: + :return: + """ from calendar import monthrange, weekday, MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY st_time = timeit.default_timer() @@ -698,7 +709,8 @@ class TemporalDistribution(object): @staticmethod def get_temporal_monthly_profile(profile_path, profile_id): """ - Extracts the monthly profile of the given ID in a dictionary format. + Extract the monthly profile of the given ID in a dictionary format. + The month (1 to 12) is the key (int) and the value (float) is the factor. :param profile_path: Path to the file that contains all the monthly profiles. @@ -739,6 +751,15 @@ class TemporalDistribution(object): @staticmethod def calculate_delta_hours(st_date, time_step_type, time_step_num, time_step_freq): + # TODO Documentation + """ + + :param st_date: + :param time_step_type: + :param time_step_num: + :param time_step_freq: + :return: + """ from datetime import timedelta from calendar import monthrange, isleap diff --git a/preproc/ceds_preproc.py b/preproc/ceds_preproc.py index 068c261..1b13729 100755 --- a/preproc/ceds_preproc.py +++ b/preproc/ceds_preproc.py @@ -23,20 +23,20 @@ import sys # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/jgcri/ceds/original_files' -output_path = '/esarchive/recon/jgcri/ceds' -list_pollutants = ['BC', 'CO', 'NH3', 'NMVOC', 'NOx', 'OC', 'SO2'] -voc_pollutants = ['VOC01', 'VOC02', 'VOC03', 'VOC04', 'VOC05', 'VOC06', 'VOC07', 'VOC08', 'VOC09', 'VOC12', 'VOC13', +INPUT_PATH = '/esarchive/recon/jgcri/ceds/original_files' +OUTPUT_PATH = '/esarchive/recon/jgcri/ceds' +LIST_POLLUTANTS = ['BC', 'CO', 'NH3', 'NMVOC', 'NOx', 'OC', 'SO2'] +VOC_POLLUTANTS = ['VOC01', 'VOC02', 'VOC03', 'VOC04', 'VOC05', 'VOC06', 'VOC07', 'VOC08', 'VOC09', 'VOC12', 'VOC13', 'VOC14', 'VOC15', 'VOC16', 'VOC17', 'VOC18', 'VOC19', 'VOC20', 'VOC21', 'VOC22', 'VOC23', 'VOC24', 'VOC25'] -list_sectors = ['agriculture', 'energy', 'industry', 'transport', 'residential', 'solvents', 'waste', 'ships'] -# list_years = from 1950 to 2014 -list_years = [2010] -input_name = '-em-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim_gr_01-12.nc' -voc_input_name = '-em-speciated-VOC_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim-supplemental-data_gr_01-12.nc' -do_air = True -air_input_name = '-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26_gr_01-12.nc' +LIST_SECTORS = ['agriculture', 'energy', 'industry', 'transport', 'residential', 'solvents', 'waste', 'ships'] +# LIST_YEARS = from 1950 to 2014 +LIST_YEARS = [2010] +INPUT_NAME = '-em-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim_gr_01-12.nc' +VOC_INPUT_NAME = '-em-speciated-VOC_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim-supplemental-data_gr_01-12.nc' +DO_AIR = True +AIR_INPUT_NAME = '-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26_gr_01-12.nc' # ============================================================== @@ -120,11 +120,11 @@ def get_input_name(pollutant, year, air=False): :rtype: str """ if air: - file_name = air_input_name.replace('', pollutant) - elif pollutant in list_pollutants: - file_name = input_name.replace('', pollutant) - elif pollutant in voc_pollutants: - file_name = voc_input_name.replace('', '{0}-{1}'.format(pollutant, voc_to_vocname(pollutant))) + file_name = AIR_INPUT_NAME.replace('', pollutant) + elif pollutant in LIST_POLLUTANTS: + file_name = INPUT_NAME.replace('', pollutant) + elif pollutant in VOC_POLLUTANTS: + file_name = VOC_INPUT_NAME.replace('', '{0}-{1}'.format(pollutant, voc_to_vocname(pollutant))) else: raise ValueError('Pollutant {0} not in pollutant list or voc list'.format(pollutant)) @@ -139,7 +139,7 @@ def get_input_name(pollutant, year, air=False): else: file_name = file_name.replace('', str(2000)).replace('', str(2014)) - return os.path.join(input_path, file_name) + return os.path.join(INPUT_PATH, file_name) def get_full_year_data(file_name, pollutant, sector, year, air=False): @@ -179,9 +179,9 @@ def get_full_year_data(file_name, pollutant, sector, year, air=False): i_time = np.where(time_array == datetime(year=year, month=1, day=1))[0][0] if air: data = nc.variables['AIR'][i_time:i_time + 12, :, :, :] - elif pollutant in list_pollutants: + elif pollutant in LIST_POLLUTANTS: data = nc.variables['{0}_em_anthro'.format(pollutant)][i_time:i_time+12, sector_to_index(sector), :, :] - elif pollutant in voc_pollutants: + elif pollutant in VOC_POLLUTANTS: data = nc.variables['{0}-{1}_em_speciated_VOC'.format( pollutant, voc_to_vocname(pollutant).replace('-', '_'))][i_time:i_time+12, sector_to_index(sector), :, :] else: @@ -222,14 +222,14 @@ def do_transformation(year): """ from datetime import datetime from hermesv3_gr.tools.netcdf_tools import extract_vars, get_grid_area, write_netcdf - for pollutant in list_pollutants + voc_pollutants: + for pollutant in LIST_POLLUTANTS + VOC_POLLUTANTS: file_name = get_input_name(pollutant, year) if os.path.exists(file_name): c_lats, c_lons, b_lats, b_lons = extract_vars(file_name, ['lat', 'lon', 'lat_bnds', 'lon_bnds']) cell_area = get_grid_area(file_name) global_attributes = get_global_attributes(file_name) - for sector in list_sectors: + for sector in LIST_SECTORS: data = get_full_year_data(file_name, pollutant, sector, year) if pollutant == 'NOx': @@ -237,7 +237,7 @@ def do_transformation(year): else: pollutant_name = pollutant.lower() - file_path = os.path.join(output_path, 'monthly_mean', '{0}_{1}'.format(pollutant_name, sector)) + file_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(pollutant_name, sector)) if not os.path.exists(file_path): os.makedirs(file_path) @@ -267,7 +267,7 @@ def do_air_transformation(year): from datetime import datetime from hermesv3_gr.tools.netcdf_tools import extract_vars, get_grid_area, write_netcdf - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: file_name = get_input_name(pollutant, year, air=True) if os.path.exists(file_name): c_lats, c_lons, b_lats, b_lons = extract_vars(file_name, ['lat', 'lon', 'lat_bnds', 'lon_bnds']) @@ -283,7 +283,7 @@ def do_air_transformation(year): pollutant_name = pollutant.lower() for sector in ['air_lto', 'air_cds', 'air_crs']: - file_path = os.path.join(output_path, 'monthly_mean', '{0}_{1}'.format(pollutant_name, sector)) + file_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(pollutant_name, sector)) if not os.path.exists(file_path): os.makedirs(file_path) @@ -314,7 +314,7 @@ def do_air_transformation(year): if __name__ == '__main__': - for y in list_years: + for y in LIST_YEARS: # do_transformation(y) - if do_air: + if DO_AIR: do_air_transformation(y) diff --git a/preproc/eclipsev5a_preproc.py b/preproc/eclipsev5a_preproc.py index 6a359b5..157dd81 100755 --- a/preproc/eclipsev5a_preproc.py +++ b/preproc/eclipsev5a_preproc.py @@ -26,14 +26,14 @@ from cf_units import Unit # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/iiasa/eclipsev5a/original_files' -output_path = '/esarchive/recon/iiasa/eclipsev5a/original_files/test' -input_name = 'ECLIPSE_base_CLE_V5a_.nc' -input_name_flaring = 'ECLIPSE_V5a_baseline_CLE_flaring.nc' -input_name_ship = "ship_CLE_emis_.nc" -monthly_pattern_file = 'ECLIPSEv5_monthly_patterns.nc' -list_years = [1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030, 2040, 2050] -list_pollutants = ['BC', 'CH4', 'CO', 'NH3', 'NOx', 'OC', 'OM', 'PM10', 'PM25', 'SO2', 'VOC'] +INPUT_PATH = '/esarchive/recon/iiasa/eclipsev5a/original_files' +OUTPUT_PATH = '/esarchive/recon/iiasa/eclipsev5a/original_files/test' +INPUT_NAME = 'ECLIPSE_base_CLE_V5a_.nc' +INPUT_NAME_FLARING = 'ECLIPSE_V5a_baseline_CLE_flaring.nc' +INPUT_NAME_SHIPS = "ship_CLE_emis_.nc" +MONTHLY_PATTERN_FILE = 'ECLIPSEv5_monthly_patterns.nc' +LIST_YEARS = [1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030, 2040, 2050] +LIST_POLLUTANTS = ['BC', 'CH4', 'CO', 'NH3', 'NOx', 'OC', 'OM', 'PM10', 'PM25', 'SO2', 'VOC'] # ============================================================== @@ -207,7 +207,7 @@ def extract_month_profile_by_sector(sector, month, pollutant=None): else: profile_name = sector_dict[sector] - nc_profiles = Dataset(os.path.join(input_path, monthly_pattern_file), mode='r') + nc_profiles = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE), mode='r') profile = nc_profiles.variables[profile_name][month, :, :] @@ -219,7 +219,7 @@ def extract_month_profile_by_sector(sector, month, pollutant=None): def get_output_name(pollutant, sector, year, month): # TODO Docuemtnation - output_path_aux = os.path.join(output_path, 'monthly_mean', '{0}_{1}'.format(pollutant, sector), ) + output_path_aux = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(pollutant, sector), ) if not(os.path.exists(output_path_aux)): os.makedirs(output_path_aux) @@ -229,7 +229,7 @@ def get_output_name(pollutant, sector, year, month): def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area): # TODO Docuemtnation - for i in xrange(len(list_years)): + for i in xrange(len(LIST_YEARS)): for month in xrange(12): # print i, list_years[i], month + 1 @@ -239,7 +239,7 @@ def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area) pollutant_name = 'nmvoc' else: pollutant_name = pollutant.lower() - output_name = get_output_name(pollutant_name.lower(), sector.lower(), list_years[i], month + 1) + output_name = get_output_name(pollutant_name.lower(), sector.lower(), LIST_YEARS[i], month + 1) profile = extract_month_profile_by_sector(sector, month, pollutant) data_aux = data[i, :, :] * profile # print factor @@ -256,13 +256,13 @@ def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area) 'units': Unit(var_units), }] write_netcdf(output_name, data_list, c_lats, c_lons, cell_area, - datetime(year=list_years[i], month=month + 1, day=1)) + datetime(year=LIST_YEARS[i], month=month + 1, day=1)) def do_transformation(): # TODO Documentation - for pollutant in list_pollutants: - file_name = os.path.join(input_path, input_name.replace('', pollutant)) + for pollutant in LIST_POLLUTANTS: + file_name = os.path.join(INPUT_PATH, INPUT_NAME.replace('', pollutant)) print file_name nc = Dataset(file_name, mode='r') c_lats = nc.variables['lat'][:] @@ -278,7 +278,7 @@ def do_transformation(): def get_flaring_output_name(pollutant, sector, year): # TODO Docuemtnation - output_path_aux = os.path.join(output_path, 'yearly_mean', '{0}_{1}'.format(pollutant, sector), ) + output_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', '{0}_{1}'.format(pollutant, sector), ) if not(os.path.exists(output_path_aux)): os.makedirs(output_path_aux) @@ -309,17 +309,17 @@ def get_flaring_var_name(nc_var): def do_flaring_transformation(): # TODO Documentation - nc_in = Dataset(os.path.join(input_path, input_name_flaring), mode='r') + nc_in = Dataset(os.path.join(INPUT_PATH, INPUT_NAME_FLARING), mode='r') c_lats = nc_in.variables['lat'][:] c_lons = nc_in.variables['lon'][:] - cell_area = get_grid_area(os.path.join(input_path, input_name_flaring)) + cell_area = get_grid_area(os.path.join(INPUT_PATH, INPUT_NAME_FLARING)) for var in nc_in.variables: var_name = get_flaring_var_name(var) if var_name is not None: data = nc_in.variables[var][:] data = np.nan_to_num(data) - for i in xrange(len(list_years)): - output_name = get_flaring_output_name(var_name, 'flaring', list_years[i]) + for i in xrange(len(LIST_YEARS)): + output_name = get_flaring_output_name(var_name, 'flaring', LIST_YEARS[i]) data_aux = data[i, :, :] data_aux = (data_aux * year_factor) / cell_area data_aux = data_aux.reshape((1,) + data_aux.shape) @@ -330,13 +330,13 @@ def do_flaring_transformation(): 'units': Unit(var_units), }] write_netcdf(output_name, data_list, c_lats, c_lons, cell_area, - datetime(year=list_years[i], month=1, day=1)) + datetime(year=LIST_YEARS[i], month=1, day=1)) nc_in.close() def get_ship_output_name(pollutant, sector, year): # TODO Docuemntation - output_path_aux = os.path.join(output_path, 'yearly_mean', '{0}_{1}'.format(pollutant, sector), ) + output_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', '{0}_{1}'.format(pollutant, sector), ) if not(os.path.exists(output_path_aux)): os.makedirs(output_path_aux) @@ -366,8 +366,8 @@ def get_ship_var_name(nc_var): def do_ship_transformation(): # TODO Documentation - for year in list_years: - in_path = os.path.join(input_path, input_name_ship.replace('', str(year))) + for year in LIST_YEARS: + in_path = os.path.join(INPUT_PATH, INPUT_NAME_SHIPS.replace('', str(year))) nc_in = Dataset(in_path, mode='r') c_lats = nc_in.variables['lat'][:] c_lons = nc_in.variables['lon'][:] diff --git a/preproc/edgarv432_ap_preproc.py b/preproc/edgarv432_ap_preproc.py index 34f77af..2f79e9c 100755 --- a/preproc/edgarv432_ap_preproc.py +++ b/preproc/edgarv432_ap_preproc.py @@ -25,22 +25,22 @@ from warnings import warn as warning # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/jrc/edgarv432_ap/original_files/' -output_path = '/esarchive/recon/jrc/edgarv432_ap' -list_pollutants = ['BC', 'CO', 'NH3', 'NOx', 'OC', 'PM10', 'PM2.5_bio', 'PM2.5_fossil', 'SO2', 'NMVOC'] -# list_years = [1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, +INPUT_PATH = '/esarchive/recon/jrc/edgarv432_ap/original_files/' +OUTPUT_PATH = '/esarchive/recon/jrc/edgarv432_ap' +LIST_POLLUTANTS = ['BC', 'CO', 'NH3', 'NOx', 'OC', 'PM10', 'PM2.5_bio', 'PM2.5_fossil', 'SO2', 'NMVOC'] +# LIST_YEARS = [1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, # 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, # 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012] -list_years = [2012] +LIST_YEARS = [2012] # To do yearly emissions -process_yearly = True -yearly_input_name = 'yearly/v432___.0.1x0.1.nc' +PROCESS_YEARLY = True +YEARLY_INPUT_NAME = 'yearly/v432___.0.1x0.1.nc' # To process monthly emissions, 2010 directly from monthly_input_name and other years calculated using bla bla bla -process_monthly = True -monthly_input_name = 'monthly/v432__2010__.0.1x0.1.nc' -monthly_pattern_file = 'temporal_profiles/v432_FM_.0.1x0.1.nc' +PROCESS_MONTHLY = True +MONTHLY_INPUT_NAME = 'monthly/v432__2010__.0.1x0.1.nc' +MONTHLY_PATTERN_FILE = 'temporal_profiles/v432_FM_.0.1x0.1.nc' # ============================================================== """ @@ -225,11 +225,11 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr def do_yearly_transformation(year): # TODO Documentation - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): file_path = os.path.join( - input_path, - yearly_input_name.replace('', pollutant).replace('', str(year)).replace('', + INPUT_PATH, + YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', ipcc)) if os.path.exists(file_path): @@ -263,7 +263,7 @@ def do_yearly_transformation(year): 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'yearly_mean', pollutant.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', pollutant.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) write_netcdf(os.path.join(out_path_aux, '{0}_{1}.nc'.format(pollutant.lower(), year)), @@ -277,11 +277,11 @@ def do_yearly_transformation(year): def do_monthly_transformation(year): # TODO Documentation - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): file_path = os.path.join( - input_path, - yearly_input_name.replace('', pollutant).replace('', str(year)).replace('', + INPUT_PATH, + YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', ipcc)) if os.path.exists(file_path): @@ -315,11 +315,11 @@ def do_monthly_transformation(year): 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'monthly_mean', pollutant.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'monthly_mean', pollutant.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) - nc_month_factors = Dataset(os.path.join(input_path, monthly_pattern_file.replace('', sector))) + nc_month_factors = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE.replace('', sector))) month_factors = nc_month_factors.variables[sector][:] for month in xrange(1, 12 + 1, 1): data_aux = data * month_factors[month - 1, :, :] @@ -336,12 +336,12 @@ def do_monthly_transformation(year): def do_2010_monthly_transformation(): # TODO Documentation - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): for month in xrange(1, 12 + 1, 1): file_path = os.path.join( - input_path, - monthly_input_name.replace('', pollutant).replace('', + INPUT_PATH, + MONTHLY_INPUT_NAME.replace('', pollutant).replace('', str(month)).replace('', ipcc)) if os.path.exists(file_path): @@ -374,7 +374,7 @@ def do_2010_monthly_transformation(): 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'monthly_mean', pollutant.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'monthly_mean', pollutant.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) write_netcdf(os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant.lower(), 2010, @@ -389,12 +389,12 @@ def do_2010_monthly_transformation(): if __name__ == '__main__': - if process_yearly: - for y in list_years: + if PROCESS_YEARLY: + for y in LIST_YEARS: do_yearly_transformation(y) - if process_monthly: - for y in list_years: + if PROCESS_MONTHLY: + for y in LIST_YEARS: if y == 2010: do_2010_monthly_transformation() else: diff --git a/preproc/edgarv432_voc_preproc.py b/preproc/edgarv432_voc_preproc.py index a42d8aa..3d7d116 100755 --- a/preproc/edgarv432_voc_preproc.py +++ b/preproc/edgarv432_voc_preproc.py @@ -25,24 +25,24 @@ from warnings import warn as warning # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/jrc/edgarv432_voc/original_files/' -output_path = '/esarchive/recon/jrc/edgarv432_voc' -list_pollutants = ['voc1', 'voc2', 'voc3', 'voc4', 'voc5', 'voc6', 'voc7', 'voc8', 'voc9', 'voc10', 'voc11', 'voc12', +INPUT_PATH = '/esarchive/recon/jrc/edgarv432_voc/original_files/' +OUTPUT_PATH = '/esarchive/recon/jrc/edgarv432_voc' +LIST_POLLUTANTS = ['voc1', 'voc2', 'voc3', 'voc4', 'voc5', 'voc6', 'voc7', 'voc8', 'voc9', 'voc10', 'voc11', 'voc12', 'voc13', 'voc14', 'voc15', 'voc16', 'voc17', 'voc18', 'voc19', 'voc20', 'voc21', 'voc22', 'voc23', 'voc24', 'voc25'] # list_years = [1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, # 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, # 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012] -list_years = [2010] +LIST_YEARS = [2010] # To do yearly emissions -process_yearly = True -yearly_input_name = 'yearly/v432_VOC_spec___.0.1x0.1.nc' +PROCESS_YEARLY = True +YEARLY_INPUT_NAME = 'yearly/v432_VOC_spec___.0.1x0.1.nc' # To process monthly emissions, 2010 directly from monthly_input_name and other years calculated using bla bla bla -process_monthly = False -monthly_input_name = 'monthly/v432_VOC_spec__2010__.0.1x0.1.nc' -monthly_pattern_file = 'temporal_profiles/v432_FM_.0.1x0.1.nc' +PROCESS_MONTHLY = False +MONTHLY_INPUT_NAME = 'monthly/v432_VOC_spec__2010__.0.1x0.1.nc' +MONTHLY_PATTERN_FILE = 'temporal_profiles/v432_FM_.0.1x0.1.nc' # ============================================================== """ @@ -222,11 +222,11 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr def do_yearly_transformation(year): # TODO Documentation print year - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): file_path = os.path.join( - input_path, - yearly_input_name.replace('', pollutant).replace('', str(year)).replace('', + INPUT_PATH, + YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', ipcc)) if os.path.exists(file_path): @@ -253,7 +253,7 @@ def do_yearly_transformation(year): 'units': 'kg.m-2.s-1', 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'yearly_mean', pollutant_aux.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', pollutant_aux.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) # print os.path.join(out_path_aux, '{0}_{1}.nc'.format(pollutant_aux.lower(), year)) @@ -269,11 +269,11 @@ def do_yearly_transformation(year): def do_monthly_transformation(year): # TODO Documentation print year - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): file_path = os.path.join( - input_path, - yearly_input_name.replace('', pollutant).replace('', str(year)).replace('', + INPUT_PATH, + YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', ipcc)) if os.path.exists(file_path): @@ -302,11 +302,11 @@ def do_monthly_transformation(year): 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'monthly_mean', pollutant_aux.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'monthly_mean', pollutant_aux.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) - nc_month_factors = Dataset(os.path.join(input_path, monthly_pattern_file.replace('', sector))) + nc_month_factors = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE.replace('', sector))) month_factors = nc_month_factors.variables[sector][:] for month in xrange(1, 12 + 1, 1): data_aux = data * month_factors[month - 1, :, :] @@ -323,12 +323,12 @@ def do_monthly_transformation(year): def do_2010_monthly_transformation(): # TODO Documentation - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): for month in xrange(1, 12 + 1, 1): file_path = os.path.join( - input_path, - monthly_input_name.replace('', pollutant).replace('', + INPUT_PATH, + MONTHLY_INPUT_NAME.replace('', pollutant).replace('', str(month)).replace('', ipcc)) if os.path.exists(file_path): @@ -358,7 +358,7 @@ def do_2010_monthly_transformation(): 'grid_mapping': 'crs'} out_path_aux = os.path.join( - output_path, 'monthly_mean', pollutant_aux.lower() + '_' + sector.lower()) + OUTPUT_PATH, 'monthly_mean', pollutant_aux.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) write_netcdf(os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format( @@ -373,12 +373,12 @@ def do_2010_monthly_transformation(): if __name__ == '__main__': - if process_yearly: - for y in list_years: + if PROCESS_YEARLY: + for y in LIST_YEARS: do_yearly_transformation(y) - if process_monthly: - for y in list_years: + if PROCESS_MONTHLY: + for y in LIST_YEARS: if y == 2010: do_2010_monthly_transformation() else: diff --git a/preproc/emep_preproc.py b/preproc/emep_preproc.py index e4f526a..6df1d37 100755 --- a/preproc/emep_preproc.py +++ b/preproc/emep_preproc.py @@ -24,12 +24,12 @@ from datetime import datetime # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/ceip/emepv18/original_files' -output_path = '/esarchive/recon/ceip/emepv18/yearly_mean' -input_name = '__2018_GRID_.txt' +INPUT_PATH = '/esarchive/recon/ceip/emepv18/original_files' +OUTPUT_PATH = '/esarchive/recon/ceip/emepv18/yearly_mean' +INPUT_NAME = '__2018_GRID_.txt' # list_years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016] -list_years = [2015] -list_pollutants = ['NOx', 'NMVOC', 'SOx', 'NH3', 'PM2_5', 'PM10', 'CO'] +LIST_YEARS = [2015] +LIST_POLLUTANTS = ['NOx', 'NMVOC', 'SOx', 'NH3', 'PM2_5', 'PM10', 'CO'] # ============================================================== @@ -83,11 +83,11 @@ def do_transformation(year): unit_factor = 1000./(365.*24.*3600.) # From Mg/year to Kg/s - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for sector in get_sectors(): in_file = os.path.join( - input_path, - input_name.replace('', str(year)).replace('', sector).replace('', pollutant)) + INPUT_PATH, + INPUT_NAME.replace('', str(year)).replace('', sector).replace('', pollutant)) if os.path.exists(in_file): print in_file @@ -123,7 +123,7 @@ def do_transformation(year): element['data'] = element['data'].reshape((1,) + element['data'].shape) - complete_output_dir = os.path.join(output_path, '{0}_{1}'.format(element['name'], sector.lower())) + complete_output_dir = os.path.join(OUTPUT_PATH, '{0}_{1}'.format(element['name'], sector.lower())) if not os.path.exists(complete_output_dir): os.makedirs(complete_output_dir) complete_output_dir = os.path.join(complete_output_dir, '{0}_{1}.nc'.format(element['name'], year)) @@ -148,5 +148,5 @@ def do_transformation(year): if __name__ == '__main__': - for y in list_years: + for y in LIST_YEARS: do_transformation(y) diff --git a/preproc/gfas12_preproc.py b/preproc/gfas12_preproc.py index ef2de31..b2591aa 100755 --- a/preproc/gfas12_preproc.py +++ b/preproc/gfas12_preproc.py @@ -26,14 +26,14 @@ import datetime from datetime import datetime, timedelta # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' -input_name = 'ga_.grb' -output_path = '/esarchive/recon/ecmwf/gfas' +INPUT_PATH = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' +INPUT_NAME = 'ga_.grb' +OUTPUT_PATH = '/esarchive/recon/ecmwf/gfas' -starting_date = datetime(year=2018, month=8, day=29) -ending_date = datetime(year=2018, month=8, day=29) +STARTING_DATE = datetime(year=2018, month=8, day=29) +ENDIND_DATE = datetime(year=2018, month=8, day=29) -parameters_file = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_Parameters.csv' +PARAMETERS_FILE = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_Parameters.csv' # ============================================================== @@ -261,13 +261,13 @@ def do_var_list(variables_file): if __name__ == '__main__': - var_list = do_var_list(parameters_file) + var_list = do_var_list(PARAMETERS_FILE) - date_aux = starting_date - while date_aux <= ending_date: - f = os.path.join(input_path, input_name.replace('', date_aux.strftime('%Y%m%d'))) + date_aux = STARTING_DATE + while date_aux <= ENDIND_DATE: + f = os.path.join(INPUT_PATH, INPUT_NAME.replace('', date_aux.strftime('%Y%m%d'))) if os.path.isfile(f): - do_transformation(f, date_aux, output_path, var_list) + do_transformation(f, date_aux, OUTPUT_PATH, var_list) else: print 'ERROR: file {0} not found'.format(f) diff --git a/preproc/htapv2_preproc.py b/preproc/htapv2_preproc.py index d178c71..5961919 100755 --- a/preproc/htapv2_preproc.py +++ b/preproc/htapv2_preproc.py @@ -19,27 +19,28 @@ import os -import sys # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/jrc/htapv2/original_files' -output_path = '/esarchive/recon/jrc/htapv2' +INPUT_PATH = '/esarchive/recon/jrc/htapv2/original_files' +OUTPUT_PATH = '/esarchive/recon/jrc/htapv2' -input_name = 'edgar_HTAP__emi___.0.1x0.1.nc' -input_name_air = 'edgar_HTAP_emi___.0.1x0.1.nc' -input_name_ships = 'edgar_HTAP__emi_SHIPS_.0.1x0.1.nc' -#HTAP auxiliary NMVOC emission data for the industry sub-sectors (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) -input_name_nmvoc_industry = 'HTAPv2_NMVOC___.0.1x0.1.nc' +INPUT_NAME = 'edgar_HTAP__emi___.0.1x0.1.nc' +INPUT_NAME_AIR = 'edgar_HTAP_emi___.0.1x0.1.nc' +INPUT_NAME_SHIPS = 'edgar_HTAP__emi_SHIPS_.0.1x0.1.nc' +# HTAP auxiliary NMVOC emission data for the industry sub-sectors +# (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) +INPUT_NAME_NMVOC_INDUSTRY = 'HTAPv2_NMVOC___.0.1x0.1.nc' # list_years = [2008, 2010] -list_years = [2010] - -#RETRO ratios applied to HTAPv2 NMVOC emissions (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) -voc_ratio_path = '/esarchive/recon/jrc/htapv2/original_files/retro_nmvoc_ratio_2000_01x01' -voc_ratio_name = 'retro_nmvoc_ratio__2000_0.1deg.nc' -voc_ratio_air_name = 'VOC_split_AIR.csv' -voc_ratio_ships_name = 'VOC_split_SHIP.csv' +LIST_YEARS = [2010] + +# RETRO ratios applied to HTAPv2 NMVOC emissions +# (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) +VOC_RATIO_PATH = '/esarchive/recon/jrc/htapv2/original_files/retro_nmvoc_ratio_2000_01x01' +VOC_RATIO_NAME = 'retro_nmvoc_ratio__2000_0.1deg.nc' +VOC_RATIO_AIR_NAME = 'VOC_split_AIR.csv' +VOC_RATIO_SHIPS_NAME = 'VOC_split_SHIP.csv' # ============================================================== @@ -64,11 +65,10 @@ def do_transformation_annual(filename, out_path, pollutant, sector, year): :return: """ - import os from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds print filename - c_lats, c_lons = extract_vars(filename, ['lat', 'lon']) + [c_lats, c_lons] = extract_vars(filename, ['lat', 'lon']) if pollutant == 'pm25': [data] = extract_vars(filename, ['emi_pm2.5'], @@ -90,7 +90,8 @@ def do_transformation_annual(filename, out_path, pollutant, sector, year): '2017-04-04: Added global attributes;\n' + '2017-04-04: Re-naming pollutant;\n' + '2017-04-04: Added cell_area variable;\n', - 'references': 'EC, JRC / US EPA, HTAP_V2. http://edgar.jrc.ec.europa.eu/htap/EDGAR-HTAP_v1_final_jan2012.pdf\n ' + + 'references': 'EC, JRC / US EPA, HTAP_V2. ' + + 'http://edgar.jrc.ec.europa.eu/htap/EDGAR-HTAP_v1_final_jan2012.pdf\n ' + 'http://edgar.jrc.ec.europa.eu/htap_v2/', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)', @@ -129,13 +130,12 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): :return: """ - import os from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds for month in xrange(1, 13): print filename_list[month - 1] - c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) + [c_lats, c_lons] = extract_vars(filename_list[month - 1], ['lat', 'lon']) if pollutant == 'pm25': [data] = extract_vars(filename_list[month - 1], ['emi_pm2.5'], @@ -157,7 +157,9 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): '2017-04-04: Added global attributes;\n' + '2017-04-04: Re-naming pollutant;\n' + '2017-04-04: Added cell_area variable;\n', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global ' + + 'emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, ' + + 'Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)', @@ -176,45 +178,57 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): def do_ratio_list(sector=None): # TODO Documentation + """ + + :param sector: + :return: + """ if sector == 'SHIPS': - return {'all': os.path.join(voc_ratio_path, voc_ratio_ships_name)} + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_SHIPS_NAME)} elif sector == 'AIR_CDS': - return {'all': os.path.join(voc_ratio_path, voc_ratio_air_name)} + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} elif sector == 'AIR_CRS': - return {'all': os.path.join(voc_ratio_path, voc_ratio_air_name)} + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} elif sector == 'AIR_LTO': - return {'all': os.path.join(voc_ratio_path, voc_ratio_air_name)} - else: - return { - 'voc01': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '01')), - 'voc02': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '02')), - 'voc03': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '03')), - 'voc04': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '04')), - 'voc05': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '05')), - 'voc06': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '06')), - 'voc07': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '07')), - 'voc08': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '08')), - 'voc09': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '09')), - 'voc12': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '12')), - 'voc13': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '13')), - 'voc14': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '14')), - 'voc15': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '15')), - 'voc16': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '16')), - 'voc17': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '17')), - 'voc18': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '18')), - 'voc19': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '19')), - 'voc20': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '20')), - 'voc21': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '21')), - 'voc22': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '22')), - 'voc23': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '23')), - 'voc24': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '24')), - 'voc25': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '25')), - } + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} + return { + 'voc01': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '01')), + 'voc02': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '02')), + 'voc03': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '03')), + 'voc04': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '04')), + 'voc05': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '05')), + 'voc06': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '06')), + 'voc07': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '07')), + 'voc08': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '08')), + 'voc09': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '09')), + 'voc12': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '12')), + 'voc13': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '13')), + 'voc14': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '14')), + 'voc15': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '15')), + 'voc16': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '16')), + 'voc17': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '17')), + 'voc18': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '18')), + 'voc19': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '19')), + 'voc20': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '20')), + 'voc21': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '21')), + 'voc22': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '22')), + 'voc23': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '23')), + 'voc24': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '24')), + 'voc25': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '25')), + } def do_nmvoc_month_transformation(filename_list, out_path, sector, year): # TODO Docuemtnation - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area + """ + + :param filename_list: + :param out_path: + :param sector: + :param year: + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf from hermesv3_gr.tools.coordinates_tools import create_bounds nmvoc_ratio_list = do_ratio_list() @@ -258,7 +272,7 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): [ratio] = extract_vars(ratio_file, [ratio_var]) data_aux = data.copy() - data_aux['data'] = data['data']*ratio['data'] + data_aux['data'] = data['data'] * ratio['data'] data_aux['data'] = data_aux['data'].reshape((1,) + data_aux['data'].shape) data_aux['name'] = voc data_aux['units'] = 'kg m-2 s-1' @@ -286,12 +300,21 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): print out_path_aux write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], boundary_latitudes=create_bounds(c_lats['data']), - boundary_longitudes=create_bounds(c_lons['data']),global_attributes=global_attributes,) + boundary_longitudes=create_bounds(c_lons['data']) ,global_attributes=global_attributes,) return True def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year): - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area + # TODO Documentation + """ + + :param filename_list: + :param out_path: + :param sector: + :param year: + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf from hermesv3_gr.tools.coordinates_tools import create_bounds nmvoc_ratio_list = do_ratio_list() @@ -314,22 +337,23 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year } if voc in ['voc02', 'voc03', 'voc04', 'voc05', 'voc07', 'voc08', 'voc12', 'voc13']: [r_inc, r_exf] = extract_vars(ratio_file, ['inc', 'exf']) - data.update({'data': ind['data']*r_inc['data'] + exf['data']*r_exf['data']}) + data.update({'data': ind['data'] * r_inc['data'] + exf['data'] * r_exf['data']}) elif voc in ['voc01', 'voc23', 'voc25']: [r_inc, r_sol] = extract_vars(ratio_file, ['inc', 'sol']) - data.update({'data': ind['data']*r_inc['data'] + sol['data']*r_sol['data']}) + data.update({'data': ind['data'] * r_inc['data'] + sol['data'] * r_sol['data']}) elif voc in ['voc09', 'voc16', 'voc21', 'voc22', 'voc24']: [r_inc] = extract_vars(ratio_file, ['inc']) - data.update({'data': ind['data']*r_inc['data']}) + data.update({'data': ind['data'] * r_inc['data']}) # elif voc in []: # [r_exf, r_sol] = extract_vars(ratio_file, ['exf', 'sol']) # data.update({'data': exf['data']*r_exf['data'] + sol['data']*r_sol['data']}) elif voc in ['voc18', 'voc19', 'voc20']: [r_sol] = extract_vars(ratio_file, ['sol']) - data.update({'data': sol['data']*r_sol['data']}) + data.update({'data': sol['data'] * r_sol['data']}) else: [r_inc, r_exf, r_sol] = extract_vars(ratio_file, ['inc', 'exf', 'sol']) - data.update({'data': ind['data']*r_inc['data'] + exf['data']*r_exf['data'] + sol['data']*r_sol['data']}) + data.update({'data': ind['data'] * r_inc['data'] + exf['data']*r_exf['data'] + + sol['data'] * r_sol['data']}) global_attributes = { 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, voc), @@ -338,7 +362,10 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year 'source': 'HTAPv2', 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + '2017-04-28: ...', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and ' + + 'global emission grid maps for 2008 and 2010 to study hemispheric transport of air ' + + 'pollution, Atmos. Chem. Phys., 15, 11411-11432, ' + + 'https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)', @@ -351,22 +378,27 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) print out_path_aux write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data], - boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), - global_attributes=global_attributes,) + boundary_latitudes=create_bounds(c_lats['data']), + boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) def do_nmvoc_year_transformation(filename, out_path, sector, year): - import os + # TODO Documentation + """ + + :param filename: + :param out_path: + :param sector: + :param year: + :return: + """ import pandas as pd - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf from hermesv3_gr.tools.coordinates_tools import create_bounds nmvoc_ratio_file = do_ratio_list(sector)['all'] nmvoc_ratio_list = pd.read_csv(nmvoc_ratio_file, sep=';') - - print 'hola->',filename - c_lats, c_lons = extract_vars(filename, ['lat', 'lon']) [data] = extract_vars(filename, ['emi_nmvoc']) @@ -375,10 +407,8 @@ def do_nmvoc_year_transformation(filename, out_path, sector, year): pollutant = voc_ratio['voc_group'] ratio = voc_ratio['factor'] - # print pollutant, ratio - data_aux = data.copy() - data_aux['data'] = data['data']*ratio + data_aux['data'] = data['data'] * ratio data_aux['data'] = data_aux['data'].reshape((1,) + data_aux['data'].shape) data_aux['name'] = pollutant data_aux['units'] = 'kg m-2 s-1' @@ -389,7 +419,9 @@ def do_nmvoc_year_transformation(filename, out_path, sector, year): 'source': 'HTAPv2', 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + '2017-04-28: ...', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global ' + + 'emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, ' + + 'Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)\n ' + @@ -409,21 +441,31 @@ def do_nmvoc_year_transformation(filename, out_path, sector, year): def get_pollutant_dict(): - pollutant_dict = { - # 'bc': 'BC', - # 'co': 'CO', - # 'nh3': 'NH3', - # 'nox_no2': 'NOx', - # 'oc': 'OC', - # 'pm10': 'PM10', - # 'pm25': 'PM2.5', - # 'so2': 'SO2', + # TODO Documentation + """ + + :return: + """ + p_dict = { + 'bc': 'BC', + 'co': 'CO', + 'nh3': 'NH3', + 'nox_no2': 'NOx', + 'oc': 'OC', + 'pm10': 'PM10', + 'pm25': 'PM2.5', + 'so2': 'SO2', 'nmvoc': 'NMVOC' } - return pollutant_dict + return p_dict def get_sector_dict(): + # TODO Documentation + """ + + :return: + """ common_dict = { 'month': ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT'], 'year': ['SHIPS', 'AIR_CDS', 'AIR_CRS', 'AIR_LTO'] @@ -444,88 +486,93 @@ def get_sector_dict(): def get_nmvoc_sector_dict(): + # TODO Documentation + """ + + :return: + """ nmvoc_sectors = {'month': ['ENERGY', 'INDUSTRY_3subsectors', 'RESIDENTIAL', 'TRANSPORT'], 'year': ['SHIPS', 'AIR_CDS', 'AIR_CRS', 'AIR_LTO']} return nmvoc_sectors def check_vocs(year): + # TODO Documentation + """ + + :param year: + :return: + """ from hermesv3_gr.tools.netcdf_tools import extract_vars - for month in xrange(1, 12 +1, 1): + for month in xrange(1, 12 + 1, 1): for snap in ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT']: - nmvoc_path = os.path.join(output_path, 'monthly_mean', 'nmvoc_{0}'.format(snap.lower()), 'nmvoc_{0}{1}.nc'.format(year, str(month).zfill(2))) - # print nmvoc_path + nmvoc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', 'nmvoc_{0}'.format(snap.lower()), + 'nmvoc_{0}{1}.nc'.format(year, str(month).zfill(2))) [new_voc] = extract_vars(nmvoc_path, ['nmvoc']) nmvoc_sum = new_voc['data'].sum() voc_sum = 0 - for voc in ['voc{0}'.format(str(x).zfill(2)) for x in xrange(1, 25 +1, 1)]: - voc_path = os.path.join(output_path, 'monthly_mean', '{0}_{1}'.format(voc, snap.lower()), '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) - # print voc_path, os.path.exists(voc_path) + for voc in ['voc{0}'.format(str(x).zfill(2)) for x in xrange(1, 25 + 1, 1)]: + voc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(voc, snap.lower()), + '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) if os.path.exists(voc_path): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format(snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum)/nmvoc_sum, month) - + print '{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum)/nmvoc_sum, month) if __name__ == '__main__': - - for y in list_years: - # check_vocs(y) - # sys.exit(1) + for y in LIST_YEARS: for pollutant_dict in get_pollutant_dict().iteritems(): for current_sector in get_sector_dict()[pollutant_dict[0]]['month']: - input_name_aux = input_name.replace('', current_sector) + input_name_aux = INPUT_NAME.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) input_name_aux = input_name_aux.replace('', pollutant_dict[1]) - file_list = [os.path.join(input_path, input_name_aux.replace('', str(aux_month))) + file_list = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) for aux_month in xrange(1, 13)] - do_transformation(file_list, os.path.join(output_path, 'monthly_mean'), pollutant_dict[0], current_sector, - y) + do_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), pollutant_dict[0], + current_sector, y) # annual inventories for current_sector in get_sector_dict()[pollutant_dict[0]]['year']: if current_sector[0:3] == 'AIR': - input_name_aux = input_name_air + input_name_aux = INPUT_NAME_AIR else: - input_name_aux = input_name_ships + input_name_aux = INPUT_NAME_SHIPS input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) input_name_aux = input_name_aux.replace('', pollutant_dict[1]) - input_name_aux = os.path.join(input_path, input_name_aux) + input_name_aux = os.path.join(INPUT_PATH, input_name_aux) - do_transformation_annual(input_name_aux, os.path.join(output_path, 'yearly_mean', ), pollutant_dict[0], + do_transformation_annual(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean', ), pollutant_dict[0], current_sector, y) for current_sector in get_nmvoc_sector_dict()['month']: if current_sector == 'INDUSTRY_3subsectors': - input_name_aux = input_name_nmvoc_industry + input_name_aux = INPUT_NAME_NMVOC_INDUSTRY else: - input_name_aux = input_name + input_name_aux = INPUT_NAME input_name_aux = input_name_aux.replace('', 'NMVOC') input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) - file_list = [os.path.join(input_path, input_name_aux.replace('', str(aux_month))) + file_list = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) for aux_month in xrange(1, 13)] if current_sector == 'INDUSTRY_3subsectors': - do_nmvoc_industry_month_transformation(file_list, os.path.join(output_path, 'monthly_mean'), current_sector, - y) + do_nmvoc_industry_month_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), + current_sector, y) else: - do_nmvoc_month_transformation(file_list, os.path.join(output_path, 'monthly_mean'), current_sector, - y) + do_nmvoc_month_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), current_sector, y) for current_sector in get_nmvoc_sector_dict()['year']: if current_sector[0:3] == 'AIR': - input_name_aux = input_name_air + input_name_aux = INPUT_NAME_AIR else: - input_name_aux = input_name_ships + input_name_aux = INPUT_NAME_SHIPS input_name_aux = input_name_aux.replace('', 'NMVOC') input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) - input_name_aux = os.path.join(input_path, input_name_aux) + input_name_aux = os.path.join(INPUT_PATH, input_name_aux) print input_name_aux - do_nmvoc_year_transformation(input_name_aux, os.path.join(output_path, 'yearly_mean'), current_sector, - y) - + do_nmvoc_year_transformation(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean'), current_sector, y) diff --git a/preproc/tno_mac_iii_preproc.py b/preproc/tno_mac_iii_preproc.py index b9841db..75db7df 100755 --- a/preproc/tno_mac_iii_preproc.py +++ b/preproc/tno_mac_iii_preproc.py @@ -22,13 +22,13 @@ import os # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/tno/tno_macc_iii/original_files/ascii' -output_path = '/esarchive/recon/tno/tno_macc_iii/yearly_mean' -input_name = 'TNO_MACC_III_emissions_v1_1_.txt' +INPUT_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/ascii' +OUTPUT_PATH = '/esarchive/recon/tno/tno_macc_iii/yearly_mean' +INPUT_NAME = 'TNO_MACC_III_emissions_v1_1_.txt' # list_years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011] -list_years = [2011] -voc_ratio_path = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' -vor_ratio_name = 'ratio_.nc' +LIST_YEARS = [2011] +VOC_RATIO_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' +VOC_RATIO_NAME = 'ratio_.nc' # ============================================================== @@ -105,7 +105,7 @@ def do_transformation(year): import pandas as pd import numpy as np - in_file = os.path.join(input_path, input_name.replace('', str(year))) + in_file = os.path.join(INPUT_PATH, INPUT_NAME.replace('', str(year))) unit_factor = 1000./(365.*24.*3600.) # To pass from Mg/year to Kg/s # unit_factor = 1000000 # To pass from Mg/m2.year to Mg/Km2.year @@ -145,7 +145,7 @@ def do_transformation(year): pollutant_list[i]['data'] = pollutant_list[i]['data'].reshape((1,) + pollutant_list[i]['data'].shape) # print pollutant_list[i]['data'].max() - aux_output_path = os.path.join(output_path, '{0}_snap{1}'.format(pollutant_list[i]['name'], name)) + aux_output_path = os.path.join(OUTPUT_PATH, '{0}_snap{1}'.format(pollutant_list[i]['name'], name)) if not os.path.exists(aux_output_path): os.makedirs(aux_output_path) aux_output_path = os.path.join(aux_output_path, '{0}_{1}.nc'.format(pollutant_list[i]['name'], year)) @@ -224,10 +224,10 @@ def do_voc_transformation(year): from warnings import warn as warning for snap in get_sector_list(): - in_path = os.path.join(output_path, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) + in_path = os.path.join(OUTPUT_PATH, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) [nmvoc, c_lats, c_lons, cell_area] = extract_vars(in_path, ['nmvoc', 'lat', 'lon', 'cell_area']) for voc in get_voc_list(): - ratio_path = os.path.join(voc_ratio_path, vor_ratio_name.replace('', voc)) + ratio_path = os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', voc)) ratios_dict = get_voc_ratio(ratio_path, snap) if ratios_dict is not None: new_voc = { @@ -241,7 +241,7 @@ def do_voc_transformation(year): new_voc['data'] = nmvoc['data'] * mask - out_dir_aux = os.path.join(output_path, '{0}_{1}'.format(voc, snap)) + out_dir_aux = os.path.join(OUTPUT_PATH, '{0}_{1}'.format(voc, snap)) if not os.path.exists(out_dir_aux): os.makedirs(out_dir_aux) # print os.path.join(out_dir_aux, '{0}_{1}.nc'.format(voc, year)) @@ -267,13 +267,13 @@ def do_voc_transformation(year): def check_vocs(year): # TODO Documentation for snap in get_sector_list(): - nmvoc_path = os.path.join(output_path, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) + nmvoc_path = os.path.join(OUTPUT_PATH, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) [new_voc] = extract_vars(nmvoc_path, ['nmvoc']) nmvoc_sum = new_voc['data'].sum() voc_sum = 0 for voc in get_voc_list(): - voc_path = os.path.join(output_path, '{0}_{1}'.format(voc, snap), '{0}_{1}.nc'.format(voc, year)) + voc_path = os.path.join(OUTPUT_PATH, '{0}_{1}'.format(voc, snap), '{0}_{1}.nc'.format(voc, year)) if os.path.exists(voc_path): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() @@ -284,7 +284,7 @@ def check_vocs(year): if __name__ == '__main__': - for y in list_years: + for y in LIST_YEARS: do_transformation(y) do_voc_transformation(y) # check_vocs(y) diff --git a/preproc/tno_mac_iii_preproc_voc_ratios.py b/preproc/tno_mac_iii_preproc_voc_ratios.py index f1800a3..08d4c5d 100755 --- a/preproc/tno_mac_iii_preproc_voc_ratios.py +++ b/preproc/tno_mac_iii_preproc_voc_ratios.py @@ -23,16 +23,22 @@ import os # ============== CONFIGURATION PARAMETERS ====================== -output_path = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' -world_info_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/temporal/tz_world_country_iso3166.csv' -tno_world_mask = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/TNO_MACC-III_WorldMask.nc' -csv_path = '/esarchive/recon/tno/tno_macc_iii/original_files/TNO_MACC_NMVOC profile_country_SNAP_12_05_2010.csv' - +OUTPUT_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' +WORLD_INFO_PATH = '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/temporal/tz_world_country_iso3166.csv' +TNO_WORLD_MASK = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/TNO_MACC-III_WorldMask.nc' +CSV_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/TNO_MACC_NMVOC profile_country_SNAP_12_05_2010.csv' # ============================================================== -def extract_vars(netcdf_path, variables_list, attributes_list=list()): +def extract_vars(netcdf_path, variables_list, attributes_list=[]): # TODO Docuemtnation + """ + + :param netcdf_path: + :param variables_list: + :param attributes_list: + :return: + """ from netCDF4 import Dataset data_list = [] # print netcdf_path @@ -59,20 +65,47 @@ def extract_vars(netcdf_path, variables_list, attributes_list=list()): def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, - RegularLatLon=False, - Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, - LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): + regular_latlon=False, + rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, + lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): # TODO Docuemtnation + """ + + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param boundary_latitudes: + :param boundary_longitudes: + :param cell_area: + :param global_attributes: + :param regular_latlon: + :param rotated: + :param rotated_lats: + :param rotated_lons: + :param north_pole_lat: + :param north_pole_lon: + :param lcc: + :param lcc_x: + :param lcc_y: + :param lat_1_2: + :param lon_0: + :param lat_0: + :return: + """ from cf_units import Unit, encode_time from netCDF4 import Dataset - if not (RegularLatLon or LambertConformalConic or Rotated): - RegularLatLon = True + if not (regular_latlon or lcc or rotated): + regular_latlon = True print netcdf_path netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -96,7 +129,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, else: print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) sys.exit(1) - elif Rotated: + elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude @@ -113,7 +146,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic: + elif lcc: var_dim = ('y', 'x',) netcdf.createDimension('y', len(lcc_y)) @@ -190,7 +223,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes - if Rotated: + if rotated: # Rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=True) rlat.long_name = "latitude in rotated pole grid" @@ -204,7 +237,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = rotated_lons - if LambertConformalConic: + if lcc: x = netcdf.createVariable('x', 'd', ('x',), zlib=True) x.units = Unit("km").symbol x.long_name = "x coordinate of projection" @@ -243,11 +276,11 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, var.coordinates = "lat lon" if cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif rotated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' # if variable['data'] is not 0: # print var[:].shape, variable['data'].shape @@ -257,19 +290,19 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: + elif rotated: # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = north_pole_lat mapping.grid_north_pole_longitude = north_pole_lon - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" @@ -295,7 +328,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, def get_grid_area(filename): """ - Calculates the area of each cell. + Calculate the area of each cell. :param filename: Full path to the NetCDF to calculate the cell areas. :type filename: str @@ -307,8 +340,8 @@ def get_grid_area(filename): from netCDF4 import Dataset cdo = Cdo() - s = cdo.gridarea(input=filename) - nc_aux = Dataset(s, mode='r') + src = cdo.gridarea(input=filename) + nc_aux = Dataset(src, mode='r') grid_area = nc_aux.variables['cell_area'][:] nc_aux.close() @@ -317,7 +350,7 @@ def get_grid_area(filename): def create_bounds(coords, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) :type coords: numpy.ndarray @@ -347,14 +380,19 @@ def create_bounds(coords, number_vertices=2): def create_voc_ratio(voc): # TODO Docuemtnation + """ + + :param voc: + :return: + """ import numpy as np - country_values, lat, lon = extract_vars(tno_world_mask, ['timezone_id', 'lat', 'lon']) + [country_values, lat, lon] = extract_vars(TNO_WORLD_MASK, ['timezone_id', 'lat', 'lon']) country_values = country_values['data'].reshape((country_values['data'].shape[1], country_values['data'].shape[1])) - print output_path - if not os.path.exists(output_path): - os.makedirs(output_path) + print OUTPUT_PATH + if not os.path.exists(OUTPUT_PATH): + os.makedirs(OUTPUT_PATH) - complete_output_path = os.path.join(output_path, 'ratio_{0}.nc'.format(voc)) + complete_output_path = os.path.join(OUTPUT_PATH, 'ratio_{0}.nc'.format(voc)) if not os.path.exists(complete_output_path): print 'Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path) data_list = [] @@ -383,9 +421,15 @@ def create_voc_ratio(voc): def get_default_ratio(voc, snap): # TODO Documentation + """ + + :param voc: + :param snap: + :return: + """ import pandas as pd - df = pd.read_csv(csv_path, sep=';') + df = pd.read_csv(CSV_PATH, sep=';') df = df.loc[df['vcode'] == voc.replace('voc', 'v'), :] df = df.loc[df['snap'] == snap, :] @@ -395,10 +439,14 @@ def get_default_ratio(voc, snap): def get_iso_codes(): # TODO Documentation + """ + + :return: + """ import pandas as pd # df = pd.read_csv(self.world_info, sep=';', index_col=False, names=["country", "country_code"]) - df = pd.read_csv(world_info_path, sep=';') + df = pd.read_csv(WORLD_INFO_PATH, sep=';') del df['time_zone'], df['time_zone_code'] df = df.drop_duplicates().dropna() df = df.set_index('country_code_alpha') @@ -410,9 +458,13 @@ def get_iso_codes(): def get_voc_list(): # TODO Documentation + """ + + :return: + """ import pandas as pd - df = pd.read_csv(csv_path, sep=';') + df = pd.read_csv(CSV_PATH, sep=';') del df['ISO3'], df['snap'], df['output substance name'], df['fr'] df = df.drop_duplicates().dropna() voc_list = df.vcode.values @@ -423,9 +475,14 @@ def get_voc_list(): def get_sector_list(voc): # TODO Documentation + """ + + :param voc: + :return: + """ import pandas as pd voc = voc.replace('voc', 'v') - df = pd.read_csv(csv_path, sep=';') + df = pd.read_csv(CSV_PATH, sep=';') df = df[df.vcode == voc] del df['ISO3'], df['vcode'], df['output substance name'], df['fr'] df = df.drop_duplicates().dropna() @@ -434,6 +491,11 @@ def get_sector_list(voc): def get_sector_list_text(voc): # TODO Documentation + """ + + :param voc: + :return: + """ voc = voc.replace('voc', 'v') sector_list = get_sector_list(voc) new_list = [] @@ -444,9 +506,15 @@ def get_sector_list_text(voc): def get_country_code_and_factor(voc, snap): # TODO Documentation + """ + + :param voc: + :param snap: + :return: + """ import pandas as pd voc = voc.replace('voc', 'v') - df = pd.read_csv(csv_path, sep=';') + df = pd.read_csv(CSV_PATH, sep=';') df = df[df.vcode == voc] df = df[df.snap == snap] del df['snap'], df['vcode'], df['output substance name'] diff --git a/preproc/wiedinmyer_preproc.py b/preproc/wiedinmyer_preproc.py index 069ca06..5af455f 100755 --- a/preproc/wiedinmyer_preproc.py +++ b/preproc/wiedinmyer_preproc.py @@ -24,13 +24,13 @@ from netCDF4 import Dataset # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/ucar/wiedinmyer/original_files/' -output_path = '/esarchive/recon/ucar/wiedinmyer/' -list_pollutants = ['co2', 'co', 'so2', 'nox_no', 'nh3', 'ch4', 'c2h2', 'c2h4', 'c3h6', 'ch3oh', 'ch2o', 'ch3cooh', +INPUT_PATH = '/esarchive/recon/ucar/wiedinmyer/original_files/' +OUTPUT_PATH = '/esarchive/recon/ucar/wiedinmyer/' +LIST_POLLUTANTS = ['co2', 'co', 'so2', 'nox_no', 'nh3', 'ch4', 'c2h2', 'c2h4', 'c3h6', 'ch3oh', 'ch2o', 'ch3cooh', 'hcn', 'c6h6', 'pcb', 'pah', 'pcdd', 'pbdd', 'nmoc', 'hcl', 'hg', 'pm25', 'pm10', 'oc', 'bc'] -input_name = 'ALL_Emiss_04282014.nc' -year = 2010 +INPUT_NAME = 'ALL_Emiss_04282014.nc' +YEAR = 2010 # ============================================================== @@ -89,7 +89,7 @@ def do_transformation(filename): factor = 1000000./(365.*24.*3600.) # To pass from Gg/m2.year to Kg/m2.s - for output_pollutant in list_pollutants: + for output_pollutant in LIST_POLLUTANTS: input_pollutant = out_pollutant_to_in_pollutant(output_pollutant) data = nc_in.variables[input_pollutant][:] @@ -103,11 +103,11 @@ def do_transformation(filename): 'grid_mapping': 'crs'} data = np.array(data) - out_path_aux = os.path.join(output_path, 'yearly_mean', output_pollutant) + out_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', output_pollutant) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) - write_netcdf(os.path.join(out_path_aux, '{0}_{1}.nc'.format(output_pollutant, year)), - data, data_attributes, lats, lons, grid_area, year, 01) + write_netcdf(os.path.join(out_path_aux, '{0}_{1}.nc'.format(output_pollutant, YEAR)), + data, data_attributes, lats, lons, grid_area, YEAR, 01) nc_in.close() @@ -229,6 +229,6 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr if __name__ == '__main__': starting_time = timeit.default_timer() - do_transformation(os.path.join(input_path, input_name)) + do_transformation(os.path.join(INPUT_PATH, INPUT_NAME)) print 'Time(s):', timeit.default_timer() - starting_time -- GitLab From 162f21822301c78900d6909aec6f9cd1d632f4a2 Mon Sep 17 00:00:00 2001 From: Carles Tena Medina Date: Tue, 11 Sep 2018 12:27:24 +0200 Subject: [PATCH 12/28] Correcting Code conventions --- hermesv3_gr/config/config.py | 6 +- .../emision_inventories/emission_inventory.py | 4 +- .../gfas_emission_inventory.py | 6 +- .../point_source_emission_inventory.py | 2 +- hermesv3_gr/modules/grids/grid.py | 26 +-- hermesv3_gr/modules/grids/grid_lcc.py | 6 +- hermesv3_gr/modules/grids/grid_mercator.py | 6 +- hermesv3_gr/modules/grids/grid_rotated.py | 4 +- hermesv3_gr/modules/regrid/regrid.py | 2 +- .../modules/regrid/regrid_conservative.py | 2 +- hermesv3_gr/modules/speciation/speciation.py | 4 +- hermesv3_gr/modules/temporal/temporal.py | 2 +- hermesv3_gr/modules/vertical/vertical.py | 10 +- hermesv3_gr/modules/vertical/vertical_gfas.py | 4 +- hermesv3_gr/modules/writing/writer_cmaq.py | 176 ++++++++---------- .../modules/writing/writer_wrf_chem.py | 2 +- hermesv3_gr/tools/coordinates_tools.py | 155 ++++----------- hermesv3_gr/tools/custom_calendar.py | 4 +- hermesv3_gr/tools/netcdf_tools.py | 133 +++++++++++-- preproc/eclipsev5a_preproc.py | 4 +- preproc/edgarv432_ap_preproc.py | 4 +- preproc/edgarv432_voc_preproc.py | 4 +- preproc/gfas12_preproc.py | 6 +- preproc/htapv2_preproc.py | 8 +- 24 files changed, 285 insertions(+), 295 deletions(-) diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index 196ddd9..36d9db4 100644 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -202,7 +202,7 @@ class Config(ArgParser): @staticmethod def create_dir(path): """ - Creates the given folder if it is not created yet. + Create the given folder if it is not created yet. :param path: Path to create. :type path: str @@ -248,7 +248,7 @@ class Config(ArgParser): @staticmethod def _parse_start_date(str_date): """ - Parses the date form string to datetime. + Parse the date form string to datetime. It accepts several ways to introduce the date: YYYYMMDD, YYYY/MM/DD, YYYYMMDDhh, YYYYYMMDD.hh, YYYY/MM/DD_hh:mm:ss, YYYY-MM-DD_hh:mm:ss, YYYY/MM/DD hh:mm:ss, YYYY-MM-DD hh:mm:ss, YYYY/MM/DD_hh, YYYY-MM-DD_hh. @@ -279,7 +279,7 @@ class Config(ArgParser): def _parse_end_date(self, end_date, start_date): """ - Parses the end date. + Parse the end date. If it's not defined it will be the same date that start_date (to do only one day). :param end_date: Date to the last day to simulate in string format. diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index 75573dd..3101a2f 100644 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -156,7 +156,7 @@ class EmissionInventory(object): def create_pollutants_dicts(self, pollutants): """ - Creates a list of dictionaries with the information of the name, paht and Dataset of each pollutant + Create a list of dictionaries with the information of the name, paht and Dataset of each pollutant :param pollutants: List of pollutants names :type pollutants: list @@ -262,7 +262,7 @@ class EmissionInventory(object): @staticmethod def make_emission_list(options, grid, vertical_output_profile, date): """ - Extracts the information of the cross table to read all the needed emissions. + Extract the information of the cross table to read all the needed emissions. :param options: Full list of parameters given by passing argument or in the configuration file. :type options: Namespace diff --git a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py index e53c8da..7490a0e 100755 --- a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py @@ -116,7 +116,7 @@ class GfasEmissionInventory(EmissionInventory): def get_altitude(self): """ - Extracts the altitude values depending on the choosen method. + Extract the altitude values depending on the choosen method. :return: Array with the alittude of each fire. :rtype: numpy.ndarray @@ -144,7 +144,7 @@ class GfasEmissionInventory(EmissionInventory): @ staticmethod def get_approach(p_vertical): """ - Extracts the given approach value. + Extract the given approach value. :return: Approach value :rtype: str @@ -167,7 +167,7 @@ class GfasEmissionInventory(EmissionInventory): @ staticmethod def get_method(p_vertical): """ - Extracts the given method value. + Extract the given method value. :return: Method value :rtype: str diff --git a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py index 2238a73..8c22e11 100755 --- a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py @@ -142,7 +142,7 @@ class PointSourceEmissionInventory(EmissionInventory): def calculate_altitudes(self, vertical_description_path): """ - Calculates the number layer to allocate the point source. + Calculate the number layer to allocate the point source. :param vertical_description_path: Path to the file that contains the vertical description :type vertical_description_path: str diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index 250ef8d..b9e19b2 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -97,7 +97,7 @@ class Grid(object): lat_1, lat_2, lon_0, lat_0, nx, ny, inc_x, inc_y, x_0, y_0, lat_ts): # TODO describe better the rotated parameters """ - Creates a Grid object depending on the grid type. + Create a Grid object depending on the grid type. :param grid_type: type of grid to create [global, rotated, lcc, mercator] :type grid_type: str @@ -208,7 +208,7 @@ class Grid(object): @staticmethod def set_vertical_levels(vertical_description_path): """ - Extracts the vertical levels. + Extract the vertical levels. :param vertical_description_path: path to the file that contain the vertical description of the required output file. @@ -248,7 +248,7 @@ class Grid(object): boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, RegularLatLon=True) - # Calculates the cell area of the auxiliary NetCDF file + # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() # Re-writes the NetCDF adding the cell area @@ -263,7 +263,7 @@ class Grid(object): def get_cell_area(self): """ - Calculates the cell area of the grid. + Calculate the cell area of the grid. :return: Area of each cell of the grid. :rtype: numpy.array @@ -276,7 +276,7 @@ class Grid(object): # Initialises the CDO cdo = Cdo() - # Creates a temporal file 's' with the cell area + # Create a temporal file 's' with the cell area s = cdo.gridarea(input=self.coords_netcdf_file) # Get the cell area of the temporal file nc_aux = Dataset(s, mode='r') @@ -290,7 +290,7 @@ class Grid(object): @staticmethod def create_regular_grid_1d_array(center, inc, boundary): """ - Creates a regular grid giving the center, boundary and increment. + Create a regular grid giving the center, boundary and increment. :param center: Center of the coordinates. :type center: float @@ -307,11 +307,11 @@ class Grid(object): st_time = timeit.default_timer() - # Calculates first center point. + # Calculate first center point. origin = center - abs(boundary) - # Calculates the quantity of cells. + # Calculate the quantity of cells. n = (abs(boundary) / inc) * 2 - # Calculates all the values + # Calculate all the values values = np.arange(origin + inc, origin + (n * inc) - inc + inc / 2, inc, dtype=np.float) settings.write_time('Grid', 'create_regular_grid_1d_array', timeit.default_timer() - st_time, level=3) @@ -321,7 +321,7 @@ class Grid(object): @staticmethod def create_bounds(coords, inc, number_vertices=2, inverse=False): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) :type coords: numpy.array @@ -342,17 +342,17 @@ class Grid(object): st_time = timeit.default_timer() settings.write_log('\t\t\tCreating boundaries.', level=3) - # Creates new arrays moving the centers half increment less and more. + # Create new arrays moving the centers half increment less and more. coords_left = coords - inc / 2 coords_right = coords + inc / 2 # Defining the number of corners needed. 2 to regular grids and 4 for irregular ones. if number_vertices == 2: - # Creates an array of N arrays of 2 elements to store the floor and the ceil values for each cell + # Create an array of N arrays of 2 elements to store the floor and the ceil values for each cell bound_coords = np.dstack((coords_left, coords_right)) bound_coords = bound_coords.reshape((len(coords), number_vertices)) elif number_vertices == 4: - # Creates an array of N arrays of 4 elements to store the corner values for each cell + # Create an array of N arrays of 4 elements to store the corner values for each cell # It can be stored in clockwise starting form the left-top element, or in inverse mode. if inverse: bound_coords = np.dstack((coords_left, coords_left, coords_right, coords_right)) diff --git a/hermesv3_gr/modules/grids/grid_lcc.py b/hermesv3_gr/modules/grids/grid_lcc.py index 43c535f..f66ea65 100644 --- a/hermesv3_gr/modules/grids/grid_lcc.py +++ b/hermesv3_gr/modules/grids/grid_lcc.py @@ -148,7 +148,7 @@ class LccGrid(Grid): LambertConformalConic=True, lcc_x=self.x, lcc_y=self.y, lat_1_2="{0}, {1}".format(self.lat_1, self.lat_2), lon_0=self.lon_0, lat_0=self.lat_0) - # Calculates the cell area of the auxiliary NetCDF file + # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() # Re-writes the NetCDF adding the cell area @@ -173,7 +173,7 @@ class LccGrid(Grid): st_time = timeit.default_timer() settings.write_log('\t\tCreating lcc coordinates', level=3) - # Creates a regular grid in metres (Two 1D arrays) + # Create a regular grid in metres (Two 1D arrays) self.x = np.arange(self.x_0, self.x_0 + self.inc_x * self.nx, self.inc_x, dtype=np.float) if len(self.x)//2 < settings.size: settings.write_log('ERROR: Check the .err file to get more info.') @@ -191,7 +191,7 @@ class LccGrid(Grid): y_b = super(LccGrid, self).create_bounds(y, self.inc_y, number_vertices=4, inverse=True) x_b = super(LccGrid, self).create_bounds(x, self.inc_x, number_vertices=4) - # Creates the LCC projection + # Create the LCC projection projection = Proj( proj='lcc', ellps='WGS84', diff --git a/hermesv3_gr/modules/grids/grid_mercator.py b/hermesv3_gr/modules/grids/grid_mercator.py index f537130..cce1491 100644 --- a/hermesv3_gr/modules/grids/grid_mercator.py +++ b/hermesv3_gr/modules/grids/grid_mercator.py @@ -134,7 +134,7 @@ class MercatorGrid(Grid): boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, Mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) - # Calculates the cell area of the auxiliary NetCDF file + # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() # Re-writes the NetCDF adding the cell area @@ -161,7 +161,7 @@ class MercatorGrid(Grid): st_time = timeit.default_timer() - # Creates a regular grid in metres (Two 1D arrays) + # Create a regular grid in metres (Two 1D arrays) self.x = np.arange(self.x_0, self.x_0 + self.inc_x * self.nx, self.inc_x, dtype=np.float) if len(self.x)//2 < settings.size: settings.write_log('ERROR: Check the .err file to get more info.') @@ -179,7 +179,7 @@ class MercatorGrid(Grid): y_b = super(MercatorGrid, self).create_bounds(y, self.inc_y, number_vertices=4, inverse=True) x_b = super(MercatorGrid, self).create_bounds(x, self.inc_x, number_vertices=4) - # Creates the LCC projection + # Create the LCC projection projection = Proj(self.crs) # UTM to Mercator diff --git a/hermesv3_gr/modules/grids/grid_rotated.py b/hermesv3_gr/modules/grids/grid_rotated.py index a21fdbc..8a6f544 100644 --- a/hermesv3_gr/modules/grids/grid_rotated.py +++ b/hermesv3_gr/modules/grids/grid_rotated.py @@ -127,7 +127,7 @@ class RotatedGrid(Grid): def rotated2latlon(self, lon_deg, lat_deg, lon_min=-180): """ - Calculates the unrotated coordinates using the rotated ones. + Calculate the unrotated coordinates using the rotated ones. :param lon_deg: Rotated longitude coordinate. :type lon_deg: numpy.array @@ -216,7 +216,7 @@ class RotatedGrid(Grid): north_pole_lat=self.new_pole_latitude_degrees, north_pole_lon=self.new_pole_longitude_degrees) - # Calculates the cell area of the auxiliary NetCDF file + # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() # Re-writes the NetCDF adding the cell area diff --git a/hermesv3_gr/modules/regrid/regrid.py b/hermesv3_gr/modules/regrid/regrid.py index d533678..9d224a9 100644 --- a/hermesv3_gr/modules/regrid/regrid.py +++ b/hermesv3_gr/modules/regrid/regrid.py @@ -51,7 +51,7 @@ class Regrid(object): def apply_weights(self, values): """ - Calculates the regridded values using the ESMF algorithm for a 3D array. + Calculate the regridded values using the ESMF algorithm for a 3D array. :param values: Input values to regrid :type values: numpy.array diff --git a/hermesv3_gr/modules/regrid/regrid_conservative.py b/hermesv3_gr/modules/regrid/regrid_conservative.py index ff8d509..f22aece 100644 --- a/hermesv3_gr/modules/regrid/regrid_conservative.py +++ b/hermesv3_gr/modules/regrid/regrid_conservative.py @@ -166,7 +166,7 @@ class ConservativeRegrid(Regrid): def apply_weights(self, values): """ - Calculates the regridded values using the ESMF algorithm for a 3D array specifically for a conservative regrid. + Calculate the regridded values using the ESMF algorithm for a 3D array specifically for a conservative regrid. :param values: Input values to regrid. :type values: numpy.array diff --git a/hermesv3_gr/modules/speciation/speciation.py b/hermesv3_gr/modules/speciation/speciation.py index ac6d17b..6084c44 100644 --- a/hermesv3_gr/modules/speciation/speciation.py +++ b/hermesv3_gr/modules/speciation/speciation.py @@ -50,7 +50,7 @@ class Speciation(object): def get_speciation_profile(self, speciation_profile_path): """ - Extracts the speciation information as a dictionary with the destiny pollutant as key and the formula as value. + Extract the speciation information as a dictionary with the destiny pollutant as key and the formula as value. :param speciation_profile_path: :type speciation_profile_path: @@ -95,7 +95,7 @@ class Speciation(object): @staticmethod def extract_molecular_weights(molecular_weights_path): """ - Extracts the molecular weights for each pollutant as a dictionary with the name of the pollutant as key and the + Extract the molecular weights for each pollutant as a dictionary with the name of the pollutant as key and the molecular weight as value. :param molecular_weights_path: Path to the CSV that contains all the molecular weights. diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 5da069d..5dcc331 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -246,7 +246,7 @@ class TemporalDistribution(object): @staticmethod def parse_tz(timezone): """ - Parse the timezone (string format). + Parse the timezone (string format). It is needed because some libraries have more timezones than others and it tries to simplify setting the strange ones into the nearest common one. diff --git a/hermesv3_gr/modules/vertical/vertical.py b/hermesv3_gr/modules/vertical/vertical.py index de9041f..ccde876 100644 --- a/hermesv3_gr/modules/vertical/vertical.py +++ b/hermesv3_gr/modules/vertical/vertical.py @@ -50,7 +50,7 @@ class VerticalDistribution(object): def get_vertical_profile(self, path): """ - Extracts the vertical v_profile from the vertical v_profile file. + Extract the vertical v_profile from the vertical v_profile file. :param path: Path to the file that contains all the vertical profiles. :type path: str @@ -93,7 +93,7 @@ class VerticalDistribution(object): @staticmethod def get_vertical_output_profile(path): """ - Extracts the vertical description of the desired output. + Extract the vertical description of the desired output. :param path: Path to the file that contains the output vertical description. :type path: str @@ -118,7 +118,7 @@ class VerticalDistribution(object): @staticmethod def get_weights(prev_layer, layer, in_weight, output_vertical_profile): """ - Calculates the weights for the given layer. + Calculate the weights for the given layer. :param prev_layer: Altitude of the low layer. 0 if it's the first. :type prev_layer: float @@ -151,7 +151,7 @@ class VerticalDistribution(object): def calculate_weights(self): """ - Calculates the weights for all the vertical layers. + Calculate the weights for all the vertical layers. :return: Weights that goes to each layer. :rtype: list of float @@ -177,7 +177,7 @@ class VerticalDistribution(object): @staticmethod def apply_weights(data, weights): """ - Calculates the vertical distribution using the given data and weights. + Calculate the vertical distribution using the given data and weights. :param data: Emissions to be vertically distributed. :type data: numpy.array diff --git a/hermesv3_gr/modules/vertical/vertical_gfas.py b/hermesv3_gr/modules/vertical/vertical_gfas.py index 08caf9f..70f0414 100644 --- a/hermesv3_gr/modules/vertical/vertical_gfas.py +++ b/hermesv3_gr/modules/vertical/vertical_gfas.py @@ -45,7 +45,7 @@ class GfasVerticalDistribution(VerticalDistribution): @staticmethod def calculate_widths(heights_list): """ - Calculates the width of each vertical level. + Calculate the width of each vertical level. :param heights_list: List of the top altitude in meters of each level. :type heights_list: list @@ -67,7 +67,7 @@ class GfasVerticalDistribution(VerticalDistribution): def get_weights(self, heights_list): """ - Calculates the proportion (%) of emission to put on each layer. + Calculate the proportion (%) of emission to put on each layer. :param heights_list: List with the width of each vertical level. :type heights_list: list diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 7da5172..a7efdd9 100644 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -18,14 +18,13 @@ # along with HERMESv3_GR. If not, see . -import os import sys -from hermesv3_gr.modules.writing.writer import Writer import timeit -from hermesv3_gr.config import settings import numpy as np from netCDF4 import Dataset from mpi4py import MPI +from hermesv3_gr.modules.writing.writer import Writer +from hermesv3_gr.config import settings class WriterCmaq(Writer): @@ -39,6 +38,13 @@ class WriterCmaq(Writer): 'XCELL', 'YCELL', 'VGTYP', 'VGTOP', 'VGLVLS', 'GDNAM', 'UPNAM', 'FILEDESC', 'HISTORY', 'VAR-LIST'] def unit_change(self, variable, data): + # TODO Documentation + """ + + :param variable: + :param data: + :return: + """ from cf_units import Unit if data is not None: @@ -63,7 +69,7 @@ class WriterCmaq(Writer): @staticmethod def change_variable_attributes(emission_list): """ - Modifies the emission list to be consistent to use the output as input for CMAQ model. + Modify the emission list to be consistent to use the output as input for CMAQ model. :param emission_list: List of emissions :type emission_list: list @@ -88,7 +94,7 @@ class WriterCmaq(Writer): @staticmethod def create_tflag(st_date, hours_array, num_vars): """ - Creates the content of the CMAQ variable TFLAG + Create the content of the CMAQ variable TFLAG :param st_date: Starting date :type st_date: datetime.datetime @@ -103,7 +109,6 @@ class WriterCmaq(Writer): :return: Array with the content of TFLAG :rtype: numpy.array """ - import numpy as np from datetime import timedelta a = np.array([[[]]]) @@ -127,7 +132,6 @@ class WriterCmaq(Writer): :return: List transformed on string. :rtype: str """ - str_var_list = "" for var in var_list: str_var_list += "{:<16}".format(var) @@ -135,6 +139,11 @@ class WriterCmaq(Writer): return str_var_list def read_global_attributes(self): + # TODO Documentation + """ + + :return: + """ import pandas as pd from warnings import warn as warning float_atts = ['VGTOP'] @@ -186,95 +195,14 @@ class WriterCmaq(Writer): def create_global_attributes(self, var_list): """ - Creates the global attributes and the order that they have to be filled. - - :param date: Starting date. - :type date: datetime.datetime - - :param nx: Number of elements on the x dimension. - :type nx: int - - :param ny: Number of elements on the y dimension. - :type ny: int - - :param nlays: Number of vertical layers. - :type nlays: int - - :param lat_1: Value of lat 1 of the Lambert Conformal Conic projection. - :type lat_1: float - - :param lat_2: Value of lat 2 of the Lambert Conformal Conic projection. - :type lat_2: float - - :param lon_0: Value of lon 0 of the Lambert Conformal Conic projection. - :type lon_0: float - - :param lat_0: Value of lat 0 of the Lambert Conformal Conic projection. - :type lat_0: float - - :param x_0: X value of the origin. - :type x_0: float - - :param y_0: Y value of the origin. - :type y_0: float - - :param inc_x: Increment (in meters) of the x values. - :type inc_x: float - - :param inc_y: Increment (in meters) of the y values. - :type inc_x: float + Create the global attributes and the order that they have to be filled. - :param var_list: List of variables. + :param var_list: List of variables :type var_list: list - :param exec_id: ID of the execution. - :type exec_id: str - - :param ftype: File data type = [CUSTOM3:1, GRDDED3:2, BNDARY3:3, IDDATA3:4, PROFIL3:5, or SMATRX3:6] - (Default = 1) - :type ftype: int - - :param tstep: time step, coded HHMMSS according to Models-3 conventions. - :type tstep: int - - :param nthik: For BNDARY3 files, perimeter thickness (cells), or for SMATRX3 files, number of matrix-columns - (unused for other file types) - :type nthik: int - - :param gdtyp: Map projection type - LATGRD3=1 (Lat-Lon), - LAMGRD3=2 (Lambert conformal conic), - MERGRD3=3 (general tangent Mercator), - STEGRD3=4 (general tangent stereographic), - UTMGRD3=5 (UTM, a special case of Mercator), - POLGRD3=6 (polar secant stereographic), - EQMGRD3=7 (equatorial secant Mercator), or - TRMGRD3=8 (transverse secant Mercator) - :type gdtyp: int - - :param vgtype: Vertical coordinate type - VGSGPH3=1 (hydrostatic sigma-P), - VGSGPN3=2 (nonhydrostatic sigma-P), - VGSIGZ3=3 (sigma-Z), - VGPRES3=4 (pressure (mb)), - VGZVAL3=5 (Z (m above sea lvl), or - VGHVAL3=6 (H (m above ground)) - :type vgtype: int - - :param vgtop: Model-top, for sigma vertical-coordinate types - :type vgtop: int - - :param vglvls: Array of vertical coordinate level values; level 1 of the grid goes from vertical coordinate - VGLEVELS[0] to VGLEVELS[1], etc. - :type vglvls: numpy.array - - :param gdnam: Grid Name - :type gdnam: str - :return: Dict of global attributes and a list with the keys ordered. :rtype: tuple """ - # TODO documentation from datetime import datetime global_attributes = self.read_global_attributes() @@ -323,6 +251,30 @@ class WriterCmaq(Writer): def create_cmaq_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, regular_lat_lon=False, rotated=False, nx=None, ny=None, lat_1=None, lat_2=None, lon_0=None, lat_0=None, x_0=None, y_0=None, inc_x=None, inc_y=None): + # TODO Documentation + """ + + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param regular_lat_lon: + :param rotated: + :param nx: + :param ny: + :param lat_1: + :param lat_2: + :param lon_0: + :param lat_0: + :param x_0: + :param y_0: + :param inc_x: + :param inc_y: + :return: + """ data_list, var_list = WriterCmaq.change_variable_attributes(data_list) @@ -346,10 +298,21 @@ class WriterCmaq(Writer): @staticmethod def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, global_attributes=None, regular_lat_lon=False, rotated=False): + # TODO Documentation + """ - import sys - from netCDF4 import Dataset - + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param global_attributes: + :param regular_lat_lon: + :param rotated: + :return: + """ if regular_lat_lon: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -394,6 +357,10 @@ class WriterCmaq(Writer): netcdf.close() def create_parallel_netcdf(self): + # TODO Documentation + """ + Create an empty netCDF + """ st_time = timeit.default_timer() settings.write_log("\tCreating parallel NetCDF file.", level=2) # netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) @@ -450,6 +417,15 @@ class WriterCmaq(Writer): settings.write_time('WriterCmaq', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) def write_parallel_netcdf(self, emission_list): + """ + Write the netCDF in parallel mode. + + :param emission_list: List of the processed emissions for the different emission inventories + :type emission_list: list + + :return: True when it finish well. + :rtype: bool + """ st_time = timeit.default_timer() settings.write_log("\tAppending data to parallel NetCDF file.", level=2) @@ -476,8 +452,18 @@ class WriterCmaq(Writer): netcdf.close() settings.write_time('WriterCmaq', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) + return True def write_serial_netcdf(self, emission_list): + """ + Write the netCDF in serial mode. + + :param emission_list: List of the processed emissions for the different emission inventories + :type emission_list: list + + :return: True when it finish well. + :rtype: bool + """ st_time = timeit.default_timer() mpi_numpy = False @@ -610,10 +596,6 @@ class WriterCmaq(Writer): sys.exit(1) elif mpi_vector: - var_time = timeit.default_timer() - - # data_list = []#np.empty(shape, dtype=np.float64) - if rank_data is not None: data = np.empty(var[:].shape, dtype=settings.precision) for i in xrange(settings.size): diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index 6711880..51f776f 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -217,7 +217,7 @@ class WriterWrfChem(Writer): def create_global_attributes(self): # TODO Documentation """ - Creates the global attributes that have to be filled. + Create the global attributes that have to be filled. """ global_attributes = self.read_global_attributes() diff --git a/hermesv3_gr/tools/coordinates_tools.py b/hermesv3_gr/tools/coordinates_tools.py index 51d074a..a61e0f2 100644 --- a/hermesv3_gr/tools/coordinates_tools.py +++ b/hermesv3_gr/tools/coordinates_tools.py @@ -17,19 +17,16 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import os -import sys - - -# Global variables - def get_grid_area(filename): - # TODO Documentation """ + Calculate the area for each cell of the grid using CDO - :param filename: - :return: + :param filename: Path to the file to calculate the cell area + :type filename: str + + :return: Area of each cell of the grid. + :rtype: numpy.array """ from cdo import Cdo from netCDF4 import Dataset @@ -59,16 +56,16 @@ def latlon2rotated(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): degrees_to_radians = math.pi / 180. radians_to_degrees = 180. / math.pi - lon_max = lon_min + 360 + # lon_max = lon_min + 360 # stlm=sin(tlm) sin_lat_pole_rad = math.sin(lat_pole_deg * degrees_to_radians) # ctlm=cos(tlm) cos_lat_pole_rad = math.cos(lat_pole_deg * degrees_to_radians) # stph=sin(tph) - sin_lon_pole_rad = math.sin(lon_pole_deg * degrees_to_radians) + # sin_lon_pole_rad = math.sin(lon_pole_deg * degrees_to_radians) # ctph=cos(tph) - cos_lon_pole_rad = math.cos(lon_pole_deg * degrees_to_radians) + # cos_lon_pole_rad = math.cos(lon_pole_deg * degrees_to_radians) # relm=(xlon-tlm0d)*dtr !distance from the centre lon (in rad) distance_from_center_lon = (lon_deg - lon_pole_deg) * degrees_to_radians @@ -117,7 +114,7 @@ def rotated2latlon(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): import math degrees_to_radians = math.pi / 180. - radians_to_degrees = 180. / math.pi + # radians_to_degrees = 180. / math.pi # Positive east to negative east lon_pole_deg -= 180 @@ -157,7 +154,7 @@ def rotated2latlon(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): # almd += 360 # elif almd > max_lon: # almd -= 360 - # TODO use lon_min + almd[almd > (lon_min + 360)] -= 360 almd[almd < lon_min] += 360 @@ -178,44 +175,9 @@ def rotated2latlon_single(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min= import math degrees_to_radians = math.pi / 180. - radians_to_degrees = 180. / math.pi + # radians_to_degrees = 180. / math.pi # lon_max = lon_min + 360 - # - # sin_lat_pole_rad = math.sin(lat_pole_deg*degrees_to_radians) - # cos_lat_pole_rad = math.cos(lat_pole_deg*degrees_to_radians) # - # sin_lon_pole_rad = math.sin(lon_pole_deg*degrees_to_radians) # stph - # cos_lon_pole_rad = math.cos(lon_pole_deg*degrees_to_radians) # ctph - # - # - # # relm=(xlon-tlm0d)*dtr !distance from the centre lon (in rad) - # distance_from_center_lon = (lon_deg - lon_pole_deg)*degrees_to_radians - # # ctlm=cos(relm) !cos of this distance - # cos_distance_from_center_lon = math.cos(distance_from_center_lon) - # # stlm=sin(relm) !sin of this distance - # sin_distance_from_center_lon = math.sin(distance_from_center_lon) - # # aph=xlat*dtr !lat in rad - # lat_rad = lat_deg*degrees_to_radians - # # ctph=cos(aph) !cos of lat - # cos_lat_rad = math.cos(lat_rad) - # # stph=sin(aph) !sin of lat - # sin_lat_rad = math.sin(lat_rad) - # - # - # - # # sph=ctph0*stph+stph0*ctph*ctlm - # sin_rotated_lat = (cos_lat_pole_rad*sin_lat_rad) + - # (sin_distance_from_center_lon*cos_lat_rad*cos_distance_from_center_lon) - # # sph=min(sph,1.) - # # sph=max(sph,-1.) - # if sin_rotated_lat > 1.: - # sin_rotated_lat = 1. - # if sin_rotated_lat < -1.: - # sin_rotated_lat = -1. - # # aph=asin(sph) - # real_latitude = math.asin(sin_rotated_lat) - # real_longitude = math.atan2(cos_lat_rad*sin_distance_from_center_lon, - # (cos_distance_from_center_lon*cos_lat_rad - sin_lat_pole_rad*sin_rotated_lat)/cos_lat_pole_rad) - math.pi # Positive east to negative east lon_pole_deg -= 180 @@ -258,7 +220,7 @@ def rotated2latlon_single(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min= def create_bounds(coords, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) :type coords: numpy.ndarray @@ -298,7 +260,7 @@ def create_bounds_esmpy(coords, spheric=False): interval = coords[1] - coords[0] - bound_coords = coords - interval/2 + bound_coords = coords - interval / 2 if not spheric: bound_coords = np.append(bound_coords, [bound_coords[-1] + interval]) @@ -322,9 +284,6 @@ def create_regular_rotated(lat_origin, lon_origin, lat_inc, lon_inc, n_lat, n_lo center_latitudes = np.arange(lat_origin, lat_origin + (n_lat*lat_inc), lat_inc, dtype=np.float) center_longitudes = np.arange(lon_origin, lon_origin + (n_lon*lon_inc), lon_inc, dtype=np.float) - # print lat_origin + (n_lat*lat_inc) - # print n_lat*lat_inc - corner_latitudes = create_bounds_esmpy(center_latitudes) corner_longitudes = create_bounds_esmpy(center_longitudes) @@ -338,65 +297,15 @@ def create_regular_old(lat_origin, lon_origin, lat_inc, lon_inc, n_lat, n_lon): center_latitudes = np.arange(lat_origin, lat_origin + (n_lat*lat_inc), lat_inc, dtype=np.float) center_longitudes = np.arange(lon_origin, lon_origin + (n_lon*lon_inc), lon_inc, dtype=np.float) - # print lat_origin + (n_lat*lat_inc) - # print n_lat*lat_inc - corner_latitudes = create_bounds(center_latitudes) corner_longitudes = create_bounds(center_longitudes) return center_latitudes, center_longitudes, corner_latitudes, corner_longitudes -# def create_regular_grid(center_lat, center_lon, west_boundary, south_boundary, inc_lat, inc_lon): -# """ -# Creates a custom grid with the given parameters. The grid is divided in 4 arrays: -# - Center Latitudes -# - Center Longitudes -# - Boundary Latitudes (# latitudes +1) -# - Boundary Longitudes (# longitudes +1) -# -# :param center_lat: Latitude of the center of the grid (degrees). -# :type center_lat: float -# -# :param center_lon: Longitude of the center of the grid (degrees). -# :type center_lon: float -# -# :param west_boundary: Distance from de center to the western boundary (degrees) -# (not to the center of the first cell) -# :type west_boundary: float -# -# :param south_boundary: Distance from de center to the southern boundary (degrees) -# (not to the center of the first cell) -# :type south_boundary: float -# -# :param inc_lat: Vertical resolution of each cell (degrees). -# :type inc_lat: float -# -# :param inc_lon: Horizontal resolution of each cell (degrees) -# :type inc_lon: float -# -# :return: Arrays with the Center Latitudes, Center Longitudes, Boundary Latitudes, Boundary Longitudes. -# :rtype: tuple (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) -# """ -# lat_origin = center_lat - abs(south_boundary) + (inc_lat/2) -# lon_origin = center_lon - abs(west_boundary) + (inc_lon/2) -# n_lat = (abs(south_boundary)/inc_lat)*2 -# n_lon = (abs(west_boundary)/inc_lon)*2 -# -# center_latitudes = np.arange(lat_origin, lat_origin + (n_lat*inc_lat), inc_lat, dtype=np.float) -# center_longitudes = np.arange(lon_origin, lon_origin + (n_lon*inc_lon), inc_lon, dtype=np.float) -# -# corner_latitudes = create_bounds(center_latitudes) -# corner_longitudes = create_bounds(center_longitudes) -# -# # print center_latitudes -# -# return center_latitudes, center_longitudes, corner_latitudes, corner_longitudes - - def create_regular_grid(center_lat, center_lon, west_boundary, south_boundary, inc_lat, inc_lon): """ - Creates a custom grid with the given parameters. The grid is divided in 4 arrays: + Create a custom grid with the given parameters. The grid is divided in 4 arrays: - Center Latitudes - Center Longitudes - Boundary Latitudes (# latitudes +1) @@ -432,44 +341,44 @@ def create_regular_grid(center_lat, center_lon, west_boundary, south_boundary, i n_lat = (abs(south_boundary)/inc_lat)*2 n_lon = (abs(west_boundary)/inc_lon)*2 - center_latitudes = np.arange(lat_origin + inc_lat, lat_origin + (n_lat*inc_lat) - inc_lat + inc_lat/2, inc_lat, + center_latitudes = np.arange(lat_origin + inc_lat, lat_origin + (n_lat*inc_lat) - inc_lat + inc_lat / 2, inc_lat, dtype=np.float) - center_longitudes = np.arange(lon_origin + inc_lon, lon_origin + (n_lon*inc_lon) - inc_lon + inc_lon/2, inc_lon, + center_longitudes = np.arange(lon_origin + inc_lon, lon_origin + (n_lon*inc_lon) - inc_lon + inc_lon / 2, inc_lon, dtype=np.float) corner_latitudes = create_bounds(center_latitudes) corner_longitudes = create_bounds(center_longitudes) center_latitudes = np.concatenate([ - [lat_origin + inc_lat/2 - inc_lat/4], + [lat_origin + inc_lat / 2 - inc_lat / 4], center_latitudes, - [lat_origin + (n_lat*inc_lat) - inc_lat/2 + inc_lat/4]]) + [lat_origin + (n_lat * inc_lat) - inc_lat / 2 + inc_lat / 4]]) center_longitudes = np.concatenate([ - [lon_origin + inc_lon/2 - inc_lon/4], + [lon_origin + inc_lon / 2 - inc_lon / 4], center_longitudes, - [lon_origin + (n_lon*inc_lon) - inc_lon/2 + inc_lon/4]]) + [lon_origin + (n_lon * inc_lon) - inc_lon / 2 + inc_lon / 4]]) corner_latitudes = np.concatenate([ - [[[lat_origin, lat_origin + inc_lat/2]]], + [[[lat_origin, lat_origin + inc_lat / 2]]], corner_latitudes, - [[[lat_origin + (n_lat*inc_lat) - inc_lat/2, lat_origin + (n_lat*inc_lat)]]]], axis=1) + [[[lat_origin + (n_lat * inc_lat) - inc_lat / 2, lat_origin + (n_lat * inc_lat)]]]], axis=1) corner_longitudes = np.concatenate([ - [[[lon_origin, lon_origin + inc_lon/2]]], + [[[lon_origin, lon_origin + inc_lon / 2]]], corner_longitudes, - [[[lon_origin + (n_lon*inc_lon) - inc_lon/2, lon_origin + (n_lon*inc_lon)]]]], axis=1) + [[[lon_origin + (n_lon * inc_lon) - inc_lon / 2, lon_origin + (n_lon * inc_lon)]]]], axis=1) return center_latitudes, center_longitudes, corner_latitudes, corner_longitudes if __name__ == '__main__': import numpy as np - new_pole_longitude_degrees = 20.0 # lonpole tlm0d - new_pole_latitude_degrees = 35.0 # latpole tph0d - print latlon2rotated(new_pole_longitude_degrees, new_pole_latitude_degrees, 20.0, 35.0) - print latlon2rotated(new_pole_longitude_degrees, new_pole_latitude_degrees, -20.2485, -9.9036) - print rotated2latlon_single(new_pole_longitude_degrees, new_pole_latitude_degrees, 0, 0) - print rotated2latlon_single(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -35.) - print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, np.array([-51., -51., -51., -51.]), + new_pole_lon_d = 20.0 # lonpole tlm0d + new_pole_lat_d = 35.0 # latpole tph0d + print latlon2rotated(new_pole_lon_d, new_pole_lat_d, 20.0, 35.0) + print latlon2rotated(new_pole_lon_d, new_pole_lat_d, -20.2485, -9.9036) + print rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, 0, 0) + print rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, -51., -35.) + print rotated2latlon(new_pole_lon_d, new_pole_lat_d, np.array([-51., -51., -51., -51.]), np.array([-35., -34.9, -34.8, -34.7])) diff --git a/hermesv3_gr/tools/custom_calendar.py b/hermesv3_gr/tools/custom_calendar.py index 256bda9..c221328 100644 --- a/hermesv3_gr/tools/custom_calendar.py +++ b/hermesv3_gr/tools/custom_calendar.py @@ -26,7 +26,7 @@ import holidays def custom_holidays(zone, year): """ - Calculates the festivity days that appear in the library holidays adding the Maundy Thursday and the God Friday + Calculate the festivity days that appear in the library holidays adding the Maundy Thursday and the God Friday :param zone: Name of the country. It has to appear and has to have the same format (capital letters) of the library holidays: https://pypi.python.org/pypi/holidays @@ -69,7 +69,7 @@ def get_holidays(zone, year): def pascua(year): """ - Calculates the "Pascua" date + Calculate the "Pascua" date :param year: Year to found the Pascua. :type year: int diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index 6a50c73..74188bb 100644 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -22,25 +22,40 @@ import sys from netCDF4 import Dataset from mpi4py import MPI -icomm = MPI.COMM_WORLD -comm = icomm.Split(color=0, key=0) -rank = comm.Get_rank() -size = comm.Get_size() +ICOMM = MPI.COMM_WORLD +COMM = ICOMM.Split(color=0, key=0) +RANK = COMM.Get_rank() +SIZE = COMM.Get_size() def open_netcdf(netcdf_path): - from netCDF4 import Dataset - nc_out = Dataset(netcdf_path, mode='a') - return nc_out + """ + Open a netCDF file. + + :param netcdf_path: Path to the netCDF file. + :type netcdf_path: str + :return: netCDF + :rtype: Dataset + """ + netcdf = Dataset(netcdf_path, mode='a') + return netcdf -def close_netcdf(nc): - nc.close() + +def close_netcdf(netcdf): + """ + Close the netCDF. + + :param netcdf: netCDF + :type netcdf: Dataset + :return: + """ + netcdf.close() def get_grid_area(filename): """ - Calculates the area of each cell. + Calculate the area of each cell. :param filename: Full path to the NetCDF to calculate the cell areas. :type filename: str @@ -51,17 +66,32 @@ def get_grid_area(filename): from cdo import Cdo cdo = Cdo() - s = cdo.gridarea(input=filename) - nc_aux = Dataset(s, mode='r') - grid_area = nc_aux.variables['cell_area'][:] - nc_aux.close() + src = cdo.gridarea(input=filename) + netcdf = Dataset(src, mode='r') + grid_area = netcdf.variables['cell_area'][:] + netcdf.close() return grid_area def extract_vars(netcdf_path, variables_list, attributes_list=list()): + """ + Get the data from the list of variabbles. + + :param netcdf_path: Path to the netCDF file + :type netcdf_path: str + + :param variables_list: List of the names of the variables to get. + :type variables_list: list + + :param attributes_list: List of the names of the variable attributes to get. + :type attributes_list: list + + :return: List of the variables from the netCDF as a dictionary with data as values and with the other keys their + attributes. + :rtype: list. + """ data_list = [] - # print netcdf_path netcdf = Dataset(netcdf_path, mode='r') for var in variables_list: if var == 'emi_nox_no2': @@ -89,6 +119,36 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, Mercator=False, lat_ts=None): + # TODO Documentation + """ + + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param boundary_latitudes: + :param boundary_longitudes: + :param cell_area: + :param global_attributes: + :param RegularLatLon: + :param Rotated: + :param rotated_lats: + :param rotated_lons: + :param north_pole_lat: + :param north_pole_lon: + :param LambertConformalConic: + :param lcc_x: + :param lcc_y: + :param lat_1_2: + :param lon_0: + :param lat_0: + :param Mercator: + :param lat_ts: + :return: + """ from cf_units import Unit, encode_time @@ -324,9 +384,35 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, regular_latlon=False, rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): + # TODO Documentation + """ + + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param boundary_latitudes: + :param boundary_longitudes: + :param cell_area: + :param global_attributes: + :param regular_latlon: + :param rotated: + :param rotated_lats: + :param rotated_lons: + :param north_pole_lat: + :param north_pole_lon: + :param lcc: + :param lcc_x: + :param lcc_y: + :param lat_1_2: + :param lon_0: + :param lat_0: + :return: + """ from cf_units import Unit, encode_time - import sys - from netCDF4 import Dataset import numpy as np if not (regular_latlon or lcc or rotated): @@ -553,6 +639,13 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, def tuple_to_index(tuple_list, bidimensional=False): + # TODO Documentation + """ + + :param tuple_list: + :param bidimensional: + :return: + """ from operator import mul new_list = [] for tuple in tuple_list: @@ -564,6 +657,12 @@ def tuple_to_index(tuple_list, bidimensional=False): def calculate_displacements(counts): + # TODO Documentation + """ + + :param counts: + :return: + """ new_list = [0] accum = 0 for counter in counts[:-1]: diff --git a/preproc/eclipsev5a_preproc.py b/preproc/eclipsev5a_preproc.py index 157dd81..1413710 100755 --- a/preproc/eclipsev5a_preproc.py +++ b/preproc/eclipsev5a_preproc.py @@ -44,7 +44,7 @@ var_units = 'kg.m-2.s-1' def get_grid_area(filename): """ - Calculates the area for each cell of the grid using CDO + Calculate the area for each cell of the grid using CDO :param filename: Path to the file to calculate the cell area :type filename: str @@ -66,7 +66,7 @@ def get_grid_area(filename): def create_bounds(coordinates, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coordinates: Coordinates in degrees (latitude or longitude) :type coordinates: numpy.ndarray diff --git a/preproc/edgarv432_ap_preproc.py b/preproc/edgarv432_ap_preproc.py index 2f79e9c..3dbf1df 100755 --- a/preproc/edgarv432_ap_preproc.py +++ b/preproc/edgarv432_ap_preproc.py @@ -87,7 +87,7 @@ def ipcc_to_sector_dict(): def create_bounds(coordinates, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coordinates: Coordinates in degrees (latitude or longitude) :type coordinates: numpy.array @@ -115,7 +115,7 @@ def create_bounds(coordinates, number_vertices=2): def get_grid_area(filename): """ - Calculates the area for each cell of the grid using CDO + Calculate the area for each cell of the grid using CDO :param filename: Path to the file to calculate the cell area :type filename: str diff --git a/preproc/edgarv432_voc_preproc.py b/preproc/edgarv432_voc_preproc.py index 3d7d116..ef834e1 100755 --- a/preproc/edgarv432_voc_preproc.py +++ b/preproc/edgarv432_voc_preproc.py @@ -56,7 +56,7 @@ Carles Tena Medina (carles.tena@bsc.es) from Barcelona Supercomputing Center (BS def create_bounds(coordinates, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coordinates: Coordinates in degrees (latitude or longitude) :type coordinates: numpy.ndarray @@ -84,7 +84,7 @@ def create_bounds(coordinates, number_vertices=2): def get_grid_area(filename): """ - Calculates the area for each cell of the grid using CDO + Calculate the area for each cell of the grid using CDO :param filename: Path to the file to calculate the cell area :type filename: str diff --git a/preproc/gfas12_preproc.py b/preproc/gfas12_preproc.py index b2591aa..3630052 100755 --- a/preproc/gfas12_preproc.py +++ b/preproc/gfas12_preproc.py @@ -39,7 +39,7 @@ PARAMETERS_FILE = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecm def create_bounds(coords, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) :type coords: numpy.ndarray @@ -69,7 +69,7 @@ def create_bounds(coords, number_vertices=2): def get_grid_area(filename): """ - Calculates the area for each cell of the grid using CDO + Calculate the area for each cell of the grid using CDO :param filename: Path to the file to calculate the cell area :type filename: str @@ -237,7 +237,7 @@ def do_transformation(input_file, date, output_dir, variables_list): def do_var_list(variables_file): """ - Creates the List of dictionaries + Create the List of dictionaries :param variables_file: CSV file with the information of each variable :type variables_file: str diff --git a/preproc/htapv2_preproc.py b/preproc/htapv2_preproc.py index 5961919..8766dde 100755 --- a/preproc/htapv2_preproc.py +++ b/preproc/htapv2_preproc.py @@ -300,7 +300,7 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): print out_path_aux write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], boundary_latitudes=create_bounds(c_lats['data']), - boundary_longitudes=create_bounds(c_lons['data']) ,global_attributes=global_attributes,) + boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) return True @@ -352,8 +352,8 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year data.update({'data': sol['data'] * r_sol['data']}) else: [r_inc, r_exf, r_sol] = extract_vars(ratio_file, ['inc', 'exf', 'sol']) - data.update({'data': ind['data'] * r_inc['data'] + exf['data']*r_exf['data'] + - sol['data'] * r_sol['data']}) + data.update({'data': ind['data'] * r_inc['data'] + exf['data'] * r_exf['data'] + + sol['data'] * r_sol['data']}) global_attributes = { 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, voc), @@ -520,7 +520,7 @@ def check_vocs(year): voc_sum += new_voc['data'].sum() print '{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum)/nmvoc_sum, month) + snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum) / nmvoc_sum, month) if __name__ == '__main__': -- GitLab From 8f945a1923e4e806401270772767cd23e7c5fa04 Mon Sep 17 00:00:00 2001 From: Carles Tena Medina Date: Tue, 11 Sep 2018 14:05:37 +0200 Subject: [PATCH 13/28] Correcting Code conventions --- .../gfas_emission_inventory.py | 2 +- hermesv3_gr/modules/grids/grid.py | 4 +- hermesv3_gr/modules/grids/grid_lcc.py | 4 +- hermesv3_gr/modules/grids/grid_mercator.py | 4 +- hermesv3_gr/modules/grids/grid_rotated.py | 4 +- hermesv3_gr/modules/masking/masking.py | 2 +- hermesv3_gr/modules/temporal/temporal.py | 6 +- hermesv3_gr/modules/vertical/vertical.py | 2 +- hermesv3_gr/modules/vertical/vertical_gfas.py | 10 +- hermesv3_gr/modules/writing/writer.py | 195 +++++++++++++----- hermesv3_gr/modules/writing/writer_cmaq.py | 26 +++ hermesv3_gr/modules/writing/writer_monarch.py | 180 ++++++++++------ .../modules/writing/writer_wrf_chem.py | 26 +++ hermesv3_gr/tools/coordinates_tools.py | 8 +- hermesv3_gr/tools/netcdf_tools.py | 66 +++--- preproc/ceds_preproc.py | 3 +- preproc/eclipsev5a_preproc.py | 24 +-- preproc/edgarv432_voc_preproc.py | 4 +- preproc/gfas12_preproc.py | 4 +- preproc/tno_mac_iii_preproc.py | 150 ++++++++++---- preproc/tno_mac_iii_preproc_voc_ratios.py | 8 +- 21 files changed, 504 insertions(+), 228 deletions(-) diff --git a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py index 7490a0e..9801db8 100755 --- a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py @@ -119,7 +119,7 @@ class GfasEmissionInventory(EmissionInventory): Extract the altitude values depending on the choosen method. :return: Array with the alittude of each fire. - :rtype: numpy.ndarray + :rtype: numpy.array """ from hermesv3_gr.tools.netcdf_tools import extract_vars diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index b9e19b2..e9227c6 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -246,7 +246,7 @@ class Grid(object): write_netcdf(self.coords_netcdf_file, self.center_latitudes, self.center_longitudes, [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, - RegularLatLon=True) + regular_latlon=True) # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() @@ -255,7 +255,7 @@ class Grid(object): write_netcdf(self.coords_netcdf_file, self.center_latitudes, self.center_longitudes, [{'name': 'var_aux', 'units': '', 'data': 0}], cell_area=self.cell_area, boundary_latitudes=self.boundary_latitudes, - boundary_longitudes=self.boundary_longitudes, RegularLatLon=True) + boundary_longitudes=self.boundary_longitudes, regular_latlon=True) else: self.cell_area = self.get_cell_area() diff --git a/hermesv3_gr/modules/grids/grid_lcc.py b/hermesv3_gr/modules/grids/grid_lcc.py index f66ea65..96ea0ec 100644 --- a/hermesv3_gr/modules/grids/grid_lcc.py +++ b/hermesv3_gr/modules/grids/grid_lcc.py @@ -145,7 +145,7 @@ class LccGrid(Grid): write_netcdf(self.coords_netcdf_file, self.center_latitudes, self.center_longitudes, [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, - LambertConformalConic=True, lcc_x=self.x, lcc_y=self.y, + lcc=True, lcc_x=self.x, lcc_y=self.y, lat_1_2="{0}, {1}".format(self.lat_1, self.lat_2), lon_0=self.lon_0, lat_0=self.lat_0) # Calculate the cell area of the auxiliary NetCDF file @@ -156,7 +156,7 @@ class LccGrid(Grid): [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, cell_area=self.cell_area, - LambertConformalConic=True, lcc_x=self.x, lcc_y=self.y, + lcc=True, lcc_x=self.x, lcc_y=self.y, lat_1_2="{0}, {1}".format(self.lat_1, self.lat_2), lon_0=self.lon_0, lat_0=self.lat_0) else: self.cell_area = self.get_cell_area() diff --git a/hermesv3_gr/modules/grids/grid_mercator.py b/hermesv3_gr/modules/grids/grid_mercator.py index cce1491..f3104fb 100644 --- a/hermesv3_gr/modules/grids/grid_mercator.py +++ b/hermesv3_gr/modules/grids/grid_mercator.py @@ -132,7 +132,7 @@ class MercatorGrid(Grid): write_netcdf(self.coords_netcdf_file, self.center_latitudes, self.center_longitudes, [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, - Mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) + mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() @@ -146,7 +146,7 @@ class MercatorGrid(Grid): ], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, cell_area=self.cell_area, - Mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) + mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) else: self.cell_area = self.get_cell_area() diff --git a/hermesv3_gr/modules/grids/grid_rotated.py b/hermesv3_gr/modules/grids/grid_rotated.py index 8a6f544..8566300 100644 --- a/hermesv3_gr/modules/grids/grid_rotated.py +++ b/hermesv3_gr/modules/grids/grid_rotated.py @@ -212,7 +212,7 @@ class RotatedGrid(Grid): [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, - Rotated=True, rotated_lats=self.rlat, rotated_lons=self.rlon, + roated=True, rotated_lats=self.rlat, rotated_lons=self.rlon, north_pole_lat=self.new_pole_latitude_degrees, north_pole_lon=self.new_pole_longitude_degrees) @@ -224,7 +224,7 @@ class RotatedGrid(Grid): [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, cell_area=self.cell_area, - Rotated=True, rotated_lats=self.rlat, rotated_lons=self.rlon, + roated=True, rotated_lats=self.rlat, rotated_lons=self.rlon, north_pole_lat=self.new_pole_latitude_degrees, north_pole_lon=self.new_pole_longitude_degrees) else: diff --git a/hermesv3_gr/modules/masking/masking.py b/hermesv3_gr/modules/masking/masking.py index 068c819..a6a98b6 100644 --- a/hermesv3_gr/modules/masking/masking.py +++ b/hermesv3_gr/modules/masking/masking.py @@ -112,7 +112,7 @@ class Masking(object): 'units': '', 'data': dst_var, }] - Writer.write_netcdf(self.world_mask_file, lat, lon, data, RegularLatLon=True) + Writer.write_netcdf(self.world_mask_file, lat, lon, data, regular_latlon=True) settings.comm.Barrier() settings.write_time('Masking', 'create_country_iso', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 5dcc331..55ed096 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -247,7 +247,7 @@ class TemporalDistribution(object): def parse_tz(timezone): """ Parse the timezone (string format). - + It is needed because some libraries have more timezones than others and it tries to simplify setting the strange ones into the nearest common one. Examples: @@ -358,7 +358,7 @@ class TemporalDistribution(object): dst_var = np.concatenate(dst_var, axis=2) data = [{'name': 'timezone_id', 'units': '', 'data': dst_var}] - write_netcdf(self.netcdf_timezones, total_lat, total_lon, data, RegularLatLon=True) + write_netcdf(self.netcdf_timezones, total_lat, total_lon, data, regular_latlon=True) settings.comm.Barrier() settings.write_time('TemporalDistribution', 'create_netcdf_timezones', timeit.default_timer() - st_time, @@ -753,7 +753,7 @@ class TemporalDistribution(object): def calculate_delta_hours(st_date, time_step_type, time_step_num, time_step_freq): # TODO Documentation """ - + :param st_date: :param time_step_type: :param time_step_num: diff --git a/hermesv3_gr/modules/vertical/vertical.py b/hermesv3_gr/modules/vertical/vertical.py index ccde876..1b309ea 100644 --- a/hermesv3_gr/modules/vertical/vertical.py +++ b/hermesv3_gr/modules/vertical/vertical.py @@ -186,7 +186,7 @@ class VerticalDistribution(object): :type weights: numpy.array :return: Emissions already vertically distributed. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np diff --git a/hermesv3_gr/modules/vertical/vertical_gfas.py b/hermesv3_gr/modules/vertical/vertical_gfas.py index 70f0414..600ab7f 100644 --- a/hermesv3_gr/modules/vertical/vertical_gfas.py +++ b/hermesv3_gr/modules/vertical/vertical_gfas.py @@ -128,13 +128,13 @@ class GfasVerticalDistribution(VerticalDistribution): Allocates the fire emissions on their top level. :param values: 2D array with the fire emissions - :type values: numpy.ndarray + :type values: numpy.array :param altitude: 2D array with the altitude of the fires. - :type altitude: numpy.ndarray + :type altitude: numpy.array :return: Emissions already allocated on the top altitude of each fire. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np @@ -159,10 +159,10 @@ class GfasVerticalDistribution(VerticalDistribution): Manages all the process to do the vertical distribution. :param values: Emissions to be vertically distributed. - :type values: numpy.ndarray + :type values: numpy.array :return: Emissions already vertically distributed. - :rtype: numpy.ndarray + :rtype: numpy.array """ st_time = timeit.default_timer() diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index 7fd70d8..411c164 100644 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -20,11 +20,38 @@ import sys import timeit +import numpy as np from hermesv3_gr.config import settings class Writer(object): + """ + Class to Write the output file. + :param path: Path to the destination file. + :type path: str + + :param grid: Grid of the destination file. + :type grid: Grid + + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. + :type compress: bool + + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool + """ def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): self.path = path @@ -42,6 +69,15 @@ class Writer(object): self.global_attributes_path = global_attributes_path def write(self, inventory_list): + """ + Write the netCDF4 file with the pollutants of the given list of inventories. + + :param inventory_list: List of inventories. + :type inventory_list: list + + :return: True at end + :rtype: bool + """ st_time = timeit.default_timer() settings.write_log('') settings.write_log("Writing netCDF output file {0} .".format(self.path)) @@ -77,10 +113,19 @@ class Writer(object): return None def set_variable_attributes(self, inventory_list): + """ + Change the variables_attribute parameter of the Writer class. + + :param inventory_list: list of invenotries. + :type inventory_list: list + + :return: True at end. + :rtype: bool + """ st_time = timeit.default_timer() empty_dict = {} - for ei in inventory_list: - for emi in ei.emissions: + for inventory in inventory_list: + for emi in inventory.emissions: if not emi['name'] in empty_dict: dict_aux = emi.copy() dict_aux['data'] = None @@ -93,17 +138,21 @@ class Writer(object): return True def calculate_data_by_var(self, variable, inventory_list, shape): - # TODO Documentation """ + Calculate the date of the given variable throw the inventory list. - :param variable: - :param inventory_list: - :param shape: - :return: - """ - import timeit - import numpy as np + :param variable: Variable to calculate. + :type variable: str + + :param inventory_list: Inventory list + :type inventory_list: list + + :param shape: Output desired shape. + :type shape: tuple + :return: Data of the given variable. + :rtype: numpy.array + """ st_time = timeit.default_timer() settings.write_log("\t\t\t\tGetting data for '{0}' pollutant.".format(variable), level=3) @@ -128,6 +177,8 @@ class Writer(object): aux_data = np.zeros((shape[1], shape[2] * shape[3])) aux_data[ei.location['layer'], ei.location['FID']] = emission['data'] aux_data = aux_data.reshape((shape[1], shape[2], shape[3])) + else: + aux_data = None settings.write_time('VerticalDistribution', 'calculate_data_by_var', timeit.default_timer() - vertical_time, level=2) @@ -153,10 +204,19 @@ class Writer(object): """ Implement on inner class """ - return None + return np.array([0]) @staticmethod def calculate_displacements(counts): + """ + Calculate the index position of all the ranks. + + :param counts: Number of elements for rank + :type counts: list + + :return: Displacements + :rtype: list + """ st_time = timeit.default_timer() new_list = [0] @@ -170,15 +230,27 @@ class Writer(object): @staticmethod def tuple_to_index(tuple_list, bidimensional=False): + """ + Get the index for a list of shapes. + + :param tuple_list: List os shapes. + :type tuple_list: list + + :param bidimensional: Indicates if the tuple is bidimensional. + :type bidimensional: bool + + :return: List of index + :rtype: list + """ from operator import mul st_time = timeit.default_timer() new_list = [] - for tuple in tuple_list: + for my_tuple in tuple_list: if bidimensional: - new_list.append(tuple[-1] * tuple[-2]) + new_list.append(my_tuple[-1] * my_tuple[-2]) else: - new_list.append(reduce(mul, tuple)) + new_list.append(reduce(mul, my_tuple)) settings.write_time('Writer', 'tuple_to_index', timeit.default_timer() - st_time, level=3) return new_list @@ -190,17 +262,29 @@ class Writer(object): :param output_model: Name of the output model. Only accepted 'MONARCH, CMAQ or WRF_CHEM. :type output_model: str - :param path: Path to the output file. + :param path: Path to the destination file. :type path: str - :param grid: Grid object of the destination. + :param grid: Grid of the destination file. :type grid: Grid - :param compress: Indicates if you want a compressed NetCDF. + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. :type compress: bool - :param parallel: Indicates if you want to write the NetCDF in parallel. - :type parallel: bool + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool :return: Writing object of the desired output model. :rtype: Writer @@ -227,19 +311,23 @@ class Writer(object): def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, - RegularLatLon=False, - Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, - LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, - Mercator=False, lat_ts=None): + regular_latlon=False, + roated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, + lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, + mercator=False, lat_ts=None): + # TODO Deprecate + """ + Will be deprecated + """ from netCDF4 import Dataset from cf_units import Unit, encode_time - if not (RegularLatLon or LambertConformalConic or Rotated or Mercator): - RegularLatLon = True + if not (regular_latlon or lcc or roated or mercator): + regular_latlon = True netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -264,7 +352,7 @@ class Writer(object): print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( len(center_longitudes.shape)) sys.exit(1) - elif Rotated: + elif roated: var_dim = ('rlat', 'rlon',) # Rotated Latitude @@ -280,7 +368,7 @@ class Writer(object): sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic or Mercator: + elif lcc or mercator: var_dim = ('y', 'x',) netcdf.createDimension('y', len(lcc_y)) @@ -288,6 +376,10 @@ class Writer(object): netcdf.createDimension('x', len(lcc_x)) lon_dim = ('y', 'x',) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if levels is not None: @@ -318,11 +410,10 @@ class Writer(object): time[:] = [0.] else: time = netcdf.createVariable('time', 'd', ('time',), zlib=True) - u = Unit('hours') # print u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second)) # Unit('hour since 1970-01-01 00:00:00.0000000 UTC') - time.units = str( - u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second))) + time.units = str(Unit('hours').offset_by_time( + encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second))) time.standard_name = "time" time.calendar = "gregorian" time.long_name = "time" @@ -357,7 +448,7 @@ class Writer(object): # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes - if Rotated: + if roated: # Rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=True) rlat.long_name = "latitude in rotated pole grid" @@ -371,18 +462,18 @@ class Writer(object): rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = rotated_lons - if LambertConformalConic or Mercator: - x = netcdf.createVariable('x', 'd', ('x',), zlib=True) - x.units = Unit("km").symbol - x.long_name = "x coordinate of projection" - x.standard_name = "projection_x_coordinate" - x[:] = lcc_x - - y = netcdf.createVariable('y', 'd', ('y',), zlib=True) - y.units = Unit("km").symbol - y.long_name = "y coordinate of projection" - y.standard_name = "projection_y_coordinate" - y[:] = lcc_y + if lcc or mercator: + x_var = netcdf.createVariable('x', 'd', ('x',), zlib=True) + x_var.units = Unit("km").symbol + x_var.long_name = "x coordinate of projection" + x_var.standard_name = "projection_x_coordinate" + x_var[:] = lcc_x + + y_var = netcdf.createVariable('y', 'd', ('y',), zlib=True) + y_var.units = Unit("km").symbol + y_var.long_name = "y coordinate of projection" + y_var.standard_name = "projection_y_coordinate" + y_var[:] = lcc_y cell_area_dim = var_dim # Levels @@ -410,13 +501,13 @@ class Writer(object): var.coordinates = "lat lon" if cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif roated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' - elif Mercator: + elif mercator: var.grid_mapping = 'mercator' try: var[:] = variable['data'] @@ -424,26 +515,26 @@ class Writer(object): print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: + elif roated: # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = north_pole_lat mapping.grid_north_pole_longitude = north_pole_lon - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" mapping.standard_parallel = lat_1_2 mapping.longitude_of_central_meridian = lon_0 mapping.latitude_of_projection_origin = lat_0 - elif Mercator: + elif mercator: # Mercator mapping = netcdf.createVariable('mercator', 'i') mapping.grid_mapping_name = "mercator" diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index a7efdd9..200878b 100644 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -28,7 +28,33 @@ from hermesv3_gr.config import settings class WriterCmaq(Writer): + """ + Class to Write the output file for CMAQ Chemical Transport Model CCTM. + :param path: Path to the destination file. + :type path: str + + :param grid: Grid of the destination file. + :type grid: Grid + + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. + :type compress: bool + + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool + """ def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterCmaq, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index 98831ad..484108d 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -18,18 +18,43 @@ # along with HERMESv3_GR. If not, see . -import os import sys -from hermesv3_gr.modules.writing.writer import Writer import timeit -from hermesv3_gr.config import settings import numpy as np from netCDF4 import Dataset from mpi4py import MPI +from hermesv3_gr.modules.writing.writer import Writer +from hermesv3_gr.config import settings class WriterMonarch(Writer): + """ + Class to Write the output file in CF-1.6 conventions. + + :param path: Path to the destination file. + :type path: str + + :param grid: Grid of the destination file. + :type grid: Grid + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. + :type compress: bool + + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool + """ def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterMonarch, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) @@ -38,12 +63,17 @@ class WriterMonarch(Writer): # } def unit_change(self, variable, data): - # TODO Documentation """ + Do the unit conversions of the data. + + :param variable: Variable to convert. + :type variable: dict - :param variable: - :param data: - :return: + :param data: Data to change. + :type data: numpy.array + + :return: Data with the new units. + :rtype: numpy.array """ from cf_units import Unit st_time = timeit.default_timer() @@ -69,10 +99,11 @@ class WriterMonarch(Writer): return data def create_parallel_netcdf(self): - # TODO Documentation """ + Create an empty netCDF4. - :return: + :return: True at end. + :rtype: bool """ from cf_units import Unit, encode_time @@ -154,6 +185,10 @@ class WriterMonarch(Writer): netcdf.createDimension('x', len(self.grid.x)) settings.write_log("\t\t\t'x' dimension: {0}".format(len(self.grid.x)), level=3) lon_dim = ('y', 'x', ) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if self.levels is not None: @@ -185,10 +220,7 @@ class WriterMonarch(Writer): time[:] = [0.] else: time = netcdf.createVariable('time', 'd', ('time',)) - u = Unit('hours') - # print u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second)) - # Unit('hour since 1970-01-01 00:00:00.0000000 UTC') - time.units = str(u.offset_by_time(encode_time(self.date.year, self.date.month, self.date.day, + time.units = str(Unit('hours').offset_by_time(encode_time(self.date.year, self.date.month, self.date.day, self.date.hour, self.date.minute, self.date.second))) time.standard_name = "time" time.calendar = "gregorian" @@ -252,21 +284,21 @@ class WriterMonarch(Writer): rlon[:] = self.grid.rlon settings.write_log("\t\t\t'rlon' variable created with size: {0}".format(rlon[:].shape), level=3) if LambertConformalConic: - x = netcdf.createVariable('x', 'd', ('x',), zlib=self.compress) - x.units = Unit("km").symbol - x.long_name = "x coordinate of projection" - x.standard_name = "projection_x_coordinate" + x_var = netcdf.createVariable('x', 'd', ('x',), zlib=self.compress) + x_var.units = Unit("km").symbol + x_var.long_name = "x coordinate of projection" + x_var.standard_name = "projection_x_coordinate" if settings.rank == 0: - x[:] = self.grid.x - settings.write_log("\t\t\t'x' variable created with size: {0}".format(x[:].shape), level=3) + x_var[:] = self.grid.x + settings.write_log("\t\t\t'x' variable created with size: {0}".format(x_var[:].shape), level=3) - y = netcdf.createVariable('y', 'd', ('y',), zlib=self.compress) - y.units = Unit("km").symbol - y.long_name = "y coordinate of projection" - y.standard_name = "projection_y_coordinate" + y_var = netcdf.createVariable('y', 'd', ('y',), zlib=self.compress) + y_var.units = Unit("km").symbol + y_var.long_name = "y coordinate of projection" + y_var.standard_name = "projection_y_coordinate" if settings.rank == 0: - y[:] = self.grid.y - settings.write_log("\t\t\t'y' variable created with size: {0}".format(y[:].shape), level=3) + y_var[:] = self.grid.y + settings.write_log("\t\t\t'y' variable created with size: {0}".format(y_var[:].shape), level=3) cell_area_dim = var_dim # Levels @@ -347,13 +379,17 @@ class WriterMonarch(Writer): netcdf.close() settings.write_time('WriterMonarch', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) + return True def write_parallel_netcdf(self, emission_list): - # TODO Documentation """ + Append the data to the netCDF4 file already created in parallel mode. + + :param emission_list: Data to append. + :type emission_list: list - :param emission_list: - :return: + :return: True at end. + :rtype: bool """ st_time = timeit.default_timer() @@ -390,13 +426,17 @@ class WriterMonarch(Writer): netcdf.close() settings.write_time('WriterMonarch', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) + return True def write_serial_netcdf(self, emission_list,): - # TODO Documentation """ + Write the netCDF4 file in serial mode. - :param emission_list: - :return: + :param emission_list: Data to append. + :type emission_list: list + + :return: True at end. + :rtype: bool """ from cf_units import Unit, encode_time @@ -416,22 +456,22 @@ class WriterMonarch(Writer): if settings.rank == 0: - RegularLatLon = False - Rotated = False - LambertConformalConic = False + regular_latlon = False + rotated = False + lcc = False if self.grid.grid_type == 'global': - RegularLatLon = True + regular_latlon = True elif self.grid.grid_type == 'rotated': - Rotated = True + rotated = True elif self.grid.grid_type == 'lcc': - LambertConformalConic = True + lcc = True settings.write_log("\tCreating NetCDF file.", level=2) netcdf = Dataset(self.path, mode='w', format="NETCDF4") # ===== Dimensions ===== settings.write_log("\t\tCreating NetCDF dimensions.", level=2) - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -471,10 +511,10 @@ class WriterMonarch(Writer): 'ERROR: Longitudes must be on a 1D or 2D array instead of {0} shape.'.format( len(self.grid.center_longitudes.shape))) sys.exit(1) - elif Rotated: + elif rotated: var_dim = ('rlat', 'rlon',) - # Rotated Latitude + # rotated Latitude if self.grid.rlat is None: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -484,7 +524,7 @@ class WriterMonarch(Writer): netcdf.createDimension('rlat', len(self.grid.rlat)) lat_dim = ('rlat', 'rlon',) - # Rotated Longitude + # rotated Longitude if self.grid.rlon is None: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -494,7 +534,7 @@ class WriterMonarch(Writer): netcdf.createDimension('rlon', len(self.grid.rlon)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic: + elif lcc: var_dim = ('y', 'x',) settings.write_log("\t\t\t'y' dimension: {0}".format(len(self.grid.y)), level=3) netcdf.createDimension('y', len(self.grid.y)) @@ -502,6 +542,10 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'x' dimension: {0}".format(len(self.grid.x)), level=3) netcdf.createDimension('x', len(self.grid.x)) lon_dim = ('y', 'x', ) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if self.levels is not None: @@ -529,8 +573,7 @@ class WriterMonarch(Writer): time[:] = [0.] else: time = netcdf.createVariable('time', 'd', ('time',)) - u = Unit('hours') - time.units = str(u.offset_by_time(encode_time( + time.units = str(Unit('hours').offset_by_time(encode_time( self.date.year, self.date.month, self.date.day, self.date.hour, self.date.minute, self.date.second))) time.standard_name = "time" @@ -574,8 +617,8 @@ class WriterMonarch(Writer): settings.write_log( "\t\t\t'lon_bnds' variable created with size: {0}".format(lon_bnds[:].shape), level=3) - if Rotated: - # Rotated Latitude + if rotated: + # rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=self.compress) rlat.long_name = "latitude in rotated pole grid" rlat.units = Unit("degrees").symbol @@ -583,27 +626,27 @@ class WriterMonarch(Writer): rlat[:] = self.grid.rlat settings.write_log("\t\t\t'rlat' variable created with size: {0}".format(rlat[:].shape), level=3) - # Rotated Longitude + # rotated Longitude rlon = netcdf.createVariable('rlon', 'f', ('rlon',), zlib=self.compress) rlon.long_name = "longitude in rotated pole grid" rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = self.grid.rlon settings.write_log("\t\t\t'rlon' variable created with size: {0}".format(rlon[:].shape), level=3) - if LambertConformalConic: - x = netcdf.createVariable('x', 'd', ('x',), zlib=self.compress) - x.units = Unit("km").symbol - x.long_name = "x coordinate of projection" - x.standard_name = "projection_x_coordinate" - x[:] = self.grid.x - settings.write_log("\t\t\t'x' variable created with size: {0}".format(x[:].shape), level=3) - - y = netcdf.createVariable('y', 'd', ('y',), zlib=self.compress) - y.units = Unit("km").symbol - y.long_name = "y coordinate of projection" - y.standard_name = "projection_y_coordinate" - y[:] = self.grid.y - settings.write_log("\t\t\t'y' variable created with size: {0}".format(y[:].shape), level=3) + if lcc: + x_var = netcdf.createVariable('x', 'd', ('x',), zlib=self.compress) + x_var.units = Unit("km").symbol + x_var.long_name = "x coordinate of projection" + x_var.standard_name = "projection_x_coordinate" + x_var[:] = self.grid.x + settings.write_log("\t\t\t'x' variable created with size: {0}".format(x_var[:].shape), level=3) + + y_var = netcdf.createVariable('y', 'd', ('y',), zlib=self.compress) + y_var.units = Unit("km").symbol + y_var.long_name = "y coordinate of projection" + y_var.standard_name = "projection_y_coordinate" + y_var[:] = self.grid.y + settings.write_log("\t\t\t'y' variable created with size: {0}".format(y_var[:].shape), level=3) cell_area_dim = var_dim # Levels @@ -631,6 +674,7 @@ class WriterMonarch(Writer): if full_shape is None: full_shape = settings.comm.allgather(rank_data.shape) # print 'Rank {0} full_shape: {1}\n'.format(settings.rank, full_shape) + if mpi_numpy: if settings.size != 1: if settings.rank == 0: @@ -698,11 +742,11 @@ class WriterMonarch(Writer): if self.grid.cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif rotated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' if mpi_numpy: @@ -746,19 +790,19 @@ class WriterMonarch(Writer): settings.write_log("\t\tCreating NetCDF metadata.", level=2) if settings.rank == 0: # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: - # Rotated pole + elif rotated: + # rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = 90 - self.grid.new_pole_latitude_degrees mapping.grid_north_pole_longitude = self.grid.new_pole_longitude_degrees - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index 51f776f..0d56e99 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -29,7 +29,33 @@ from mpi4py import MPI class WriterWrfChem(Writer): + """ + Class to Write the output file for the WRF-CHEM Chemical Transport Model. + :param path: Path to the destination file. + :type path: str + + :param grid: Grid of the destination file. + :type grid: Grid + + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. + :type compress: bool + + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool + """ def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterWrfChem, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) diff --git a/hermesv3_gr/tools/coordinates_tools.py b/hermesv3_gr/tools/coordinates_tools.py index a61e0f2..8dbc7cd 100644 --- a/hermesv3_gr/tools/coordinates_tools.py +++ b/hermesv3_gr/tools/coordinates_tools.py @@ -43,6 +43,7 @@ def get_grid_area(filename): def latlon2rotated(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): # TODO Documentation """ + Transform lat lon degrees into the rotated coordinates. :param lon_pole_deg: :param lat_pole_deg: @@ -102,6 +103,7 @@ def latlon2rotated(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): def rotated2latlon(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): # TODO Documentation """ + Transform rotated coordinates into lat lon degrees. :param lon_pole_deg: :param lat_pole_deg: @@ -223,14 +225,14 @@ def create_bounds(coords, number_vertices=2): Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) - :type coords: numpy.ndarray + :type coords: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np @@ -332,7 +334,7 @@ def create_regular_grid(center_lat, center_lon, west_boundary, south_boundary, i :type inc_lon: float :return: Arrays with the Center Latitudes, Center Longitudes, Boundary Latitudes, Boundary Longitudes. - :rtype: tuple (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) + :rtype: tuple (numpy.array, numpy.array, numpy.array, numpy.array) """ import numpy as np diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index 74188bb..39a69f4 100644 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -61,7 +61,7 @@ def get_grid_area(filename): :type filename: str :return: Returns the area of each cell. - :rtype: numpy.ndarray + :rtype: numpy.array """ from cdo import Cdo @@ -74,7 +74,7 @@ def get_grid_area(filename): return grid_area -def extract_vars(netcdf_path, variables_list, attributes_list=list()): +def extract_vars(netcdf_path, variables_list, attributes_list=()): """ Get the data from the list of variabbles. @@ -115,10 +115,10 @@ def extract_vars(netcdf_path, variables_list, attributes_list=list()): def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, - RegularLatLon=False, - Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, - LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, - Mercator=False, lat_ts=None): + regular_latlon=False, + rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, + lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, + mercator=False, lat_ts=None): # TODO Documentation """ @@ -133,31 +133,31 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, :param boundary_longitudes: :param cell_area: :param global_attributes: - :param RegularLatLon: - :param Rotated: + :param regular_latlon: + :param rotated: :param rotated_lats: :param rotated_lons: :param north_pole_lat: :param north_pole_lon: - :param LambertConformalConic: + :param lcc: :param lcc_x: :param lcc_y: :param lat_1_2: :param lon_0: :param lat_0: - :param Mercator: + :param mercator: :param lat_ts: :return: """ from cf_units import Unit, encode_time - if not (RegularLatLon or LambertConformalConic or Rotated or Mercator): - RegularLatLon = True + if not (regular_latlon or lcc or rotated or mercator): + regular_latlon = True netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -181,7 +181,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, else: print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) sys.exit(1) - elif Rotated: + elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude @@ -197,7 +197,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic or Mercator: + elif lcc or mercator: var_dim = ('y', 'x',) netcdf.createDimension('y', len(lcc_y)) @@ -205,6 +205,10 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('x', len(lcc_x)) lon_dim = ('y', 'x', ) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if levels is not None: @@ -270,7 +274,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes - if Rotated: + if rotated: # Rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=True) rlat.long_name = "latitude in rotated pole grid" @@ -284,7 +288,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = rotated_lons - if LambertConformalConic or Mercator: + if lcc or mercator: x = netcdf.createVariable('x', 'd', ('x',), zlib=True) x.units = Unit("km").symbol x.long_name = "x coordinate of projection" @@ -323,13 +327,13 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, var.coordinates = "lat lon" if cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif rotated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' - elif Mercator: + elif mercator: var.grid_mapping = 'mercator' try: var[:] = variable['data'] @@ -337,26 +341,26 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: + elif rotated: # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = north_pole_lat mapping.grid_north_pole_longitude = north_pole_lon - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" mapping.standard_parallel = lat_1_2 mapping.longitude_of_central_meridian = lon_0 mapping.latitude_of_projection_origin = lat_0 - elif Mercator: + elif mercator: # Mercator mapping = netcdf.createVariable('mercator', 'i') mapping.grid_mapping_name = "mercator" @@ -470,6 +474,10 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('x', len(lcc_x)) lon_dim = ('y', 'x', ) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if levels is not None: @@ -597,7 +605,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # print 'HOURSSSSSSSSSSSSSSSSSSSSS:', hours # if variable['data'] is not 0: # print var[:].shape, variable['data'].shape, variable['data'].max() - shape = None + shape = tuple() exec ("shape = (len(hours), {0}.size, {1}.size, {2}.size)".format(var_dim[0], var_dim[1], var_dim[2])) # exit() print shape @@ -648,11 +656,11 @@ def tuple_to_index(tuple_list, bidimensional=False): """ from operator import mul new_list = [] - for tuple in tuple_list: + for my_tuple in tuple_list: if bidimensional: - new_list.append(tuple[-1] * tuple[-2]) + new_list.append(my_tuple[-1] * my_tuple[-2]) else: - new_list.append(reduce(mul, tuple)) + new_list.append(reduce(mul, my_tuple)) return new_list diff --git a/preproc/ceds_preproc.py b/preproc/ceds_preproc.py index 1b13729..c458dff 100755 --- a/preproc/ceds_preproc.py +++ b/preproc/ceds_preproc.py @@ -34,7 +34,8 @@ LIST_SECTORS = ['agriculture', 'energy', 'industry', 'transport', 'residential', # LIST_YEARS = from 1950 to 2014 LIST_YEARS = [2010] INPUT_NAME = '-em-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim_gr_01-12.nc' -VOC_INPUT_NAME = '-em-speciated-VOC_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim-supplemental-data_gr_01-12.nc' +VOC_INPUT_NAME = '-em-speciated-VOC_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sector' + \ + 'Dim-supplemental-data_gr_01-12.nc' DO_AIR = True AIR_INPUT_NAME = '-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26_gr_01-12.nc' # ============================================================== diff --git a/preproc/eclipsev5a_preproc.py b/preproc/eclipsev5a_preproc.py index 1413710..5a8049d 100755 --- a/preproc/eclipsev5a_preproc.py +++ b/preproc/eclipsev5a_preproc.py @@ -37,9 +37,9 @@ LIST_POLLUTANTS = ['BC', 'CH4', 'CO', 'NH3', 'NOx', 'OC', 'OM', 'PM10', 'PM25', # ============================================================== -month_factor = 1000000. / (30. * 24. * 3600.) # To pass from kt/month to Kg/s -year_factor = 1000000. / (365. * 24. * 3600.) # To pass from kt/year to Kg/s -var_units = 'kg.m-2.s-1' +MONTH_FACTOR = 1000000. / (30. * 24. * 3600.) # To pass from kt/month to Kg/s +YEAR_FACTOR = 1000000. / (365. * 24. * 3600.) # To pass from kt/year to Kg/s +VAR_UNITS = 'kg.m-2.s-1' def get_grid_area(filename): @@ -69,14 +69,14 @@ def create_bounds(coordinates, number_vertices=2): Calculate the vertices coordinates. :param coordinates: Coordinates in degrees (latitude or longitude) - :type coordinates: numpy.ndarray + :type coordinates: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ interval = coordinates[1] - coordinates[0] @@ -164,7 +164,7 @@ def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cel nc_output.setncattr('source', 'IIASA', ) nc_output.setncattr('history', 'Re-writing of the ECLIPSEv5a input to follow the CF-1.6 conventions;\n' + '2017-11-28: Creating;\n') - nc_output.setncattr('http://www.iiasa.ac.at/web/home/research/researchPrograms/air/ECLIPSEv5a.html') + nc_output.setncattr('web', 'http://www.iiasa.ac.at/web/home/research/researchPrograms/air/ECLIPSEv5a.html') nc_output.setncattr('comment', 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)', ) @@ -243,7 +243,7 @@ def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area) profile = extract_month_profile_by_sector(sector, month, pollutant) data_aux = data[i, :, :] * profile # print factor - data_aux = (data_aux * month_factor) / cell_area + data_aux = (data_aux * MONTH_FACTOR) / cell_area # #data_aux = data_aux / cell_area # print 'original: ', data[i, 192, 404] # print 'factor: ', profile[192, 404] @@ -253,7 +253,7 @@ def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area) 'name': pollutant_name, 'long_name': pollutant_name, 'data': data_aux, - 'units': Unit(var_units), + 'units': Unit(VAR_UNITS), }] write_netcdf(output_name, data_list, c_lats, c_lons, cell_area, datetime(year=LIST_YEARS[i], month=month + 1, day=1)) @@ -321,13 +321,13 @@ def do_flaring_transformation(): for i in xrange(len(LIST_YEARS)): output_name = get_flaring_output_name(var_name, 'flaring', LIST_YEARS[i]) data_aux = data[i, :, :] - data_aux = (data_aux * year_factor) / cell_area + data_aux = (data_aux * YEAR_FACTOR) / cell_area data_aux = data_aux.reshape((1,) + data_aux.shape) data_list = [{ 'name': var_name, 'long_name': var_name, 'data': data_aux, - 'units': Unit(var_units), + 'units': Unit(VAR_UNITS), }] write_netcdf(output_name, data_list, c_lats, c_lons, cell_area, datetime(year=LIST_YEARS[i], month=1, day=1)) @@ -380,13 +380,13 @@ def do_ship_transformation(): data = nc_in.variables[var][0, :, :] data = np.nan_to_num(data) - data = (data * year_factor) / cell_area + data = (data * YEAR_FACTOR) / cell_area data = data.reshape((1,) + data.shape) data_list = [{ 'name': var_name, 'long_name': var_name, 'data': data, - 'units': Unit(var_units), + 'units': Unit(VAR_UNITS), }] write_netcdf(get_ship_output_name(var_name, 'ship', year), data_list, c_lats, c_lons, cell_area, diff --git a/preproc/edgarv432_voc_preproc.py b/preproc/edgarv432_voc_preproc.py index ef834e1..8f8e6cc 100755 --- a/preproc/edgarv432_voc_preproc.py +++ b/preproc/edgarv432_voc_preproc.py @@ -59,14 +59,14 @@ def create_bounds(coordinates, number_vertices=2): Calculate the vertices coordinates. :param coordinates: Coordinates in degrees (latitude or longitude) - :type coordinates: numpy.ndarray + :type coordinates: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ interval = coordinates[1] - coordinates[0] diff --git a/preproc/gfas12_preproc.py b/preproc/gfas12_preproc.py index 3630052..6dd4a62 100755 --- a/preproc/gfas12_preproc.py +++ b/preproc/gfas12_preproc.py @@ -42,14 +42,14 @@ def create_bounds(coords, number_vertices=2): Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) - :type coords: numpy.ndarray + :type coords: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np diff --git a/preproc/tno_mac_iii_preproc.py b/preproc/tno_mac_iii_preproc.py index 75db7df..73c6282 100755 --- a/preproc/tno_mac_iii_preproc.py +++ b/preproc/tno_mac_iii_preproc.py @@ -29,13 +29,12 @@ INPUT_NAME = 'TNO_MACC_III_emissions_v1_1_.txt' LIST_YEARS = [2011] VOC_RATIO_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' VOC_RATIO_NAME = 'ratio_.nc' - # ============================================================== def get_pollutants(in_path): """ - Finds the pollutants on the ASCII emissions table. + Find the pollutants on the ASCII emissions table. :param in_path: Path to the ASCII file that contains the information of the TNO_MAC-III emissions. :type in_path: str @@ -50,33 +49,55 @@ def get_pollutants(in_path): def calculate_grid_definition(in_path): - # TODO Documentation + """ + Calculate the latitude and longitude coordinates of the cell. + + :param in_path: Path to the file that contains all the information. + :type in_path: str + + :return: Latitudes array, Longitudes array, Latitude interval, Longitude interval. + :rtype: numpy.array, numpy.array, float, float + """ import pandas as pd import numpy as np - df = pd.read_table(in_path, sep=';') - df = df[df.SourceType != 'P'] + dataframe = pd.read_table(in_path, sep=';') + dataframe = dataframe[dataframe.SourceType != 'P'] # Longitudes - lons = np.sort(np.unique(df.Lon)) + lons = np.sort(np.unique(dataframe.Lon)) lons_interval = lons[1:] - lons[:-1] print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - df.Lon.min(), df.Lon.max(), lons_interval.min(), len(lons)) + dataframe.Lon.min(), dataframe.Lon.max(), lons_interval.min(), len(lons)) # Latitudes - lats = np.sort(np.unique(df.Lat)) + lats = np.sort(np.unique(dataframe.Lat)) lats_interval = lats[1:] - lats[:-1] print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - df.Lat.min(), df.Lat.max(), lats_interval.min(), len(lats)) + dataframe.Lat.min(), dataframe.Lat.max(), lats_interval.min(), len(lats)) - lats = np.arange(-90 + lats_interval.min()/2, 90, lats_interval.min(), dtype=np.float64) - lons = np.arange(-180 + lons_interval.min()/2, 180, lons_interval.min(), dtype=np.float64) + lats = np.arange(-90 + lats_interval.min() / 2, 90, lats_interval.min(), dtype=np.float64) + lons = np.arange(-180 + lons_interval.min() / 2, 180, lons_interval.min(), dtype=np.float64) return lats, lons, lats_interval.min(), lons_interval.min() def create_pollutant_empty_list(in_path, len_c_lats, len_c_lons): - # TODO Documentation + """ + Crate an empty pollutant list. + + :param in_path: Path to the file that conains the information. + :type in_path: str + + :param len_c_lats: Number of elements on the latitude array + :type len_c_lats: int + + :param len_c_lons: Number of elements on the longitude array + :type len_c_lons: int + + :return: Pollutant list + :rtype: list + """ import numpy as np pollutant_list = [] @@ -98,7 +119,15 @@ def create_pollutant_empty_list(in_path, len_c_lats, len_c_lons): def do_transformation(year): - # TODO Docuemtnation + """ + Make al the process to transform the emissions of the current year. + + :param year: year to process. + :type year: int + + :return: True when everything finish well. + :rtype: Bool + """ from hermesv3_gr.tools.netcdf_tools import write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds from datetime import datetime @@ -107,7 +136,7 @@ def do_transformation(year): in_file = os.path.join(INPUT_PATH, INPUT_NAME.replace('', str(year))) - unit_factor = 1000./(365.*24.*3600.) # To pass from Mg/year to Kg/s + unit_factor = 1000. / (365. * 24. * 3600.) # To pass from Mg/year to Kg/s # unit_factor = 1000000 # To pass from Mg/m2.year to Mg/Km2.year c_lats, c_lons, lat_interval, lon_interval = calculate_grid_definition(in_file) @@ -115,20 +144,20 @@ def do_transformation(year): b_lats = create_bounds(c_lats, number_vertices=2) b_lons = create_bounds(c_lons, number_vertices=2) - df = pd.read_table(in_file, sep=';') + dataframe = pd.read_table(in_file, sep=';') - df_np = df[df.SourceType != 'P'] - df_p = df[df.SourceType == 'P'] + df_np = dataframe[dataframe.SourceType != 'P'] + df_p = dataframe[dataframe.SourceType == 'P'] df_np.loc[:, 'row_lat'] = np.array((df_np.Lat - (-90 + lat_interval / 2)) / lat_interval, dtype=np.int32) df_np.loc[:, 'col_lon'] = np.array((df_np.Lon - (-180 + lon_interval / 2)) / lon_interval, dtype=np.int32) - df_p.loc[:, 'row_lat'] = abs(np.array([c_lats]*len(df_p.Lat)) - df_p.Lat.values[:, None]).argmin(axis=1) - df_p.loc[:, 'col_lon'] = abs(np.array([c_lons]*len(df_p.Lon)) - df_p.Lon.values[:, None]).argmin(axis=1) + df_p.loc[:, 'row_lat'] = abs(np.array([c_lats] * len(df_p.Lat)) - df_p.Lat.values[:, None]).argmin(axis=1) + df_p.loc[:, 'col_lon'] = abs(np.array([c_lons] * len(df_p.Lon)) - df_p.Lon.values[:, None]).argmin(axis=1) - df = pd.concat([df_np, df_p]) + dataframe = pd.concat([df_np, df_p]) - for name, group in df.groupby('SNAP'): + for name, group in dataframe.groupby('SNAP'): print 'snap', name pollutant_list = create_pollutant_empty_list(in_file, len(c_lats), len(c_lons)) @@ -153,7 +182,7 @@ def do_transformation(year): boundary_latitudes=b_lats, boundary_longitudes=b_lons) cell_area = get_grid_area(aux_output_path) - pollutant_list[i]['data'] = pollutant_list[i]['data']*unit_factor/cell_area + pollutant_list[i]['data'] = pollutant_list[i]['data'] * unit_factor/cell_area write_netcdf(aux_output_path, c_lats, c_lons, [pollutant_list[i]], date=datetime(year, month=1, day=1), boundary_latitudes=b_lats, boundary_longitudes=b_lons, cell_area=cell_area, @@ -164,14 +193,27 @@ def do_transformation(year): 'for air quality modelling Atmospheric Chemistry and Physics 14 ' + '10963-10976 2014', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + - '(Barcelona Supercomputing Center)' - } - ) + '(Barcelona Supercomputing Center)'}) return True -def extract_vars(netcdf_path, variables_list, attributes_list=list()): - # TODO Documentation +def extract_vars(netcdf_path, variables_list, attributes_list=()): + """ + Get the data from the list of variabbles. + + :param netcdf_path: Path to the netCDF file + :type netcdf_path: str + + :param variables_list: List of the names of the variables to get. + :type variables_list: list + + :param attributes_list: List of the names of the variable attributes to get. + :type attributes_list: list + + :return: List of the variables from the netCDF as a dictionary with data as values and with the other keys their + attributes. + :rtype: list. + """ from netCDF4 import Dataset data_list = [] # print netcdf_path @@ -196,7 +238,18 @@ def extract_vars(netcdf_path, variables_list, attributes_list=list()): def get_voc_ratio(ratio_path, snap): - # TODO Documentation + """ + Get the ratio of the VOC for the current SNAP. + + :param ratio_path: Path to the file with the ratios. + :type ratio_path: str + + :param snap: SNAP to get the ratio. + :type snap: str + + :return: VOC Ratio + :rtype: dict + """ if snap == 'snap34': snap = 'snap3' try: @@ -207,21 +260,39 @@ def get_voc_ratio(ratio_path, snap): def get_voc_list(): - # TODO Documentation + """ + Get the VOC list. + + :return: VOC list + :rtype: list + """ return ['voc{0}'.format(str(x).zfill(2)) for x in [1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]] def get_sector_list(): - # TODO Documentation + """ + Get the sector list. + + :return: Sector list + :rtype: list + """ return ['snap{0}'.format(x) for x in [1, 2, 34, 5, 6, 71, 72, 73, 74, 8, 9]] def do_voc_transformation(year): - # TODO Docuemtnation + """ + Make al the process to transform the VOC emissions of the current year. + + :param year: year to process. + :type year: int + + :return: True when everything finish well. + :rtype: Bool + """ + from warnings import warn as warning from hermesv3_gr.tools.netcdf_tools import write_netcdf, extract_vars from hermesv3_gr.tools.coordinates_tools import create_bounds - from warnings import warn as warning for snap in get_sector_list(): in_path = os.path.join(OUTPUT_PATH, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) @@ -255,8 +326,7 @@ def do_voc_transformation(year): 'for air quality modelling Atmospheric Chemistry and Physics 14 ' + '10963-10976 2014', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + - '(Barcelona Supercomputing Center)' - }) + '(Barcelona Supercomputing Center)'}) else: warning("The pollutant {0} for the sector {1} does not exist.\n SNAP not found: {2}".format( voc, snap, ratio_path)) @@ -265,7 +335,15 @@ def do_voc_transformation(year): def check_vocs(year): - # TODO Documentation + """ + Check that the VOCs are calculated correctly. + + :param year: Year to evaluate + :type year: int + + :return: True when finish. + :rtype: bool + """ for snap in get_sector_list(): nmvoc_path = os.path.join(OUTPUT_PATH, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) [new_voc] = extract_vars(nmvoc_path, ['nmvoc']) @@ -279,7 +357,7 @@ def check_vocs(year): voc_sum += new_voc['data'].sum() print '{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum)/nmvoc_sum) + snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum) return True diff --git a/preproc/tno_mac_iii_preproc_voc_ratios.py b/preproc/tno_mac_iii_preproc_voc_ratios.py index 08d4c5d..9aea926 100755 --- a/preproc/tno_mac_iii_preproc_voc_ratios.py +++ b/preproc/tno_mac_iii_preproc_voc_ratios.py @@ -30,7 +30,7 @@ CSV_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/TNO_MACC_NMVOC prof # ============================================================== -def extract_vars(netcdf_path, variables_list, attributes_list=[]): +def extract_vars(netcdf_path, variables_list, attributes_list=()): # TODO Docuemtnation """ @@ -334,7 +334,7 @@ def get_grid_area(filename): :type filename: str :return: Returns the area of each cell. - :rtype: numpy.ndarray + :rtype: numpy.array """ from cdo import Cdo from netCDF4 import Dataset @@ -353,14 +353,14 @@ def create_bounds(coords, number_vertices=2): Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) - :type coords: numpy.ndarray + :type coords: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np -- GitLab From 3a5cdddc87d80c1bd3e4494f2bdbae6e8c6275d9 Mon Sep 17 00:00:00 2001 From: Carles Tena Medina Date: Tue, 11 Sep 2018 14:25:36 +0200 Subject: [PATCH 14/28] Correcting Code conventions --- hermesv3_gr/modules/grids/grid.py | 1 + hermesv3_gr/modules/masking/masking.py | 71 +++++++++++++++---- hermesv3_gr/modules/writing/writer.py | 1 + hermesv3_gr/modules/writing/writer_cmaq.py | 1 + hermesv3_gr/modules/writing/writer_monarch.py | 1 + .../modules/writing/writer_wrf_chem.py | 1 + preproc/eclipsev5a_preproc.py | 5 +- 7 files changed, 63 insertions(+), 18 deletions(-) diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index e9227c6..0c424d1 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -39,6 +39,7 @@ class Grid(object): :param temporal_path: Path to the temporal folder. :type temporal_path: str """ + def __init__(self, grid_type, vertical_description_path, temporal_path): st_time = timeit.default_timer() # settings.write_log('Creating Grid...', level=1) diff --git a/hermesv3_gr/modules/masking/masking.py b/hermesv3_gr/modules/masking/masking.py index a6a98b6..36b1c93 100644 --- a/hermesv3_gr/modules/masking/masking.py +++ b/hermesv3_gr/modules/masking/masking.py @@ -20,11 +20,30 @@ import os import timeit -import hermesv3_gr.config.settings as settings from warnings import warn as warning +import hermesv3_gr.config.settings as settings class Masking(object): + """ + Masking object to apply simple mask or factor mask. + + :param world_info: Path to the file that contains the ISO Codes and other relevant information. + :type world_info: str + + :param factors_mask_values: List of the factor mask values. + :type factors_mask_values: list + + :param regrid_mask_values: List of the mask values. + :type regrid_mask_values: list + + :param grid: Grid. + :type grid: Grid + + :param world_mask_file: + :type world_mask_file: str + """ + def __init__(self, world_info, factors_mask_values, regrid_mask_values, grid, world_mask_file=None): from timezonefinder import TimezoneFinder @@ -39,37 +58,53 @@ class Masking(object): self.regrid_mask_values = self.parse_masking_values(regrid_mask_values) self.regrid_mask = None self.scale_mask = None - self.tf = TimezoneFinder() + self.timezonefinder = TimezoneFinder() self.grid = grid settings.write_time('Masking', 'Init', timeit.default_timer() - st_time, level=3) def get_country_codes(self): + """ + Get the country code information. + + :return: Dictionary of country codes. + :rtype: dict + """ import pandas as pd st_time = timeit.default_timer() - # settings.write_log('\t\t\tGetting country codes.', level=3) - # df = pd.read_csv(self.world_info, sep=';', index_col=False, names=["country", "country_code"]) - df = pd.read_csv(self.world_info, sep=';') - del df['time_zone'], df['time_zone_code'] - df = df.drop_duplicates().dropna() - df = df.set_index('country_code_alpha') - countries_dict = df.to_dict() + dataframe = pd.read_csv(self.world_info, sep=';') + del dataframe['time_zone'], dataframe['time_zone_code'] + dataframe = dataframe.drop_duplicates().dropna() + dataframe = dataframe.set_index('country_code_alpha') + countries_dict = dataframe.to_dict() countries_dict = countries_dict['country_code'] settings.write_time('Masking', 'get_country_codes', timeit.default_timer() - st_time, level=3) return countries_dict @staticmethod - def partlst(lst, n): + def partlst(lst, num): + """ + Split a Array in N balanced arrays. + + :param lst: Array to split + :type lst: numpy.array + + :param num: Number of mini arrays. + :type num: int + + :return: Array + :type: numpy.array + """ import itertools - """Partition @lst in @n balanced parts, in given order""" - parts, rest = divmod(len(lst), n) + # Partition @lst in @n balanced parts, in given order + parts, rest = divmod(len(lst), num) lstiter = iter(lst) - for j in xrange(n): - plen = len(lst) / n + (1 if rest > 0 else 0) + for j in xrange(num): + plen = len(lst) / num + (1 if rest > 0 else 0) rest -= 1 yield list(itertools.islice(lstiter, plen)) @@ -128,7 +163,7 @@ class Masking(object): elif longitude > +180: longitude -= 360 - tz = self.tf.timezone_at(lng=longitude, lat=latitude) + tz = self.timezonefinder.timezone_at(lng=longitude, lat=latitude) settings.write_time('Masking', 'find_timezone', timeit.default_timer() - st_time, level=3) @@ -151,6 +186,12 @@ class Masking(object): return code[0] def parse_factor_values(self, values): + """ + + :param values: + :return: + :rtype: dict + """ import re st_time = timeit.default_timer() diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index 411c164..b72284c 100644 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -52,6 +52,7 @@ class Writer(object): :param parallel: Indicates if you want to write in parallel mode. :type parallel. bool """ + def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): self.path = path diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 200878b..4d94843 100644 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -55,6 +55,7 @@ class WriterCmaq(Writer): :param parallel: Indicates if you want to write in parallel mode. :type parallel. bool """ + def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterCmaq, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index 484108d..86b1fc9 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -55,6 +55,7 @@ class WriterMonarch(Writer): :param parallel: Indicates if you want to write in parallel mode. :type parallel. bool """ + def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterMonarch, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index 0d56e99..51027f8 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -56,6 +56,7 @@ class WriterWrfChem(Writer): :param parallel: Indicates if you want to write in parallel mode. :type parallel. bool """ + def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterWrfChem, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) diff --git a/preproc/eclipsev5a_preproc.py b/preproc/eclipsev5a_preproc.py index 5a8049d..df523d2 100755 --- a/preproc/eclipsev5a_preproc.py +++ b/preproc/eclipsev5a_preproc.py @@ -53,11 +53,10 @@ def get_grid_area(filename): :rtype: numpy.array """ from cdo import Cdo - from netCDF4 import Dataset cdo = Cdo() - s = cdo.gridarea(input=filename) - nc_aux = Dataset(s, mode='r') + src = cdo.gridarea(input=filename) + nc_aux = Dataset(src, mode='r') grid_area = nc_aux.variables['cell_area'][:] nc_aux.close() -- GitLab From 9e908b6814cc2786105ea5f260ff3b1ea55ef31d Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 13 Sep 2018 17:19:22 +0200 Subject: [PATCH 15/28] deleting tests --- tests/unit/test_temporal.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_temporal.py b/tests/unit/test_temporal.py index 729c07a..6174229 100644 --- a/tests/unit/test_temporal.py +++ b/tests/unit/test_temporal.py @@ -289,11 +289,11 @@ class TestTemporalDistribution(unittest.TestCase): # 22: 1., # 23: 1.} # - self.assertEqual( - temporal.calculate_2d_temporal_factors( - datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), timezones).tolist(), - [[20., 1.], [1., 1.]]) - + # self.assertEqual( + # temporal.calculate_2d_temporal_factors( + # datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), timezones).tolist(), + # [[20., 1.], [1., 1.]]) + # # def testing_do_temporal(self): # import numpy as np # from hermesv3_gr.modules.grids.grid import Grid -- GitLab From d3845e3296b88590ce4eb14b933bcdd13ff4a864 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 14 Sep 2018 11:04:31 +0200 Subject: [PATCH 16/28] trying to solve MPI problem --- conf/EI_configuration.csv | 10 +++++----- conf/hermes.conf | 11 +++-------- hermesv3_gr/hermes.py | 10 ++++++++++ hermesv3_gr/modules/regrid/regrid_conservative.py | 2 +- hermesv3_gr/modules/writing/writer_monarch.py | 2 +- hermesv3_gr/tools/netcdf_tools.py | 4 ++-- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/conf/EI_configuration.csv b/conf/EI_configuration.csv index d366331..15e01cb 100644 --- a/conf/EI_configuration.csv +++ b/conf/EI_configuration.csv @@ -1,17 +1,17 @@ ei;sector;ref_year;active;factor_mask;regrid_mask;pollutants;path;frequency;source_type;p_vertical;p_month;p_day;p_hour;p_speciation;comment HTAPv2;energy;2010;0;;;so2;/jrc/htapv2/monthly_mean;monthly;area;V001;;D002;H002;E998;added 05/2017 HTAPv2;industry;2010;0;;;so2;/jrc/htapv2/monthly_mean;monthly;area;V002;;D003;H004;E998;added 05/2017 -HTAPv2;residential;2010;1;;- FRA;so2;/jrc/htapv2/monthly_mean;monthly;area;;;;;E998;added 05/2017 -HTAPv2;residential;2010;1;FRA 50;+ FRA;so2;/jrc/htapv2/monthly_mean;monthly;area;;;;;E998;added 05/2017 +HTAPv2;residential;2010;0;;- FRA;so2;/jrc/htapv2/monthly_mean;monthly;area;;;;;E998;added 05/2017 +HTAPv2;residential;2010;0;FRA 50;+ FRA;so2;/jrc/htapv2/monthly_mean;monthly;area;;;;;E998;added 05/2017 HTAPv2;transport;2010;1;;;so2;/jrc/htapv2/monthly_mean;monthly;area;;;D001;weekday=H001, saturday=H002, sunday=H003;E998;added 05/2017 -HTAPv2;air_lto;2010;1;;;so2;/jrc/htapv2/yearly_mean;yearly;area;V003;M001;D001;H001;E998;added 05/2017 +HTAPv2;air_lto;2010;0;;;so2;/jrc/htapv2/yearly_mean;yearly;area;V003;M001;D001;H001;E998;added 05/2017 HTAPv2;air_cds;2010;0;;;so2;/jrc/htapv2/yearly_mean;yearly;area;V004;M001;D001;H001;E998;added 05/2017 HTAPv2;air_crs;2010;0;;;so2;/jrc/htapv2/yearly_mean;yearly;area;V005;M001;D001;H001;E998;added 05/2017 HTAPv2;ships;2010;0;;;so2;/jrc/htapv2/yearly_mean;yearly;area;;M001;D001;H001;E008;added 05/2017 wiedinmyer;;2014;0;;;so2;/ucar/wiedinmyer/yearly_mean;yearly;area;;M001;D001;H001;E998;added 05/2017 ECLIPSEv5a;flaring;2010;0;;;so2;/iiasa/eclipsev5a/yearly_mean;area;yearly;V006;M001;D001;H001;E998;added 11/2017 -GFASv12;;2015;1;;;so2,nox_no;/ecmwf/gfas/daily_mean;daily;area;method=sovief,approach=uniform;;;H001;E997;added 05/2017 +GFASv12;;2015;0;;;so2,nox_no;/ecmwf/gfas/daily_mean;daily;area;method=sovief,approach=uniform;;;H001;E997;added 05/2017 ECLIPSEv5a;transport;2010;0;;+ CHN,IND;so2;/iiasa/eclipsev5a/monthly_mean;monthly;area;;;D001;H001;E998;added 11/2017 ECLIPSEv5a;transport;2010;0;;;nox_no2;/iiasa/eclipsev5a/monthly_mean;monthly;area;;;D005;weekday=H006, saturday=H009, sunday=H010;E999;added 11/2017 -CARN;;2015;1;;;so2;/mtu/carnetal/yearly_mean;yearly;point;;M001;D001;H001;E998;added ... +CARN;;2015;0;;;so2;/mtu/carnetal/yearly_mean;yearly;point;;M001;D001;H001;E998;added ... Maestra;;2015;0;;;nox_no2;/home/Earth/ctena/Models/HERMESv3/;yearly;point;;M001;D001;H001;E999;added ... diff --git a/conf/hermes.conf b/conf/hermes.conf index 82ed49f..c893141 100644 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -1,11 +1,11 @@ [GENERAL] log_level = 3 # input_dir = /gpfs/projects/bsc32/bsc32538/HERMESv3_GR_rotated/IN -input_dir = /home/Earth/ctena/Models/HERMESv3/IN +input_dir = /home/Earth/ctena/Models/hermesv3_gr # data_path = /gpfs/scratch/bsc32/bsc32538/HERMES_data data_path = /esarchive/recon #output_dir = /gpfs/projects/bsc32/bsc32538/HERMESv3_GR_rotated/OUT -output_dir = /home/carles/HERMES_out +output_dir = /home/Earth/ctena/HERMES_out output_name = HERMESv3_.nc start_date = 2014/09/02 00:00:00 # ***** end_date = start_date [DEFAULT] ***** @@ -89,12 +89,7 @@ auxiliar_files_path = /data/auxiliar_files/_ [EMISSION_INVENTORY_CONFIGURATION] -# cross_table = /conf/EI_configuration_Scalability.csv -# cross_table = /conf/EI_configuration_gridded.csv -# cross_table = /conf/EI_configuration_test.csv -# cross_table = /conf/EI_configuration_publi.csv -# cross_table = /conf/EI_configuration_EU_aerosol_gas.csv -cross_table = /conf/EI_configuration_WRF_CHEM_Rene.csv +cross_table = /conf/EI_configuration.csv [EMISSION_INVENTORY_PROFILES] diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index f094111..386463e 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -17,6 +17,12 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . +import os +import sys +parentPath = os.path.abspath(os.path.join('..')) +if parentPath not in sys.path: + sys.path.insert(0, parentPath) + import timeit from hermesv3_gr.config import settings @@ -54,6 +60,10 @@ class Hermes(object): settings.write_log('Starting HERMESv3 initialization:') + print settings.size + print settings.rank + sys.exit(1) + if self.options.output_model in ['CMAQ', 'WRF_CHEM'] and self.options.domain_type == 'global': settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: diff --git a/hermesv3_gr/modules/regrid/regrid_conservative.py b/hermesv3_gr/modules/regrid/regrid_conservative.py index f22aece..f2de4d1 100644 --- a/hermesv3_gr/modules/regrid/regrid_conservative.py +++ b/hermesv3_gr/modules/regrid/regrid_conservative.py @@ -46,7 +46,7 @@ class ConservativeRegrid(Regrid): src_grid = self.grid.create_esmf_grid_from_file(self.pollutant_dicts[0]['path']) src_field = ESMF.Field(src_grid, name='my input field') src_field.read(filename=self.pollutant_dicts[0]['path'], variable=self.pollutant_dicts[0]['name'], - timeslice=[0]) + timeslice=0) dst_grid = self.grid.esmf_grid dst_field = ESMF.Field(dst_grid, name='my outut field') diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index 86b1fc9..a7316b9 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -55,7 +55,7 @@ class WriterMonarch(Writer): :param parallel: Indicates if you want to write in parallel mode. :type parallel. bool """ - + def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterMonarch, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index 39a69f4..f0d72de 100644 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -45,10 +45,10 @@ def open_netcdf(netcdf_path): def close_netcdf(netcdf): """ Close the netCDF. - + :param netcdf: netCDF :type netcdf: Dataset - :return: + :return: """ netcdf.close() -- GitLab From dc6b1aa685d5de9cde6f815984a952b7b4a2300d Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 14 Sep 2018 11:45:57 +0200 Subject: [PATCH 17/28] trying to solve MPI problem --- environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 2916618..79dfb4c 100644 --- a/environment.yml +++ b/environment.yml @@ -17,6 +17,8 @@ dependencies: - esmpy >= 7.1.0r - pytz - timezonefinder +# - openmpi +# - mpich - mpi4py # Testing - pytest -- GitLab From f1c657d90dcb9cc6f1fc9088c61ae0b74a011a08 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 18 Sep 2018 16:00:53 +0200 Subject: [PATCH 18/28] Corrected Parallel write. Mandatory to not use UNLIMITED dimensions --- environment.yml | 7 +- hermesv3_gr/config/settings.py | 7 +- hermesv3_gr/hermes.py | 10 -- hermesv3_gr/modules/temporal/temporal.py | 4 +- hermesv3_gr/modules/writing/writer.py | 51 ++++++- hermesv3_gr/modules/writing/writer_cmaq.py | 126 +++++++----------- hermesv3_gr/modules/writing/writer_monarch.py | 103 +++++--------- .../modules/writing/writer_wrf_chem.py | 87 ++++-------- run_test.py | 1 + 9 files changed, 171 insertions(+), 225 deletions(-) diff --git a/environment.yml b/environment.yml index 79dfb4c..6b0fe67 100644 --- a/environment.yml +++ b/environment.yml @@ -3,11 +3,16 @@ name: hermesv3_gr channels: +# - anaconda - conda-forge + - spectraldns dependencies: - python = 2 - numpy + - hdf5 = 1.8.17 + - hdf5-parallel = 1.8.17 + - libnetcdf-parallel - netcdf4 >= 1.3.1 - python-cdo >= 1.3.3 - geopandas @@ -17,8 +22,6 @@ dependencies: - esmpy >= 7.1.0r - pytz - timezonefinder -# - openmpi -# - mpich - mpi4py # Testing - pytest diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index 52c5c39..1b93cfa 100644 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -30,7 +30,7 @@ global writing_serial writing_serial = False global compressed_netcdf -compressed_netcdf = False +compressed_netcdf = True if not writing_serial: compressed_netcdf = False @@ -67,8 +67,9 @@ def define_log_file(log_path, date): # TODO Documentation log_path = os.path.join(log_path, 'logs') if not os.path.exists(log_path): - os.makedirs(log_path) - + if rank == 0: + os.makedirs(log_path) + comm.Barrier() log_path = os.path.join(log_path, 'HERMESv3_{0}_Rank{1}_Procs{2}.log'.format( date.strftime('%Y%m%d%H'), str(rank).zfill(4), str(size).zfill(4))) if os.path.exists(log_path): diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 386463e..f094111 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -17,12 +17,6 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import os -import sys -parentPath = os.path.abspath(os.path.join('..')) -if parentPath not in sys.path: - sys.path.insert(0, parentPath) - import timeit from hermesv3_gr.config import settings @@ -60,10 +54,6 @@ class Hermes(object): settings.write_log('Starting HERMESv3 initialization:') - print settings.size - print settings.rank - sys.exit(1) - if self.options.output_model in ['CMAQ', 'WRF_CHEM'] and self.options.domain_type == 'global': settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 55ed096..82e1e36 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -412,7 +412,7 @@ class TemporalDistribution(object): try: timezone = self.get_tz_from_id(id_aux) tz_list[timezones == id_aux] = timezone - except: + except IndexError: pass settings.write_time('TemporalDistribution', 'calculate_timezones', timeit.default_timer() - st_time, level=3) @@ -440,7 +440,7 @@ class TemporalDistribution(object): try: df['local'] = df.groupby('tz')['utc'].apply( lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert(x.name).dt.tz_localize(None)) - except: + except pytz.exceptions.UnknownTimeZoneError: df['local'] = df.groupby('tz')['utc'].apply( lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert( self.parse_tz(x.name)).dt.tz_localize(None)) diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index b72284c..06e6f34 100644 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -21,6 +21,8 @@ import sys import timeit import numpy as np +from mpi4py import MPI +from netCDF4 import Dataset from hermesv3_gr.config import settings @@ -84,6 +86,7 @@ class Writer(object): settings.write_log("Writing netCDF output file {0} .".format(self.path)) self.set_variable_attributes(inventory_list) + self.change_variable_attributes() if self.parallel: if settings.rank == 0: self.create_parallel_netcdf() @@ -95,6 +98,9 @@ class Writer(object): settings.write_time('Writer', 'write', timeit.default_timer() - st_time) return True + def change_variable_attributes(self): + pass + def create_parallel_netcdf(self): """ Implemented on inner class. @@ -103,9 +109,50 @@ class Writer(object): def write_parallel_netcdf(self, emission_list): """ - Implemented on inner class. + Append the data to the netCDF4 file already created in parallel mode. + + :param emission_list: Data to append. + :type emission_list: list + + :return: True at end. + :rtype: bool """ - return None + + st_time = timeit.default_timer() + + settings.write_log("\tAppending data to parallel NetCDF file.", level=2) + if settings.size > 1: + netcdf = Dataset(self.path, mode='a', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) + else: + netcdf = Dataset(self.path, mode='a', format="NETCDF4") + settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) + index = 0 + # print "Rank {0} 2".format(rank) + for var_name in self.variables_attributes.iterkeys(): + + data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) + st_time = timeit.default_timer() + index += 1 + + var = netcdf.variables[var_name] + if settings.size > 1: + var.set_collective(True) + # Correcting NAN + if data is None: + data = 0 + var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] = data + + settings.write_log("\t\t\t'{0}' variable filled".format(var_name)) + + if 'cell_area' in netcdf.variables: + c_area = netcdf.variables['cell_area'] + c_area[self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] = self.grid.cell_area + + netcdf.close() + settings.write_time('Writer', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) + return True def write_serial_netcdf(self, emission_list): """ diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 4d94843..7b3480c 100644 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -76,9 +76,9 @@ class WriterCmaq(Writer): if data is not None: units = None - for var in self.variables_attributes: - if var['name'] == variable: - units = var['units'] + for var_name in self.variables_attributes: + if var_name == variable: + units = self.variables_attributes[var_name]['units'] break if Unit(units).symbol == Unit('mol.s-1').symbol: @@ -93,30 +93,37 @@ class WriterCmaq(Writer): sys.exit(1) return data - @staticmethod - def change_variable_attributes(emission_list): + def change_variable_attributes(self): """ Modify the emission list to be consistent to use the output as input for CMAQ model. - :param emission_list: List of emissions - :type emission_list: list - :return: Emission list ready for CMAQ - :rtype: list + :rtype: dict """ from cf_units import Unit - var_list = [] - for i in xrange(len(emission_list)): - emission_list[i]['var_desc'] = "{:<80}".format(emission_list[i]['long_name']) - emission_list[i]['long_name'] = "{:<16}".format(emission_list[i]['name']) + new_variable_dict = {} + for variable in self.variables_attributes: + if Unit(variable['units']).symbol == Unit('mol.s-1').symbol: + new_variable_dict[variable['name']] = { + 'units': "{:<16}".format('mole/s'), + 'var_desc': "{:<80}".format(variable['long_name']), + 'long_name': "{:<16}".format(variable['name']), + } + elif Unit(variable['units']).symbol == Unit('g.s-1').symbol: + new_variable_dict[variable['name']] = { + 'units': "{:<16}".format('g/s'), + 'var_desc': "{:<80}".format(variable['long_name']), + 'long_name': "{:<16}".format(variable['name']), + } + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise TypeError("The unit '{0}' of specie {1} is not ".format(variable['units'], variable['name']) + + "defined correctly. Should be 'mol.s-1' or 'g.s-1'") + sys.exit(1) - if Unit(emission_list[i]['units']).symbol == Unit('mol.s-1').symbol: - emission_list[i]['units'] = "{:<16}".format('mole/s') - elif Unit(emission_list[i]['units']).symbol == Unit('g.s-1').symbol: - emission_list[i]['units'] = "{:<16}".format('g/s') - var_list.append(emission_list[i]['name']) - return emission_list, var_list + self.variables_attributes = new_variable_dict @staticmethod def create_tflag(st_date, hours_array, num_vars): @@ -395,8 +402,8 @@ class WriterCmaq(Writer): # ===== Dimensions ===== settings.write_log("\t\tCreating NetCDF dimensions.", level=2) - # netcdf.createDimension('TSTEP', len(self.hours)) - netcdf.createDimension('TSTEP', None) + netcdf.createDimension('TSTEP', len(self.hours)) + # netcdf.createDimension('TSTEP', None) settings.write_log("\t\t\t'TSTEP' dimension: {0}".format('UNLIMITED ({0})'.format(len(self.hours))), level=3) netcdf.createDimension('DATE-TIME', 2) @@ -423,19 +430,17 @@ class WriterCmaq(Writer): settings.write_log("\t\t\t'TFLAG' variable created with size: {0}".format(tflag[:].shape), level=3) index = 0 - data_list, var_list = self.change_variable_attributes(self.variables_attributes) - for variable in self.variables_attributes: + # data_list, var_list = self.change_variable_attributes(self.variables_attributes) + for var_name in self.variables_attributes.iterkeys(): index += 1 - var = netcdf.createVariable(variable['name'], 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) - var.units = variable['units'] - var.long_name = str(variable['long_name']) - var.var_desc = str(variable['var_desc']) - settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(variable['name'], var[:].shape) + - "\n\t\t\t\t'{0}' variable will be filled later.".format(variable['name']), level=3) + var = netcdf.createVariable(var_name, 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) + var.setncatts(self.variables_attributes[var_name]) + settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(var_name, var[:].shape) + + "\n\t\t\t\t'{0}' variable will be filled later.".format(var_name), level=3) # ===== Global attributes ===== settings.write_log("\t\tCreating NetCDF metadata.", level=2) - global_attributes = self.create_global_attributes(var_list) + global_attributes = self.create_global_attributes(self.variables_attributes.keys()) for attribute in self.global_attributes_order: netcdf.setncattr(attribute, global_attributes[attribute]) @@ -443,42 +448,6 @@ class WriterCmaq(Writer): settings.write_time('WriterCmaq', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) - def write_parallel_netcdf(self, emission_list): - """ - Write the netCDF in parallel mode. - - :param emission_list: List of the processed emissions for the different emission inventories - :type emission_list: list - - :return: True when it finish well. - :rtype: bool - """ - st_time = timeit.default_timer() - settings.write_log("\tAppending data to parallel NetCDF file.", level=2) - - if settings.size > 1: - netcdf = Dataset(self.path, mode='a', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) - else: - netcdf = Dataset(self.path, mode='a', format="NETCDF4") - settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) - index = 0 - for variable in self.variables_attributes: - data = self.calculate_data_by_var(variable['name'], emission_list, self.grid.shape) - - index += 1 - - var = netcdf.variables[variable['name']] - if settings.size > 1: - var.set_collective(True) - # Correcting NAN - if data is None: - data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, - self.grid.y_lower_bound:self.grid.y_upper_bound] = data - settings.write_log("\t\t\t'{0}' variable filled".format(variable['name'])) - - netcdf.close() - settings.write_time('WriterCmaq', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) return True def write_serial_netcdf(self, emission_list): @@ -534,11 +503,11 @@ class WriterCmaq(Writer): full_shape = None index = 0 - data_list, var_list = self.change_variable_attributes(self.variables_attributes) - for variable in data_list: + # data_list, var_list = self.change_variable_attributes(self.variables_attributes) + for var_name in self.variables_attributes.iterkeys(): if settings.size != 1: - settings.write_log("\t\t\tGathering {0} data.".format(variable['name']), level=3) - rank_data = self.calculate_data_by_var(variable['name'], emission_list, self.grid.shape) + settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) + rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) if mpi_numpy or mpi_vector: if rank_data is not None: root_shape = settings.comm.bcast(rank_data.shape, root=0) @@ -592,15 +561,16 @@ class WriterCmaq(Writer): if settings.size != 1: try: data = np.concatenate(data, axis=3) - except: + except (UnboundLocalError, TypeError, IndexError): data = 0 st_time = timeit.default_timer() index += 1 - var = netcdf.createVariable(variable['name'], 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) - var.units = variable['units'] - var.long_name = str(variable['long_name']) - var.var_desc = str(variable['var_desc']) + var = netcdf.createVariable(var_name, 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) + var.setncatts(self.variables_attributes[var_name]) + # var.units = variable['units'] + # var.long_name = str(variable['long_name']) + # var.var_desc = str(variable['var_desc']) # var[:] = variable['data'] if mpi_numpy: @@ -614,7 +584,7 @@ class WriterCmaq(Writer): else: var[:, :, :, full_position[i][2]:full_position[i][3]] = \ recvbuf[i, :, :, :, : full_shape[i][-1]] - except: + except ValueError: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise TypeError("ERROR on i {0} ".format(i) + @@ -640,12 +610,12 @@ class WriterCmaq(Writer): var[:] = data else: var[:] = data - settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(variable['name'], var[:].shape), + settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(var_name, var[:].shape), level=3) settings.write_log("\t\tCreating NetCDF metadata.", level=2) if settings.rank == 0: # ===== Global attributes ===== - global_attributes = self.create_global_attributes(var_list) + global_attributes = self.create_global_attributes(self.variables_attributes.keys()) for attribute in self.global_attributes_order: netcdf.setncattr(attribute, global_attributes[attribute]) diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index a7316b9..3321b06 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -81,9 +81,9 @@ class WriterMonarch(Writer): if data is not None: units = None - for var in self.variables_attributes: - if var['name'] == variable: - units = var['units'] + for var_name in self.variables_attributes: + if var_name == variable: + units = self.variables_attributes[var_name]['units'] break if Unit(units).symbol == Unit('mol.s-1.m-2').symbol: @@ -99,6 +99,20 @@ class WriterMonarch(Writer): settings.write_time('WriterMonarch', 'unit_change', timeit.default_timer() - st_time, level=3) return data + def change_variable_attributes(self): + """ + Modify the emission list to be consistent to use the output as input for CMAQ model. + + :return: Emission list ready for CMAQ + :rtype: dict + """ + new_variable_dict = {} + for variable in self.variables_attributes: + new_variable_dict[variable['name']] = variable + del new_variable_dict[variable['name']]['name'] + + self.variables_attributes = new_variable_dict + def create_parallel_netcdf(self): """ Create an empty netCDF4. @@ -319,10 +333,10 @@ class WriterMonarch(Writer): var[:] = 0 index = 0 - for variable in self.variables_attributes: + for var_name, variable in self.variables_attributes.iteritems(): index += 1 - var = netcdf.createVariable(variable['name'], 'f', ('time',) + var_dim, zlib=self.compress) + var = netcdf.createVariable(var_name, 'f', ('time',) + var_dim, zlib=self.compress) var.units = Unit(variable['units']).symbol if 'long_name' in variable: @@ -340,8 +354,8 @@ class WriterMonarch(Writer): var.grid_mapping = 'rotated_pole' elif LambertConformalConic: var.grid_mapping = 'Lambert_conformal' - settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(variable['name'], var[:].shape) + - "\n\t\t\t\t'{0}' variable will be filled later.".format(variable['name']), level=3) + settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(var_name, var[:].shape) + + "\n\t\t\t\t'{0}' variable will be filled later.".format(var_name), level=3) settings.write_log("\t\tCreating NetCDF metadata.", level=2) # Grid mapping @@ -382,53 +396,6 @@ class WriterMonarch(Writer): settings.write_time('WriterMonarch', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) return True - def write_parallel_netcdf(self, emission_list): - """ - Append the data to the netCDF4 file already created in parallel mode. - - :param emission_list: Data to append. - :type emission_list: list - - :return: True at end. - :rtype: bool - """ - - st_time = timeit.default_timer() - - settings.write_log("\tAppending data to parallel NetCDF file.", level=2) - if settings.size > 1: - netcdf = Dataset(self.path, mode='a', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) - else: - netcdf = Dataset(self.path, mode='a', format="NETCDF4") - settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) - index = 0 - # print "Rank {0} 2".format(rank) - for variable in self.variables_attributes: - - data = self.calculate_data_by_var(variable['name'], emission_list, self.grid.shape) - st_time = timeit.default_timer() - index += 1 - - var = netcdf.variables[variable['name']] - if settings.size > 1: - var.set_collective(True) - # Correcting NAN - if data is None: - data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, - self.grid.y_lower_bound:self.grid.y_upper_bound] = data - - settings.write_log("\t\t\t'{0}' variable filled".format(variable['name'])) - - if self.grid.cell_area is not None: - c_area = netcdf.variables['cell_area'] - c_area[self.grid.x_lower_bound:self.grid.x_upper_bound, - self.grid.y_lower_bound:self.grid.y_upper_bound] = self.grid.cell_area - - netcdf.close() - settings.write_time('WriterMonarch', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) - return True - def write_serial_netcdf(self, emission_list,): """ Write the netCDF4 file in serial mode. @@ -665,10 +632,10 @@ class WriterMonarch(Writer): full_shape = None index = 0 - for variable in self.variables_attributes: + for var_name in self.variables_attributes.iterkeys(): if settings.size != 1: - settings.write_log("\t\t\tGathering {0} data.".format(variable['name']), level=3) - rank_data = self.calculate_data_by_var(variable['name'], emission_list, self.grid.shape) + settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) + rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) if mpi_numpy or mpi_vector: if rank_data is not None: root_shape = settings.comm.bcast(rank_data.shape, root=0) @@ -723,21 +690,21 @@ class WriterMonarch(Writer): if settings.size != 1: try: data = np.concatenate(data, axis=3) - except: + except (UnboundLocalError, TypeError, IndexError): data = 0 index += 1 - var = netcdf.createVariable(variable['name'], 'f', ('time',) + var_dim, zlib=self.compress) + var = netcdf.createVariable(var_name, 'f', ('time',) + var_dim, zlib=self.compress) - var.units = Unit(variable['units']).symbol + var.units = Unit(self.variables_attributes[var_name]['units']).symbol - if 'long_name' in variable: - var.long_name = str(variable['long_name']) + if 'long_name' in self.variables_attributes[var_name]: + var.long_name = str(self.variables_attributes[var_name]['long_name']) - if 'standard_name' in variable: - var.standard_name = str(variable['standard_name']) + if 'standard_name' in self.variables_attributes[var_name]: + var.standard_name = str(self.variables_attributes[var_name]['standard_name']) - if 'cell_method' in variable: - var.cell_method = str(variable['cell_method']) + if 'cell_method' in self.variables_attributes[var_name]: + var.cell_method = str(self.variables_attributes[var_name]['cell_method']) var.coordinates = "lat lon" @@ -761,7 +728,7 @@ class WriterMonarch(Writer): else: var[:, :, :, full_position[i][2]:full_position[i][3]] = \ recvbuf[i, :, :, :, : full_shape[i][-1]] - except: + except ValueError: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise TypeError("ERROR on i {0} ".format(i) + @@ -786,7 +753,7 @@ class WriterMonarch(Writer): var[:] = data else: var[:] = data - settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(variable['name'], var[:].shape), + settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(var_name, var[:].shape), level=3) settings.write_log("\t\tCreating NetCDF metadata.", level=2) if settings.rank == 0: diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index 51027f8..fcbdd5c 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -20,13 +20,12 @@ import os import sys -from hermesv3_gr.modules.writing.writer import Writer import timeit -from hermesv3_gr.config import settings import numpy as np from netCDF4 import Dataset from mpi4py import MPI - +from hermesv3_gr.config import settings +from hermesv3_gr.modules.writing.writer import Writer class WriterWrfChem(Writer): """ @@ -141,6 +140,7 @@ class WriterWrfChem(Writer): raise TypeError("The unit '{0}' of specie {1} is not ".format(variable['units'], variable['name']) + "defined correctly. Should be 'mol.h-1.km-2' or 'ug.s-1.m-2'") sys.exit(1) + self.variables_attributes = new_variable_dict def read_global_attributes(self): @@ -320,30 +320,29 @@ class WriterWrfChem(Writer): settings.write_log("\tCreating parallel NetCDF file.", level=2) netcdf = Dataset(self.path, mode='w', format="NETCDF4") - if settings.rank == 0: - # ===== Dimensions ===== - settings.write_log("\t\tCreating NetCDF dimensions.", level=2) - netcdf.createDimension('Time', None) - settings.write_log("\t\t\t'Time' dimension: {0}".format('UNLIMITED ({0})'.format(len(self.hours))), - level=3) - netcdf.createDimension('DateStrLen', 19) - settings.write_log("\t\t\t'DateStrLen' dimension: 19", level=3) - netcdf.createDimension('west_east', self.grid.center_longitudes.shape[1]) - settings.write_log("\t\t\t'west_east' dimension: {0}".format(len(self.hours)), level=3) - netcdf.createDimension('south_north', self.grid.center_latitudes.shape[0]) - settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), - level=3) - netcdf.createDimension('emissions_zdim', len(self.levels)) - settings.write_log("\t\t\t'emissions_zdim' dimension: {0}".format(len(self.levels)), level=3) - - # ===== Variables ===== - settings.write_log("\t\tCreating NetCDF variables.", level=2) - times = netcdf.createVariable('Times', 'S1', ('Time', 'DateStrLen', )) - times[:] = self.create_times_var() - settings.write_log("\t\t\t'Times' variable created with size: {0}".format(times[:].shape), level=3) + # ===== Dimensions ===== + settings.write_log("\t\tCreating NetCDF dimensions.", level=2) + netcdf.createDimension('Time', len(self.hours)) + # netcdf.createDimension('Time', None) + settings.write_log("\t\t\t'Time' dimension: {0}".format('UNLIMITED ({0})'.format(len(self.hours))), + level=3) + netcdf.createDimension('DateStrLen', 19) + settings.write_log("\t\t\t'DateStrLen' dimension: 19", level=3) + netcdf.createDimension('west_east', self.grid.center_longitudes.shape[1]) + settings.write_log("\t\t\t'west_east' dimension: {0}".format(len(self.hours)), level=3) + netcdf.createDimension('south_north', self.grid.center_latitudes.shape[0]) + settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) + netcdf.createDimension('emissions_zdim', len(self.levels)) + settings.write_log("\t\t\t'emissions_zdim' dimension: {0}".format(len(self.levels)), level=3) + + # ===== Variables ===== + settings.write_log("\t\tCreating NetCDF variables.", level=2) + times = netcdf.createVariable('Times', 'S1', ('Time', 'DateStrLen', )) + times[:] = self.create_times_var() + settings.write_log("\t\t\t'Times' variable created with size: {0}".format(times[:].shape), level=3) index = 0 - self.change_variable_attributes() for var_name in self.variables_attributes.iterkeys(): index += 1 var = netcdf.createVariable(var_name, 'f', ('Time', 'emissions_zdim', 'south_north', 'west_east',), @@ -362,39 +361,7 @@ class WriterWrfChem(Writer): settings.write_time('WriterCmaq', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) - def write_parallel_netcdf(self, emission_list): - # TODO Documentation - """ - - :param emission_list: - :return: - """ - st_time = timeit.default_timer() - settings.write_log("\tAppending data to parallel NetCDF file.", level=2) - - if settings.size > 1: - netcdf = Dataset(self.path, mode='a', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) - else: - netcdf = Dataset(self.path, mode='a', format="NETCDF4") - settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) - index = 0 - for var_name in self.variables_attributes.iterkeys(): - data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) - - index += 1 - - var = netcdf.variables[var_name] - if settings.size > 1: - var.set_collective(True) - # Correcting NAN - if data is None: - data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, - self.grid.y_lower_bound:self.grid.y_upper_bound] = data - settings.write_log("\t\t\t'{0}' variable filled".format(var_name)) - - netcdf.close() - settings.write_time('WriterCmaq', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) + return True def write_serial_netcdf(self, emission_list): # TODO Documentation @@ -441,7 +408,7 @@ class WriterWrfChem(Writer): full_shape = None index = 0 - self.change_variable_attributes() + # self.change_variable_attributes() for var_name in self.variables_attributes.iterkeys(): if settings.size != 1: @@ -476,7 +443,7 @@ class WriterWrfChem(Writer): if settings.size != 1: try: data = np.concatenate(data, axis=3) - except: + except (UnboundLocalError, TypeError, IndexError): data = 0 st_time = timeit.default_timer() index += 1 diff --git a/run_test.py b/run_test.py index 663775c..4a8b7d8 100644 --- a/run_test.py +++ b/run_test.py @@ -4,6 +4,7 @@ import os import sys import pytest + work_path = os.path.abspath(os.path.join(os.path.dirname(__file__))) os.chdir(work_path) print(work_path) -- GitLab From aaab75cb8fc4d6f0dc1ea44676247a6b66eae692 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 18 Sep 2018 16:04:27 +0200 Subject: [PATCH 19/28] old conda enviroment, not working on parallel netCDF --- environment.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/environment.yml b/environment.yml index 6b0fe67..2916618 100644 --- a/environment.yml +++ b/environment.yml @@ -3,16 +3,11 @@ name: hermesv3_gr channels: -# - anaconda - conda-forge - - spectraldns dependencies: - python = 2 - numpy - - hdf5 = 1.8.17 - - hdf5-parallel = 1.8.17 - - libnetcdf-parallel - netcdf4 >= 1.3.1 - python-cdo >= 1.3.3 - geopandas -- GitLab From 54445bb5873dbe676bc011ad6ffced8089e70683 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 18 Sep 2018 16:12:03 +0200 Subject: [PATCH 20/28] corrected PEP-8 conventions --- hermesv3_gr/modules/writing/writer_wrf_chem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index fcbdd5c..5cf01cf 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -18,7 +18,6 @@ # along with HERMESv3_GR. If not, see . -import os import sys import timeit import numpy as np @@ -27,6 +26,7 @@ from mpi4py import MPI from hermesv3_gr.config import settings from hermesv3_gr.modules.writing.writer import Writer + class WriterWrfChem(Writer): """ Class to Write the output file for the WRF-CHEM Chemical Transport Model. -- GitLab From 73712c45ea1e708f3e6a2f1f6d6893964806de4d Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 18 Sep 2018 16:18:07 +0200 Subject: [PATCH 21/28] corrected PEP-8 conventions --- run_test.py | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/run_test.py b/run_test.py index 4a8b7d8..6eb2926 100644 --- a/run_test.py +++ b/run_test.py @@ -5,6 +5,7 @@ import os import sys import pytest + work_path = os.path.abspath(os.path.join(os.path.dirname(__file__))) os.chdir(work_path) print(work_path) diff --git a/setup.py b/setup.py index 69e262c..7bae174 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,8 @@ setup( 'holidays', 'pytz', 'timezonefinder', - 'mpi4py' + 'mpi4py', + 'pytest', ], packages=find_packages(), classifiers=[ -- GitLab From ccd9d9bbb7adcbd2bab876da5462a5e0496e2cd7 Mon Sep 17 00:00:00 2001 From: Carles Tena Medina Date: Wed, 19 Sep 2018 15:32:49 +0200 Subject: [PATCH 22/28] Corrected minor --- CHANGELOG | 4 +++- README.md | 2 +- data/profiles/vertical/1layer_vertical_description.csv | 2 -- hermesv3_gr/tools/sample_files.py | 10 +++++----- setup.py | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 data/profiles/vertical/1layer_vertical_description.csv diff --git a/CHANGELOG b/CHANGELOG index 2409f3c..02048fd 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,2 +1,4 @@ 0.0.0 - HERMESv3_GR first release \ No newline at end of file + 2018/09/18 + + HERMESv3_GR beta version first release \ No newline at end of file diff --git a/README.md b/README.md index ca8c806..b393f1f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # HERMESv3 Global/Regional -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/34fc5d6c803444178034b99dd28c7e3c)](https://www.codacy.com/app/carlestena/hermesv3_gr?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/hermesv3_gr&utm_campaign=Badge_Grade) \ No newline at end of file +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/34fc5d6c803444178034b99dd28c7e3c)](https://www.codacy.com/app/carlestena/hermesv3_gr?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/hermesv3_gr&utm_campaign=Badge_Grade) diff --git a/data/profiles/vertical/1layer_vertical_description.csv b/data/profiles/vertical/1layer_vertical_description.csv deleted file mode 100644 index defd16d..0000000 --- a/data/profiles/vertical/1layer_vertical_description.csv +++ /dev/null @@ -1,2 +0,0 @@ -Ilayer;height_magl -1;1000 \ No newline at end of file diff --git a/hermesv3_gr/tools/sample_files.py b/hermesv3_gr/tools/sample_files.py index 987b806..07c642e 100644 --- a/hermesv3_gr/tools/sample_files.py +++ b/hermesv3_gr/tools/sample_files.py @@ -43,6 +43,7 @@ def make_profiles_file_list(): os.path.join(main_dir, 'data', 'global_attributes.csv'), {'profiles': [{ 'speciation': [ + os.path.join(main_dir, 'data', 'profiles', 'speciation', 'MolecularWeights.csv'), os.path.join(main_dir, 'data', 'profiles', 'speciation', 'Speciation_profile_cb05_aero5_CMAQ.csv'), os.path.join(main_dir, 'data', 'profiles', 'speciation', 'Speciation_profile_cb05_aero5_MONARCH_aerosols.csv'), @@ -58,7 +59,6 @@ def make_profiles_file_list(): os.path.join(main_dir, 'data', 'profiles', 'temporal', 'tz_world_country_iso3166.csv'), ]}, {'vertical': [ - os.path.join(main_dir, 'data', 'profiles', 'vertical', '1layer_vertical_description.csv'), os.path.join(main_dir, 'data', 'profiles', 'vertical', 'CMAQ_15layers_vertical_description.csv'), os.path.join(main_dir, 'data', 'profiles', 'vertical', 'MONARCH_Global_48layers_vertical_description.csv'), @@ -118,11 +118,11 @@ def query_yes_no(question, default="yes"): def check_args(args, exe_str): if len(args) == 0: print("Missing destination path after '{0}'. e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES/sample_files".format(exe_str)) + "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str)) sys.exit(1) elif len(args) > 1: print("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES/sample_files".format(exe_str)) + "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str)) sys.exit(1) else: dir_path = args[0] @@ -153,7 +153,7 @@ def copy_files(file_list, directory): def copy_config_files(): argv = sys.argv[1:] - parent_dir = check_args(argv, 'hermesv3_copy_config_files') + parent_dir = check_args(argv, 'hermesv3_gr_copy_config_files') copy_files(make_conf_file_list(), parent_dir) copy_files(make_profiles_file_list(), parent_dir) @@ -162,7 +162,7 @@ def copy_config_files(): def copy_preproc_files(): argv = sys.argv[1:] - parent_dir = check_args(argv, 'hermesv3_copy_preproc_files') + parent_dir = check_args(argv, 'hermesv3_gr_copy_preproc_files') copy_files(make_preproc_file_list(), parent_dir) diff --git a/setup.py b/setup.py index 7bae174..88e6bae 100644 --- a/setup.py +++ b/setup.py @@ -81,8 +81,8 @@ setup( entry_points={ 'console_scripts': [ 'hermesv3_gr = hermesv3_gr.hermes:run', - 'hermesv3_copy_config_files = hermesv3_gr.tools.sample_files:copy_config_files', - 'hermesv3_copy_preproc_files = hermesv3_gr.tools.sample_files:copy_preproc_files', + 'hermesv3_gr_copy_config_files = hermesv3_gr.tools.sample_files:copy_config_files', + 'hermesv3_gr_copy_preproc_files = hermesv3_gr.tools.sample_files:copy_preproc_files', ], }, ) -- GitLab From 8b54346a9ffb0df969654ad6eebc3148e7aba37b Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 20 Sep 2018 09:28:23 +0200 Subject: [PATCH 23/28] Updated pipeline --- .gitlab-ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87db4e1..67e39c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,8 @@ test_python2: - conda env update -f environment.yml -n hermesv3_gr python=2.7 - source activate hermesv3_gr - python run_test.py + - pip install codacy-coverage --upgrade + - python-codacy-coverage -r test/report/python2/coverage.xml #test_python3: # stage: test @@ -33,13 +35,6 @@ test_python2: # - source activate earthdiagnostics3 # - python run_test.py -report_codacy: - stage: report - script: - - source activate hermesv3_gr - - pip install codacy-coverage --upgrade - - python-codacy-coverage -r test/report/python2/coverage.xml - clean: stage: clean script: -- GitLab From d555728e4512cc67b02eaf4cf5d8469e8eaf5f75 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 20 Sep 2018 09:33:15 +0200 Subject: [PATCH 24/28] fixed typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67e39c1..08c1355 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ test_python2: - source activate hermesv3_gr - python run_test.py - pip install codacy-coverage --upgrade - - python-codacy-coverage -r test/report/python2/coverage.xml + - python-codacy-coverage -r tests/report/python2/coverage.xml #test_python3: # stage: test -- GitLab From e80a8fd37e2bc5ec80543884c84cfe379fc60816 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 20 Sep 2018 11:05:47 +0200 Subject: [PATCH 25/28] updated setup.py --- setup.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 88e6bae..d96c650 100644 --- a/setup.py +++ b/setup.py @@ -18,18 +18,16 @@ # along with HERMESv3_GR. If not, see . -from os import path from setuptools import find_packages from setuptools import setup -here = path.abspath(path.dirname(__file__)) # Get the version number from the relevant file -with open(path.join(here, 'VERSION')) as f: +with open('VERSION') as f: version = f.read().strip() -with open("README.md", "r") as fh: - long_description = fh.read() +with open("README.md", "r") as f: + long_description = f.read() setup( name='hermesv3_gr', -- GitLab From 624d32635431edbc38fc9753ee48662ecd881323 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 20 Sep 2018 11:18:00 +0200 Subject: [PATCH 26/28] updated setup.py --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index d96c650..aa5b670 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,9 @@ setup( 'CHANGELOG', 'VERSION', 'LICENSE', + 'data', + 'conf', + 'preproc' ] }, -- GitLab From d53e4ad944fd8ce14ad47fb033a8102ddb7f6c4e Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 20 Sep 2018 11:26:13 +0200 Subject: [PATCH 27/28] updated setup.py --- hermesv3_gr/__init__.py | 1 + setup.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hermesv3_gr/__init__.py b/hermesv3_gr/__init__.py index e69de29..6c8e6b9 100644 --- a/hermesv3_gr/__init__.py +++ b/hermesv3_gr/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.0" diff --git a/setup.py b/setup.py index aa5b670..41fab0d 100644 --- a/setup.py +++ b/setup.py @@ -20,11 +20,12 @@ from setuptools import find_packages from setuptools import setup +from hermesv3_gr import __version__ # Get the version number from the relevant file with open('VERSION') as f: - version = f.read().strip() + version = __version__ with open("README.md", "r") as f: long_description = f.read() @@ -73,9 +74,9 @@ setup( 'CHANGELOG', 'VERSION', 'LICENSE', - 'data', - 'conf', - 'preproc' + 'data/', + 'conf/', + 'preproc/' ] }, -- GitLab From aa8bf20306fd35095c08791d243cc5472b3cd3c4 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 20 Sep 2018 11:49:27 +0200 Subject: [PATCH 28/28] updated setup.py --- VERSION | 1 - setup.py | 45 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 10 deletions(-) delete mode 100644 VERSION diff --git a/VERSION b/VERSION deleted file mode 100644 index bd52db8..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 41fab0d..f7c0126 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,7 @@ from hermesv3_gr import __version__ # Get the version number from the relevant file -with open('VERSION') as f: - version = __version__ +version = __version__ with open("README.md", "r") as f: long_description = f.read() @@ -68,17 +67,45 @@ setup( "Operating System :: OS Independent", "Topic :: Scientific/Engineering :: Atmospheric Science" ], - include_package_data=True, - package_data={'hermesv3_gr': [ - 'README', + package_data={'': [ + 'README.md', 'CHANGELOG', - 'VERSION', 'LICENSE', - 'data/', - 'conf/', - 'preproc/' + 'data/*', + 'conf/*', + 'preproc/*' ] }, + data_files=[('.', ['LICENSE', 'CHANGELOG', ]), + ('conf', ['conf/hermes.conf', + 'conf/EI_configuration.csv', ]), + ('data', ['data/global_attributes.csv', + 'data/profiles/speciation/MolecularWeights.csv', + 'data/profiles/speciation/Speciation_profile_cb05_aero5_CMAQ.csv', + 'data/profiles/speciation/Speciation_profile_cb05_aero5_MONARCH_aerosols.csv', + 'data/profiles/speciation/Speciation_profile_cb05_aero5_MONARCH_fullchem.csv', + 'data/profiles/speciation/Speciation_profile_radm2_madesorgam_WRF_CHEM.csv', + 'data/profiles/temporal/TemporalProfile_Daily.csv', + 'data/profiles/temporal/TemporalProfile_Hourly.csv', + 'data/profiles/temporal/TemporalProfile_Monthly.csv', + 'data/profiles/temporal/tz_world_country_iso3166.csv', + 'data/profiles/vertical/CMAQ_15layers_vertical_description.csv', + 'data/profiles/vertical/MONARCH_Global_48layers_vertical_description.csv', + 'data/profiles/vertical/MONARCH_regional_48layers_vertical_description.csv', + 'data/profiles/vertical/Vertical_profile.csv', ]), + ('preproc', ['preproc/ceds_preproc.py', + 'preproc/eclipsev5a_preproc.py', + 'preproc/edgarv432_ap_preproc.py', + 'preproc/edgarv432_voc_preproc.py', + 'preproc/emep_preproc.py', + 'preproc/gfas12_preproc.py', + 'preproc/htapv2_preproc.py', + 'preproc/tno_mac_iii_preproc.py', + 'preproc/tno_mac_iii_preproc_voc_ratios.py', + 'preproc/wiedinmyer_preproc.py', ]), + ], + + include_package_data=True, entry_points={ 'console_scripts': [ -- GitLab