From 61cfd0a8583702cfee278a06e2ae1afe58474c69 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 21 Oct 2019 11:17:19 +0200 Subject: [PATCH 01/28] 2to3-3.7 run for all project --- hermesv3_gr/config/config.py | 10 ++-- hermesv3_gr/config/settings.py | 7 +-- .../emision_inventories/emission_inventory.py | 6 +-- .../gfas_emission_inventory.py | 6 +-- .../point_gfas_emission_inventory.py | 14 ++--- .../point_source_emission_inventory.py | 2 +- hermesv3_gr/modules/grids/grid.py | 8 +-- hermesv3_gr/modules/grids/grid_global.py | 2 +- hermesv3_gr/modules/grids/grid_latlon.py | 2 +- hermesv3_gr/modules/grids/grid_lcc.py | 2 +- hermesv3_gr/modules/grids/grid_mercator.py | 2 +- hermesv3_gr/modules/grids/grid_rotated.py | 2 +- hermesv3_gr/modules/masking/masking.py | 6 +-- .../modules/regrid/regrid_conservative.py | 10 ++-- hermesv3_gr/modules/speciation/speciation.py | 2 +- hermesv3_gr/modules/temporal/temporal.py | 26 ++++----- hermesv3_gr/modules/vertical/vertical.py | 4 +- hermesv3_gr/modules/vertical/vertical_gfas.py | 12 ++--- hermesv3_gr/modules/writing/writer.py | 17 +++--- hermesv3_gr/modules/writing/writer_cmaq.py | 16 +++--- hermesv3_gr/modules/writing/writer_monarch.py | 20 +++---- .../modules/writing/writer_wrf_chem.py | 12 ++--- hermesv3_gr/tools/coordinates_tools.py | 12 ++--- hermesv3_gr/tools/download_benchmark.py | 10 ++-- hermesv3_gr/tools/netcdf_tools.py | 21 ++++---- hermesv3_gr/tools/sample_files.py | 12 ++--- preproc/cams_glob_ant_preproc.py | 4 +- preproc/cams_glob_ocean_preproc.py | 2 +- preproc/cams_glob_ship_preproc.py | 2 +- preproc/cams_glob_soil_preproc.py | 2 +- preproc/cams_reg_ap_preproc.py | 20 +++---- preproc/cams_reg_ap_preproc_pm_ratios.py | 32 +++++------ preproc/cams_reg_ap_preproc_voc_ratios.py | 36 ++++++------- preproc/cams_reg_ghg_preproc.py | 20 +++---- preproc/ceds_preproc.py | 6 +-- preproc/eclipsev5a_preproc.py | 10 ++-- preproc/edgarv432_ap_preproc.py | 18 +++---- preproc/edgarv432_voc_preproc.py | 28 +++++----- preproc/emep_preproc.py | 10 ++-- preproc/gfas12_h_preproc.py | 8 +-- preproc/gfas12_preproc.py | 4 +- preproc/htapv2_preproc.py | 54 +++++++++---------- preproc/tno_mac_iii_preproc.py | 16 +++--- preproc/tno_mac_iii_preproc_voc_ratios.py | 26 ++++----- preproc/wiedinmyer_preproc.py | 12 ++--- 45 files changed, 278 insertions(+), 275 deletions(-) diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index d174e04..4a73416 100755 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -247,10 +247,10 @@ class Config(ArgParser): elif str_bool in false_options: return False else: - print 'WARNING: Boolean value not contemplated use {0} for True values and {1} for the False ones'.format( + print('WARNING: Boolean value not contemplated use {0} for True values and {1} for the False ones'.format( true_options, false_options - ) - print '/t Using False as default' + )) + print('/t Using False as default') return False @staticmethod @@ -308,10 +308,10 @@ class Config(ArgParser): """ Defines the log_level using the common script settings. """ - import settings + from . import settings settings.define_global_vars(self.options.log_level) if __name__ == '__main__': config = Config() - print config.options + print(config.options) diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index 1b93cfa..18b6a6a 100755 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -20,6 +20,7 @@ import os import numpy as np +from functools import reduce global refresh_log @@ -118,9 +119,9 @@ def finish_logs(output_dir, date): os.remove(times_path) df_merged = reduce(lambda left, right: pd.merge(left, right, on=['Class', 'Function'], how='outer'), data_frames) - df_merged['min'] = df_merged.loc[:, range(size)].min(axis=1) - df_merged['max'] = df_merged.loc[:, range(size)].max(axis=1) - df_merged['mean'] = df_merged.loc[:, range(size)].mean(axis=1) + df_merged['min'] = df_merged.loc[:, list(range(size))].min(axis=1) + df_merged['max'] = df_merged.loc[:, list(range(size))].max(axis=1) + df_merged['mean'] = df_merged.loc[:, list(range(size))].mean(axis=1) df_merged.to_csv(times_path) comm.Barrier() diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index ed0a837..43e0db7 100755 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -286,9 +286,9 @@ class EmissionInventory(object): """ import pandas as pd import re - from point_gfas_emission_inventory import PointGfasEmissionInventory - from gfas_emission_inventory import GfasEmissionInventory - from point_source_emission_inventory import PointSourceEmissionInventory + from .point_gfas_emission_inventory import PointGfasEmissionInventory + from .gfas_emission_inventory import GfasEmissionInventory + from .point_source_emission_inventory import PointSourceEmissionInventory st_time = timeit.default_timer() settings.write_log('Loading emissions') diff --git a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py index b237bc5..33a5328 100755 --- a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py @@ -21,7 +21,7 @@ import os import timeit import hermesv3_gr.config.settings as settings -from emission_inventory import EmissionInventory +from .emission_inventory import EmissionInventory class GfasEmissionInventory(EmissionInventory): @@ -132,7 +132,7 @@ class GfasEmissionInventory(EmissionInventory): else: alt_var = None - print "ERROR: Only 'sovief' and 'prm' methods are accepted." + print("ERROR: Only 'sovief' and 'prm' methods are accepted.") [alt] = extract_vars(self.get_input_path(), [alt_var]) @@ -211,7 +211,7 @@ class GfasEmissionInventory(EmissionInventory): st_time = timeit.default_timer() settings.write_log("\tRegridding", level=2) - for i in xrange(len(self.emissions)): + for i in range(len(self.emissions)): self.emissions[i]["data"] = self.do_vertical_allocation(self.emissions[i]["data"]) regridded_emissions = self.regrid.start_regridding(gfas=True, vertical=self.vertical) diff --git a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py index 0eb27d3..78167bb 100755 --- a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py @@ -23,7 +23,7 @@ import timeit import warnings import hermesv3_gr.config.settings as settings -from emission_inventory import EmissionInventory +from .emission_inventory import EmissionInventory class PointGfasEmissionInventory(EmissionInventory): @@ -148,7 +148,7 @@ class PointGfasEmissionInventory(EmissionInventory): else: alt_var = None - print "ERROR: Only 'sovief' and 'prm' methods are accepted." + print("ERROR: Only 'sovief' and 'prm' methods are accepted.") [alt] = extract_vars(self.get_input_path(), [alt_var]) @@ -213,7 +213,7 @@ class PointGfasEmissionInventory(EmissionInventory): :return: Emissions already allocated on the top altitude of each fire. :rtype: numpy.array """ - print 'do_vertical_allocation' + print('do_vertical_allocation') sys.exit() st_time = timeit.default_timer() @@ -336,7 +336,7 @@ class PointGfasEmissionInventory(EmissionInventory): # sys.exit() gdf = gpd.sjoin(gdf.to_crs(grid_shp.crs), grid_shp, how='inner') - print gdf + print(gdf) gdf = np.array_split(gdf, settings.size) else: gdf = None @@ -346,7 +346,7 @@ class PointGfasEmissionInventory(EmissionInventory): for num, pollutant in enumerate(self.pollutant_dicts): settings.write_log('\t\tPollutant {0} ({1}/{2})'.format( pollutant['name'], num + 1, len(self.pollutant_dicts)), level=3) - print ('\t\tPollutant {0} ({1}/{2})'.format(pollutant['name'], num + 1, len(self.pollutant_dicts))) + print(('\t\tPollutant {0} ({1}/{2})'.format(pollutant['name'], num + 1, len(self.pollutant_dicts)))) aux = netcdf.variables[pollutant['name']][:].flatten()[gdf['src_index']] gdf[pollutant['name']] = aux * gdf['src_area'] # print 'masa {0}: {1} '.format(pollutant['name'], gdf[pollutant['name']].sum()) @@ -357,7 +357,7 @@ class PointGfasEmissionInventory(EmissionInventory): netcdf.close() settings.write_time('PointGfasEmissionInventory', 'do_regrid', timeit.default_timer() - st_time, level=2) - print 'regrid done' + print('regrid done') del gdf['src_index'], gdf['index_right'] self.emissions = gdf @@ -393,7 +393,7 @@ class PointGfasEmissionInventory(EmissionInventory): warnings.warn('WARNING: One or more fires have an altitude of fire emission injection higher than the top' + ' layer of the model defined in the {0} file'.format(vertical_description_path)) - print self.emissions.loc[~self.emissions['altitude'].isna()] + print(self.emissions.loc[~self.emissions['altitude'].isna()]) del self.emissions['altitude'] self.emissions = self.emissions.groupby(['FID', 'layer']).sum() diff --git a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py index 60e32b8..791f60f 100755 --- a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py @@ -20,7 +20,7 @@ import timeit import hermesv3_gr.config.settings as settings -from emission_inventory import EmissionInventory +from .emission_inventory import EmissionInventory class PointSourceEmissionInventory(EmissionInventory): diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index ce1133b..dca2013 100755 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -499,13 +499,13 @@ class Grid(object): df = pd.concat([df_lats, df_lons], axis=1) # Substituate 8 columns by 4 with the two coordinates - df['p1'] = zip(df.b_lon_1, df.b_lat_1) + df['p1'] = list(zip(df.b_lon_1, df.b_lat_1)) del df['b_lat_1'], df['b_lon_1'] - df['p2'] = zip(df.b_lon_2, df.b_lat_2) + df['p2'] = list(zip(df.b_lon_2, df.b_lat_2)) del df['b_lat_2'], df['b_lon_2'] - df['p3'] = zip(df.b_lon_3, df.b_lat_3) + df['p3'] = list(zip(df.b_lon_3, df.b_lat_3)) del df['b_lat_3'], df['b_lon_3'] - df['p4'] = zip(df.b_lon_4, df.b_lat_4) + df['p4'] = list(zip(df.b_lon_4, df.b_lat_4)) del df['b_lat_4'], df['b_lon_4'] # Make a list of list of tuples diff --git a/hermesv3_gr/modules/grids/grid_global.py b/hermesv3_gr/modules/grids/grid_global.py index ee0b1d1..a3ca236 100755 --- a/hermesv3_gr/modules/grids/grid_global.py +++ b/hermesv3_gr/modules/grids/grid_global.py @@ -23,7 +23,7 @@ import sys import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class GlobalGrid(Grid): diff --git a/hermesv3_gr/modules/grids/grid_latlon.py b/hermesv3_gr/modules/grids/grid_latlon.py index f66f80e..0409d7b 100755 --- a/hermesv3_gr/modules/grids/grid_latlon.py +++ b/hermesv3_gr/modules/grids/grid_latlon.py @@ -23,7 +23,7 @@ import sys import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class LatLonGrid(Grid): diff --git a/hermesv3_gr/modules/grids/grid_lcc.py b/hermesv3_gr/modules/grids/grid_lcc.py index c0b4d4e..d360d74 100755 --- a/hermesv3_gr/modules/grids/grid_lcc.py +++ b/hermesv3_gr/modules/grids/grid_lcc.py @@ -22,7 +22,7 @@ import os import sys import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class LccGrid(Grid): diff --git a/hermesv3_gr/modules/grids/grid_mercator.py b/hermesv3_gr/modules/grids/grid_mercator.py index c715b8e..38bb722 100755 --- a/hermesv3_gr/modules/grids/grid_mercator.py +++ b/hermesv3_gr/modules/grids/grid_mercator.py @@ -22,7 +22,7 @@ import os import sys import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class MercatorGrid(Grid): diff --git a/hermesv3_gr/modules/grids/grid_rotated.py b/hermesv3_gr/modules/grids/grid_rotated.py index eabf8e7..24c0f9a 100755 --- a/hermesv3_gr/modules/grids/grid_rotated.py +++ b/hermesv3_gr/modules/grids/grid_rotated.py @@ -22,7 +22,7 @@ import sys import os import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class RotatedGrid(Grid): diff --git a/hermesv3_gr/modules/masking/masking.py b/hermesv3_gr/modules/masking/masking.py index 36b1c93..f02f736 100755 --- a/hermesv3_gr/modules/masking/masking.py +++ b/hermesv3_gr/modules/masking/masking.py @@ -103,7 +103,7 @@ class Masking(object): # Partition @lst in @n balanced parts, in given order parts, rest = divmod(len(lst), num) lstiter = iter(lst) - for j in xrange(num): + for j in range(num): plen = len(lst) / num + (1 if rest > 0 else 0) rest -= 1 yield list(itertools.islice(lstiter, plen)) @@ -123,7 +123,7 @@ class Masking(object): dst_var = [] num = 0 - points = np.array(zip(lat.flatten(), lon.flatten())) + points = np.array(list(zip(lat.flatten(), lon.flatten()))) points_list = list(self.partlst(points, settings.size)) @@ -289,7 +289,7 @@ class Masking(object): values = values['data'] mask = np.ones(values.shape) - for code, factor in self.factors_mask_values.iteritems(): + for code, factor in self.factors_mask_values.items(): mask[values == code] = factor settings.write_time('Masking', 'custom_scale_mask', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/regrid/regrid_conservative.py b/hermesv3_gr/modules/regrid/regrid_conservative.py index f2de4d1..35c718f 100755 --- a/hermesv3_gr/modules/regrid/regrid_conservative.py +++ b/hermesv3_gr/modules/regrid/regrid_conservative.py @@ -23,7 +23,7 @@ import timeit import ESMF import hermesv3_gr.config.settings as settings -from regrid import Regrid +from .regrid import Regrid class ConservativeRegrid(Regrid): @@ -151,18 +151,18 @@ class ConservativeRegrid(Regrid): if os.path.exists(self.weight_matrix_file): pre_size = 0 post_size = 1 - print "I'm {0}".format(settings.rank), 'Writing Weight Matrix {0}'.format(self.weight_matrix_file) + print("I'm {0}".format(settings.rank), 'Writing Weight Matrix {0}'.format(self.weight_matrix_file)) # find = True while pre_size != post_size: - print "I'm {0}".format(settings.rank), pre_size, post_size + print("I'm {0}".format(settings.rank), pre_size, post_size) pre_size = post_size post_size = os.path.getsize(self.weight_matrix_file) time.sleep(1) find = True - print "I'm {0}".format(settings.rank), 'FINISHED' + print("I'm {0}".format(settings.rank), 'FINISHED') else: time.sleep(5) - print "I'm {0}".format(settings.rank), 'Waiting Weight Matrix' + print("I'm {0}".format(settings.rank), 'Waiting Weight Matrix') def apply_weights(self, values): """ diff --git a/hermesv3_gr/modules/speciation/speciation.py b/hermesv3_gr/modules/speciation/speciation.py index 6084c44..9d0c1b8 100755 --- a/hermesv3_gr/modules/speciation/speciation.py +++ b/hermesv3_gr/modules/speciation/speciation.py @@ -81,7 +81,7 @@ class Speciation(object): long_name_dict = df.loc[df[df.ID == 'short_description'].index[0]].to_dict() long_name_dict.pop('ID', None) profile_list = [] - for key in formulas_dict.iterkeys(): + for key in formulas_dict.keys(): profile_list.append({ 'name': key, 'formula': formulas_dict[key], diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 858edd5..4fdc08e 100755 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -359,12 +359,12 @@ class TemporalDistribution(object): weekdays_count = self.calculate_weekdays(date) if isinstance(profile, dict): factor = self.calculate_weekday_factor(profile, weekdays_count) - for dict_key in profile.iterkeys(): + for dict_key in profile.keys(): profile[dict_key] = profile[dict_key] + factor elif isinstance(profile, np.ndarray): # Gridded factor = self.calculate_weekday_gridded_factor(profile, weekdays_count) - for weekday in xrange(7): + for weekday in range(7): profile[weekday, :] = profile[weekday, :] + factor else: settings.write_log('ERROR: Check the .err file to get more info.') @@ -460,7 +460,7 @@ class TemporalDistribution(object): profile = self.parse_hourly_profile_id(profile_id) if profile is not None: - for profile_type, profile_id_aux in profile.iteritems(): + for profile_type, profile_id_aux in profile.items(): # Gridded monthly profile if os.path.exists(profile_id_aux): profile_aux = self.get_gridded_temporal_profile('Fhour', profile_id_aux) @@ -564,7 +564,7 @@ class TemporalDistribution(object): 'Asia/Famagusta': 'Asia/Nicosia', } - if timezone in tz_dict.iterkeys(): + if timezone in iter(tz_dict.keys()): timezone = tz_dict[timezone] return timezone @@ -625,7 +625,7 @@ class TemporalDistribution(object): dst_var = [] num = 0 - points = zip(lat.flatten(), lon.flatten()) + points = list(zip(lat.flatten(), lon.flatten())) for lat_aux, lon_aux in points: num += 1 @@ -692,7 +692,7 @@ class TemporalDistribution(object): nc_in.close() tz_list = np.chararray(timezones.shape, itemsize=32) - for id_aux in xrange(timezones.min(), timezones.max() + 1): + for id_aux in range(timezones.min(), timezones.max() + 1): try: timezone = self.get_tz_from_id(id_aux) tz_list[timezones == id_aux] = timezone @@ -739,7 +739,7 @@ class TemporalDistribution(object): if isinstance(weekday_profile, dict): df['weekday_factor'] = df['hour'].map(weekday_profile) else: - for hour in xrange(24): + for hour in range(24): df.loc[df['hour'] == hour, 'weekday_factor'] = weekday_profile[ hour, df[df['hour'] == hour].index] # SATURDAY @@ -747,7 +747,7 @@ class TemporalDistribution(object): if isinstance(saturday_profile, dict): df['saturday_factor'] = df['hour'].map(saturday_profile) else: - for hour in xrange(24): + for hour in range(24): df.loc[df['hour'] == hour, 'saturday_factor'] = saturday_profile[ hour, df[df['hour'] == hour].index] # SUNDAY @@ -755,7 +755,7 @@ class TemporalDistribution(object): if isinstance(sunday_profile, dict): df['sunday_factor'] = df['hour'].map(sunday_profile) else: - for hour in xrange(24): + for hour in range(24): df.loc[df['hour'] == hour, 'sunday_factor'] = sunday_profile[ hour, df[df['hour'] == hour].index] @@ -782,7 +782,7 @@ class TemporalDistribution(object): if isinstance(self.weekly_profile[month], dict): df.loc[df['month'] == month, 'weekly_factor'] = df['weekday'].map(self.weekly_profile[month]) else: - for weekday in xrange(7): + for weekday in range(7): df.loc[df['weekday'] == weekday, 'weekly_factor'] = self.weekly_profile[month][ weekday, df[df['weekday'] == weekday].index] @@ -871,7 +871,7 @@ class TemporalDistribution(object): weekdays_factors = 0 num_days = 0 - for weekday in xrange(7): + for weekday in range(7): weekdays_factors += profile[weekday] * weekdays[weekday] num_days += weekdays[weekday] @@ -894,7 +894,7 @@ class TemporalDistribution(object): weekdays_factors = np.zeros((profile.shape[-1])) num_days = 0 - for weekday in xrange(7): + for weekday in range(7): weekdays_factors += profile[weekday, :] * weekdays[weekday] num_days += weekdays[weekday] @@ -918,7 +918,7 @@ class TemporalDistribution(object): st_time = timeit.default_timer() weekdays = [MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY] - days = [weekday(date.year, date.month, d + 1) for d in xrange(monthrange(date.year, date.month)[1])] + days = [weekday(date.year, date.month, d + 1) for d in range(monthrange(date.year, date.month)[1])] weekdays_dict = {} count = 0 for day in weekdays: diff --git a/hermesv3_gr/modules/vertical/vertical.py b/hermesv3_gr/modules/vertical/vertical.py index 1b309ea..34baa1a 100755 --- a/hermesv3_gr/modules/vertical/vertical.py +++ b/hermesv3_gr/modules/vertical/vertical.py @@ -84,7 +84,7 @@ class VerticalDistribution(object): " The v_profile '{0}' of the '{1}' file doesn't match.".format(self.id, path)) sys.exit(1) else: - return_value = zip(v_profile['layers'], v_profile['weights']) + return_value = list(zip(v_profile['layers'], v_profile['weights'])) settings.write_time('VerticalDistribution', 'get_vertical_profile', timeit.default_timer() - st_time, level=3) @@ -140,7 +140,7 @@ class VerticalDistribution(object): index = len([s for s in output_vertical_profile if s < prev_layer]) origin_diff_factor = in_weight / (layer - prev_layer) weight_list = [] - for i in xrange(len(output_vertical_profile_aux) - 1): + for i in range(len(output_vertical_profile_aux) - 1): weight = (abs(output_vertical_profile_aux[i] - output_vertical_profile_aux[i + 1])) * origin_diff_factor weight_list.append({'index': index, 'weight': weight}) index += 1 diff --git a/hermesv3_gr/modules/vertical/vertical_gfas.py b/hermesv3_gr/modules/vertical/vertical_gfas.py index 3090926..5e65c19 100755 --- a/hermesv3_gr/modules/vertical/vertical_gfas.py +++ b/hermesv3_gr/modules/vertical/vertical_gfas.py @@ -20,7 +20,7 @@ import timeit import hermesv3_gr.config.settings as settings -from vertical import VerticalDistribution +from .vertical import VerticalDistribution class GfasVerticalDistribution(VerticalDistribution): @@ -68,7 +68,7 @@ class GfasVerticalDistribution(VerticalDistribution): st_time = timeit.default_timer() widths = [] - for i in xrange(len(heights_list)): + for i in range(len(heights_list)): if i == 0: widths.append(heights_list[i]) else: @@ -126,10 +126,10 @@ class GfasVerticalDistribution(VerticalDistribution): st_time = timeit.default_timer() fires = np.zeros(top_fires.shape) - for i in xrange(len(self.output_heights)): + for i in range(len(self.output_heights)): if top_fires[i].sum() != 0: weight_list = self.get_weights(list(self.output_heights[0: i + 1])) - for i_weight in xrange(len(weight_list)): + for i_weight in range(len(weight_list)): fires[i_weight] += top_fires[i] * weight_list[i_weight] settings.write_time('GfasVerticalDistribution', 'apply_approach', timeit.default_timer() - st_time, level=3) @@ -185,7 +185,7 @@ class GfasVerticalDistribution(VerticalDistribution): return fire_list def calculate_weight_layer_dict(self, layer): - weight_layer_dict = {x: None for x in xrange(layer + 1)} + weight_layer_dict = {x: None for x in range(layer + 1)} if self.approach == '50_top': weight_layer_dict[layer] = 0.5 to_distribute = 0.5 @@ -213,7 +213,7 @@ class GfasVerticalDistribution(VerticalDistribution): vert_emissions.append(emis) else: weight_layer_dict = self.calculate_weight_layer_dict(layer) - for layer, weight in weight_layer_dict.iteritems(): + for layer, weight in weight_layer_dict.items(): aux_emis = emis.copy() aux_emis.loc[:, 'layer'] = layer aux_emis.loc[:, input_pollutant_list] = aux_emis[input_pollutant_list].multiply(weight) diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index 06e6f34..8cb46f5 100755 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -24,6 +24,7 @@ import numpy as np from mpi4py import MPI from netCDF4 import Dataset from hermesv3_gr.config import settings +from functools import reduce class Writer(object): @@ -128,7 +129,7 @@ class Writer(object): settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) index = 0 # print "Rank {0} 2".format(rank) - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) st_time = timeit.default_timer() @@ -179,7 +180,7 @@ class Writer(object): dict_aux['data'] = None empty_dict[emi['name']] = dict_aux - self.variables_attributes = empty_dict.values() + self.variables_attributes = list(empty_dict.values()) settings.write_time('Writer', 'set_variable_attributes', timeit.default_timer() - st_time, level=3) @@ -386,7 +387,7 @@ class Writer(object): netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat',) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -397,22 +398,22 @@ class Writer(object): netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat',) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( - len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( + len(center_longitudes.shape))) sys.exit(1) elif roated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -560,7 +561,7 @@ class Writer(object): try: var[:] = variable['data'] except ValueError: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 7b3480c..3c88760 100755 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -198,7 +198,7 @@ class WriterCmaq(Writer): if self.global_attributes_path is not None: df = pd.read_csv(self.global_attributes_path) - for att in atts_dict.iterkeys(): + for att in atts_dict.keys(): try: if att in int_atts: atts_dict[att] = np.int32(df.loc[df['attribute'] == att, 'value'].item()) @@ -220,7 +220,7 @@ class WriterCmaq(Writer): settings.write_log('WARNING: Check the .err file to get more information.') message = 'WARNING: No output attributes defined, check the output_attributes' message += ' parameter of the configuration file.\nUsing default values:' - for key, value in atts_dict.iteritems(): + for key, value in atts_dict.items(): message += '\n\t{0} = {1}'.format(key, value) if settings.rank == 0: warning(message) @@ -431,7 +431,7 @@ class WriterCmaq(Writer): index = 0 # data_list, var_list = self.change_variable_attributes(self.variables_attributes) - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): index += 1 var = netcdf.createVariable(var_name, 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) var.setncatts(self.variables_attributes[var_name]) @@ -440,7 +440,7 @@ class WriterCmaq(Writer): # ===== Global attributes ===== settings.write_log("\t\tCreating NetCDF metadata.", level=2) - global_attributes = self.create_global_attributes(self.variables_attributes.keys()) + global_attributes = self.create_global_attributes(list(self.variables_attributes.keys())) for attribute in self.global_attributes_order: netcdf.setncattr(attribute, global_attributes[attribute]) @@ -504,7 +504,7 @@ class WriterCmaq(Writer): full_shape = None index = 0 # data_list, var_list = self.change_variable_attributes(self.variables_attributes) - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): if settings.size != 1: settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) @@ -575,7 +575,7 @@ class WriterCmaq(Writer): if mpi_numpy: data = np.ones(var[:].shape, dtype=settings.precision) * 100 - for i in xrange(settings.size): + for i in range(settings.size): try: if i == 0: var[:, :, :, :full_position[i][3]] = recvbuf[i] @@ -595,7 +595,7 @@ class WriterCmaq(Writer): elif mpi_vector: if rank_data is not None: data = np.empty(var[:].shape, dtype=settings.precision) - for i in xrange(settings.size): + for i in range(settings.size): # print 'Resizeing {0}'.format(i) if not i == settings.size - 1: data[:, :, full_position[i][0]:full_position[i][1], @@ -615,7 +615,7 @@ class WriterCmaq(Writer): settings.write_log("\t\tCreating NetCDF metadata.", level=2) if settings.rank == 0: # ===== Global attributes ===== - global_attributes = self.create_global_attributes(self.variables_attributes.keys()) + global_attributes = self.create_global_attributes(list(self.variables_attributes.keys())) for attribute in self.global_attributes_order: netcdf.setncattr(attribute, global_attributes[attribute]) diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index 8749ed5..1bd6758 100755 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -154,8 +154,8 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) lat_dim = ('lon', 'lat', ) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format( - len(self.grid.center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format( + len(self.grid.center_latitudes.shape))) sys.exit(1) # Longitude @@ -168,15 +168,15 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[1]), level=3) lon_dim = ('lon', 'lat', ) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( - len(self.grid.center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( + len(self.grid.center_longitudes.shape))) sys.exit(1) elif Rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if self.grid.rlat is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(self.grid.rlat)) settings.write_log("\t\t\t'rlat' dimension: {0}".format(len(self.grid.rlat)), level=3) @@ -184,7 +184,7 @@ class WriterMonarch(Writer): # Rotated Longitude if self.grid.rlon is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(self.grid.rlon)) settings.write_log("\t\t\t'rlon' dimension: {0}".format(len(self.grid.rlon)), level=3) @@ -333,7 +333,7 @@ class WriterMonarch(Writer): var[:] = 0 index = 0 - for var_name, variable in self.variables_attributes.iteritems(): + for var_name, variable in self.variables_attributes.items(): index += 1 var = netcdf.createVariable(var_name, 'f', ('time',) + var_dim, zlib=self.compress) @@ -632,7 +632,7 @@ class WriterMonarch(Writer): full_shape = None index = 0 - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): if settings.size != 1: settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) @@ -719,7 +719,7 @@ class WriterMonarch(Writer): if mpi_numpy: data = np.ones(var[:].shape, dtype=settings.precision) * 100 - for i in xrange(settings.size): + for i in range(settings.size): try: if i == 0: var[:, :, :, :full_position[i][3]] = recvbuf[i] @@ -739,7 +739,7 @@ class WriterMonarch(Writer): elif mpi_vector: if rank_data is not None: data = np.empty(var[:].shape, dtype=settings.precision) - for i in xrange(settings.size): + for i in range(settings.size): if not i == settings.size - 1: data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = \ diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index 364dbcd..407e623 100755 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -216,7 +216,7 @@ class WriterWrfChem(Writer): if self.global_attributes_path is not None: df = pd.read_csv(self.global_attributes_path) - for att in atts_dict.iterkeys(): + for att in atts_dict.keys(): try: if att in int_atts: atts_dict[att] = np.int32(df.loc[df['attribute'] == att, 'value'].item()) @@ -225,7 +225,7 @@ class WriterWrfChem(Writer): elif att in str_atts: atts_dict[att] = str(df.loc[df['attribute'] == att, 'value'].item()) except ValueError: - print 'A warning has occurred. Check the .err file to get more information.' + print('A warning has occurred. Check the .err file to get more information.') if settings.rank == 0: warning('The global attribute {0} is not defined; Using default value {1}'.format( att, atts_dict[att])) @@ -234,7 +234,7 @@ class WriterWrfChem(Writer): settings.write_log('WARNING: Check the .err file to get more information.') message = 'WARNING: No output attributes defined, check the output_attributes' message += ' parameter of the configuration file.\nUsing default values:' - for key, value in atts_dict.iteritems(): + for key, value in atts_dict.items(): message += '\n\t{0} = {1}'.format(key, value) if settings.rank == 0: warning(message) @@ -343,7 +343,7 @@ class WriterWrfChem(Writer): settings.write_log("\t\t\t'Times' variable created with size: {0}".format(times[:].shape), level=3) index = 0 - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): index += 1 var = netcdf.createVariable(var_name, 'f', ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=self.compress) @@ -410,7 +410,7 @@ class WriterWrfChem(Writer): # self.change_variable_attributes() - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): if settings.size != 1: settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) @@ -458,7 +458,7 @@ class WriterWrfChem(Writer): if rank_data is not None: data = np.empty(var[:].shape, dtype=settings.precision) - for i in xrange(settings.size): + for i in range(settings.size): # print 'Resizeing {0}'.format(i) if not i == settings.size - 1: data[:, :, full_position[i][0]:full_position[i][1], diff --git a/hermesv3_gr/tools/coordinates_tools.py b/hermesv3_gr/tools/coordinates_tools.py index 56d929a..00baa67 100755 --- a/hermesv3_gr/tools/coordinates_tools.py +++ b/hermesv3_gr/tools/coordinates_tools.py @@ -296,9 +296,9 @@ if __name__ == '__main__': import numpy as np new_pole_lon_d = 20.0 # lonpole tlm0d new_pole_lat_d = 35.0 # latpole tph0d - print latlon2rotated(new_pole_lon_d, new_pole_lat_d, 20.0, 35.0) - print latlon2rotated(new_pole_lon_d, new_pole_lat_d, -20.2485, -9.9036) - print rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, 0, 0) - print rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, -51., -35.) - print rotated2latlon(new_pole_lon_d, new_pole_lat_d, np.array([-51., -51., -51., -51.]), - np.array([-35., -34.9, -34.8, -34.7])) + print(latlon2rotated(new_pole_lon_d, new_pole_lat_d, 20.0, 35.0)) + print(latlon2rotated(new_pole_lon_d, new_pole_lat_d, -20.2485, -9.9036)) + print(rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, 0, 0)) + print(rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, -51., -35.)) + print(rotated2latlon(new_pole_lon_d, new_pole_lat_d, np.array([-51., -51., -51., -51.]), + np.array([-35., -34.9, -34.8, -34.7]))) diff --git a/hermesv3_gr/tools/download_benchmark.py b/hermesv3_gr/tools/download_benchmark.py index 5d58bc8..ef08a18 100755 --- a/hermesv3_gr/tools/download_benchmark.py +++ b/hermesv3_gr/tools/download_benchmark.py @@ -36,7 +36,7 @@ def query_yes_no(question, default="yes"): while True: sys.stdout.write(question + prompt) - choice = raw_input().lower() + choice = input().lower() if default is not None and choice == '': return valid[default] elif choice in valid: @@ -47,12 +47,12 @@ def query_yes_no(question, default="yes"): def check_args(args, exe_str): if len(args) == 0: - print("Missing destination path after '{0}'. e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES".format(exe_str)) + print(("Missing destination path after '{0}'. e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES".format(exe_str))) sys.exit(1) elif len(args) > 1: - print("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES".format(exe_str)) + print(("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES".format(exe_str))) sys.exit(1) else: dir_path = args[0] diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index 955468e..a56842a 100755 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -21,6 +21,7 @@ import sys from netCDF4 import Dataset from mpi4py import MPI +from functools import reduce ICOMM = MPI.COMM_WORLD COMM = ICOMM.Split(color=0, key=0) @@ -168,7 +169,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat', ) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -179,21 +180,21 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat', ) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -337,7 +338,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, try: var[:] = variable['data'] except ValueError: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -435,7 +436,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat', ) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -446,21 +447,21 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat', ) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -607,7 +608,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, shape = tuple() exec ("shape = (len(hours), {0}.size, {1}.size, {2}.size)".format(var_dim[0], var_dim[1], var_dim[2])) # exit() - print shape + print(shape) var[:] = np.zeros(shape) # Grid mapping diff --git a/hermesv3_gr/tools/sample_files.py b/hermesv3_gr/tools/sample_files.py index d8fee3f..17c7eac 100755 --- a/hermesv3_gr/tools/sample_files.py +++ b/hermesv3_gr/tools/sample_files.py @@ -131,7 +131,7 @@ def query_yes_no(question, default="yes"): while True: sys.stdout.write(question + prompt) - choice = raw_input().lower() + choice = input().lower() if default is not None and choice == '': return valid[default] elif choice in valid: @@ -142,12 +142,12 @@ def query_yes_no(question, default="yes"): def check_args(args, exe_str): if len(args) == 0: - print("Missing destination path after '{0}'. e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str)) + print(("Missing destination path after '{0}'. e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str))) sys.exit(1) elif len(args) > 1: - print("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str)) + print(("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str))) sys.exit(1) else: dir_path = args[0] @@ -169,7 +169,7 @@ def copy_files(file_list, directory): for element in file_list: if dict == type(element): - copy_files(element.values()[0], os.path.join(directory, element.keys()[0])) + copy_files(list(element.values())[0], os.path.join(directory, list(element.keys())[0])) else: copy2(element, directory) return True diff --git a/preproc/cams_glob_ant_preproc.py b/preproc/cams_glob_ant_preproc.py index 2d6ae51..44113e5 100755 --- a/preproc/cams_glob_ant_preproc.py +++ b/preproc/cams_glob_ant_preproc.py @@ -213,7 +213,7 @@ def create_bounds(coordinates, number_vertices=2): def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cell_area, date): # TODO Documentation - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -341,7 +341,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/cams_glob_ocean_preproc.py b/preproc/cams_glob_ocean_preproc.py index b4c42db..8b9fae2 100755 --- a/preproc/cams_glob_ocean_preproc.py +++ b/preproc/cams_glob_ocean_preproc.py @@ -184,7 +184,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/cams_glob_ship_preproc.py b/preproc/cams_glob_ship_preproc.py index 49e98d6..9e2a184 100755 --- a/preproc/cams_glob_ship_preproc.py +++ b/preproc/cams_glob_ship_preproc.py @@ -190,7 +190,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/cams_glob_soil_preproc.py b/preproc/cams_glob_soil_preproc.py index cade121..3fc4e7e 100755 --- a/preproc/cams_glob_soil_preproc.py +++ b/preproc/cams_glob_soil_preproc.py @@ -204,7 +204,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/cams_reg_ap_preproc.py b/preproc/cams_reg_ap_preproc.py index 57b3794..46eb47b 100755 --- a/preproc/cams_reg_ap_preproc.py +++ b/preproc/cams_reg_ap_preproc.py @@ -78,14 +78,14 @@ def calculate_grid_definition(in_path): # Longitudes lons = np.sort(np.unique(dataframe.Lon_rounded)) lons_interval = lons[1:] - lons[:-1] - print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - dataframe.Lon_rounded.min(), dataframe.Lon_rounded.max(), lons_interval.min(), len(lons)) + print('Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( + dataframe.Lon_rounded.min(), dataframe.Lon_rounded.max(), lons_interval.min(), len(lons))) # Latitudes lats = np.sort(np.unique(dataframe.Lat_rounded)) lats_interval = lats[1:] - lats[:-1] - print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - dataframe.Lat_rounded.min(), dataframe.Lat_rounded.max(), lats_interval.min(), len(lats)) + print('Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( + dataframe.Lat_rounded.min(), dataframe.Lat_rounded.max(), lats_interval.min(), len(lats))) lats = np.arange(-90 + lats_interval.min() / 2, 90, lats_interval.min(), dtype=np.float64) lons = np.arange(-180 + lons_interval.min() / 2, 180, lons_interval.min(), dtype=np.float64) @@ -172,7 +172,7 @@ def do_transformation(year): #dataframe = pd.concat([df_np, df_p]) for name, group in dataframe.groupby('GNFR_Sector'): - print 'gnfr', name + print('gnfr', name) pollutant_list = create_pollutant_empty_list(in_file, len(c_lats), len(c_lons)) # Other mobile sources ignoring sea cells (shipping emissions) @@ -182,7 +182,7 @@ def do_transformation(year): group = group.groupby(['row_lat', 'col_lon']).sum().reset_index() - for i in xrange(len(pollutant_list)): + for i in range(len(pollutant_list)): # pollutant_list[i]['data'][group.col_lon, group.row_lat] = group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'][group.row_lat, group.col_lon] += group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'] = pollutant_list[i]['data'].reshape((1,) + pollutant_list[i]['data'].shape) @@ -396,8 +396,8 @@ def check_vocs(year): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - gnfr, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum) + print('{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + gnfr, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum)) return True def do_pm_transformation(year): @@ -475,8 +475,8 @@ def check_pm(year): [new_pm] = extract_vars(pm_path, [pm]) pm_sum += new_pm['data'].sum() - print '{0} PM2.5 sum: {1}; PM sum: {2}; %diff: {3}'.format( - gnfr, pm25_sum, pm_sum, 100*(pm25_sum - pm_sum) / pm25_sum) + print('{0} PM2.5 sum: {1}; PM sum: {2}; %diff: {3}'.format( + gnfr, pm25_sum, pm_sum, 100*(pm25_sum - pm_sum) / pm25_sum)) return True if __name__ == '__main__': diff --git a/preproc/cams_reg_ap_preproc_pm_ratios.py b/preproc/cams_reg_ap_preproc_pm_ratios.py index d2ff01c..ee18136 100755 --- a/preproc/cams_reg_ap_preproc_pm_ratios.py +++ b/preproc/cams_reg_ap_preproc_pm_ratios.py @@ -60,7 +60,7 @@ def extract_vars(netcdf_path, variables_list, attributes_list=()): data_list.append(dict_aux) netcdf.close() - print data_list + print(data_list) return data_list @@ -104,7 +104,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, if not (regular_latlon or lcc or rotated): regular_latlon = True - print netcdf_path + print(netcdf_path) netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== @@ -119,7 +119,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat',) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -130,21 +130,21 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat',) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -218,7 +218,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lons.axis = "X" lons.long_name = "longitude coordinate" lons.standard_name = "longitude" - print 'lons:', lons[:].shape, center_longitudes.shape + print('lons:', lons[:].shape, center_longitudes.shape) lons[:] = center_longitudes if boundary_longitudes is not None: lons.bounds = "lon_bnds" @@ -290,7 +290,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, try: var[:] = variable['data'] except Exception: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -391,22 +391,22 @@ def create_pm_ratio(pm): import numpy as np [country_values, lat, lon] = extract_vars(TNO_WORLD_MASK, ['timezone_id', 'lat', 'lon']) country_values = country_values['data'].reshape((country_values['data'].shape[1], country_values['data'].shape[1])) - print country_values + print(country_values) #sys.exit() - print OUTPUT_PATH + print(OUTPUT_PATH) if not os.path.exists(OUTPUT_PATH): os.makedirs(OUTPUT_PATH) complete_output_path = os.path.join(OUTPUT_PATH, 'ratio_{0}_{1}.nc'.format(pm, YEAR)) if not os.path.exists(complete_output_path): - print 'Creating ratio file for {0}\npath: {1}'.format(pm, complete_output_path) + print('Creating ratio file for {0}\npath: {1}'.format(pm, complete_output_path)) data_list = [] for gnfr in get_sector_list(pm): - print gnfr + print(gnfr) mask_factor = np.zeros(country_values.shape) iso_codes = get_iso_codes() - for country_code, factor in get_country_code_and_factor(pm, gnfr, YEAR).iteritems(): + for country_code, factor in get_country_code_and_factor(pm, gnfr, YEAR).items(): try: mask_factor[country_values == iso_codes[country_code]] = factor except Exception: @@ -421,7 +421,7 @@ def create_pm_ratio(pm): }) write_netcdf(complete_output_path, lat['data'], lon['data'], data_list) else: - print 'Ratio file for {0} already created\npath: {1}'.format(pm, complete_output_path) + print('Ratio file for {0} already created\npath: {1}'.format(pm, complete_output_path)) return True @@ -475,7 +475,7 @@ def get_pm_list(): del df['ISO3'], df['gnfr'], df['fr'], df['year'] df = df.drop_duplicates().dropna() pm_list = df.pmcode.values - print df.pmcode.values + print(df.pmcode.values) return df.pmcode.values @@ -493,7 +493,7 @@ def get_sector_list(pm): df = df[df.pmcode == pm] del df['ISO3'], df['pmcode'], df['year'], df['fr'] df = df.drop_duplicates().dropna() - print df.gnfr.values + print(df.gnfr.values) return df.gnfr.values diff --git a/preproc/cams_reg_ap_preproc_voc_ratios.py b/preproc/cams_reg_ap_preproc_voc_ratios.py index a3bc49c..6833243 100755 --- a/preproc/cams_reg_ap_preproc_voc_ratios.py +++ b/preproc/cams_reg_ap_preproc_voc_ratios.py @@ -60,7 +60,7 @@ def extract_vars(netcdf_path, variables_list, attributes_list=()): data_list.append(dict_aux) netcdf.close() - print data_list + print(data_list) return data_list @@ -104,7 +104,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, if not (regular_latlon or lcc or rotated): regular_latlon = True - print netcdf_path + print(netcdf_path) netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== @@ -119,7 +119,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat',) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -130,21 +130,21 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat',) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -218,7 +218,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lons.axis = "X" lons.long_name = "longitude coordinate" lons.standard_name = "longitude" - print 'lons:', lons[:].shape, center_longitudes.shape + print('lons:', lons[:].shape, center_longitudes.shape) lons[:] = center_longitudes if boundary_longitudes is not None: lons.bounds = "lon_bnds" @@ -290,7 +290,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, try: var[:] = variable['data'] except Exception: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -391,22 +391,22 @@ def create_voc_ratio(voc): import numpy as np [country_values, lat, lon] = extract_vars(TNO_WORLD_MASK, ['timezone_id', 'lat', 'lon']) country_values = country_values['data'].reshape((country_values['data'].shape[1], country_values['data'].shape[1])) - print country_values + print(country_values) #sys.exit() - print OUTPUT_PATH + print(OUTPUT_PATH) if not os.path.exists(OUTPUT_PATH): os.makedirs(OUTPUT_PATH) complete_output_path = os.path.join(OUTPUT_PATH, 'ratio_{0}_{1}.nc'.format(voc, YEAR)) if not os.path.exists(complete_output_path): - print 'Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path) + print('Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path)) data_list = [] for gnfr in get_sector_list(voc): - print gnfr + print(gnfr) mask_factor = np.zeros(country_values.shape) iso_codes = get_iso_codes() - for country_code, factor in get_country_code_and_factor(voc, gnfr, YEAR).iteritems(): + for country_code, factor in get_country_code_and_factor(voc, gnfr, YEAR).items(): try: mask_factor[country_values == iso_codes[country_code]] = factor except Exception: @@ -421,7 +421,7 @@ def create_voc_ratio(voc): }) write_netcdf(complete_output_path, lat['data'], lon['data'], data_list) else: - print 'Ratio file for {0} already created\npath: {1}'.format(voc, complete_output_path) + print('Ratio file for {0} already created\npath: {1}'.format(voc, complete_output_path)) return True @@ -472,14 +472,14 @@ def get_voc_list(): import pandas as pd df = pd.read_csv(CSV_PATH, sep=',') - print list(df.columns.values) + print(list(df.columns.values)) # sys.exit() del df['year'], df['ISO3'], df['gnfr'], df['fr'] df = df.drop_duplicates().dropna() voc_list = df.vcode.values - for i in xrange(len(voc_list)): + for i in range(len(voc_list)): voc_list[i] = voc_list[i].replace('v', 'voc') - print df.vcode.values + print(df.vcode.values) return df.vcode.values @@ -498,7 +498,7 @@ def get_sector_list(voc): df = df[df.vcode == voc] del df['ISO3'], df['vcode'], df['year'], df['fr'] df = df.drop_duplicates().dropna() - print df.gnfr.values + print(df.gnfr.values) return df.gnfr.values diff --git a/preproc/cams_reg_ghg_preproc.py b/preproc/cams_reg_ghg_preproc.py index 8447f4c..5981021 100755 --- a/preproc/cams_reg_ghg_preproc.py +++ b/preproc/cams_reg_ghg_preproc.py @@ -74,14 +74,14 @@ def calculate_grid_definition(in_path): # Longitudes lons = np.sort(np.unique(dataframe.Lon_rounded)) lons_interval = lons[1:] - lons[:-1] - print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - dataframe.Lon_rounded.min(), dataframe.Lon_rounded.max(), lons_interval.min(), len(lons)) + print('Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( + dataframe.Lon_rounded.min(), dataframe.Lon_rounded.max(), lons_interval.min(), len(lons))) # Latitudes lats = np.sort(np.unique(dataframe.Lat_rounded)) lats_interval = lats[1:] - lats[:-1] - print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - dataframe.Lat_rounded.min(), dataframe.Lat_rounded.max(), lats_interval.min(), len(lats)) + print('Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( + dataframe.Lat_rounded.min(), dataframe.Lat_rounded.max(), lats_interval.min(), len(lats))) lats = np.arange(-90 + lats_interval.min() / 2, 90, lats_interval.min(), dtype=np.float64) lons = np.arange(-180 + lons_interval.min() / 2, 180, lons_interval.min(), dtype=np.float64) @@ -168,7 +168,7 @@ def do_transformation(year): #dataframe = pd.concat([df_np, df_p]) for name, group in dataframe.groupby('GNFR_Sector'): - print 'gnfr', name + print('gnfr', name) pollutant_list = create_pollutant_empty_list(in_file, len(c_lats), len(c_lons)) # Other mobile sources ignoring sea cells (shipping emissions) @@ -178,7 +178,7 @@ def do_transformation(year): group = group.groupby(['row_lat', 'col_lon']).sum().reset_index() - for i in xrange(len(pollutant_list)): + for i in range(len(pollutant_list)): # pollutant_list[i]['data'][group.col_lon, group.row_lat] = group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'][group.row_lat, group.col_lon] += group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'] = pollutant_list[i]['data'].reshape((1,) + pollutant_list[i]['data'].shape) @@ -392,8 +392,8 @@ def check_vocs(year): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - gnfr, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum) + print('{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + gnfr, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum)) return True def do_pm_transformation(year): @@ -471,8 +471,8 @@ def check_pm(year): [new_pm] = extract_vars(pm_path, [pm]) pm_sum += new_pm['data'].sum() - print '{0} PM2.5 sum: {1}; PM sum: {2}; %diff: {3}'.format( - gnfr, pm25_sum, pm_sum, 100*(pm25_sum - pm_sum) / pm25_sum) + print('{0} PM2.5 sum: {1}; PM sum: {2}; %diff: {3}'.format( + gnfr, pm25_sum, pm_sum, 100*(pm25_sum - pm_sum) / pm25_sum)) return True if __name__ == '__main__': diff --git a/preproc/ceds_preproc.py b/preproc/ceds_preproc.py index 5a5385b..7a3ee0b 100755 --- a/preproc/ceds_preproc.py +++ b/preproc/ceds_preproc.py @@ -248,7 +248,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', @@ -301,10 +301,10 @@ def do_air_transformation(year): elif sector == 'air_crs': data_aux = data[:, 15:24 + 1, :, :].sum(axis=1) else: - print 'ERROR' + print('ERROR') sys.exit(1) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/eclipsev5a_preproc.py b/preproc/eclipsev5a_preproc.py index 9dd7658..55c3b8b 100755 --- a/preproc/eclipsev5a_preproc.py +++ b/preproc/eclipsev5a_preproc.py @@ -99,7 +99,7 @@ def create_bounds(coordinates, number_vertices=2): def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cell_area, date): # TODO Documentation - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -236,9 +236,9 @@ def get_output_name(pollutant, sector, year, month): def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area): # TODO Docuemtnation - for i in xrange(len(LIST_YEARS)): + for i in range(len(LIST_YEARS)): - for month in xrange(12): + for month in range(12): # print i, list_years[i], month + 1 if pollutant == 'NOx': pollutant_name = 'nox_no2' @@ -270,7 +270,7 @@ def do_transformation(): # TODO Documentation for pollutant in LIST_POLLUTANTS: file_name = os.path.join(INPUT_PATH, INPUT_NAME.replace('', pollutant)) - print file_name + print(file_name) nc = Dataset(file_name, mode='r') c_lats = nc.variables['lat'][:] c_lons = nc.variables['lon'][:] @@ -325,7 +325,7 @@ def do_flaring_transformation(): if var_name is not None: data = nc_in.variables[var][:] data = np.nan_to_num(data) - for i in xrange(len(LIST_YEARS)): + for i in range(len(LIST_YEARS)): output_name = get_flaring_output_name(var_name, 'flaring', LIST_YEARS[i]) data_aux = data[i, :, :] data_aux = (data_aux * YEAR_FACTOR) / cell_area diff --git a/preproc/edgarv432_ap_preproc.py b/preproc/edgarv432_ap_preproc.py index 67ee72b..6da9f0a 100755 --- a/preproc/edgarv432_ap_preproc.py +++ b/preproc/edgarv432_ap_preproc.py @@ -146,7 +146,7 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr month=None): # TODO Documentation # Creating NetCDF & Dimensions - print output_name_path + print(output_name_path) nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) nc_output.createDimension('lon', center_lons.shape[0]) @@ -233,7 +233,7 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr def do_yearly_transformation(year): # TODO Documentation for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): + for ipcc in list(ipcc_to_sector_dict().keys()): file_path = os.path.join( INPUT_PATH, YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', @@ -241,7 +241,7 @@ def do_yearly_transformation(year): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') if pollutant in ['PM2.5_bio', 'PM2.5_fossil']: @@ -285,7 +285,7 @@ def do_yearly_transformation(year): def do_monthly_transformation(year): # TODO Documentation for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): + for ipcc in list(ipcc_to_sector_dict().keys()): file_path = os.path.join( INPUT_PATH, YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', @@ -293,7 +293,7 @@ def do_monthly_transformation(year): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') if pollutant in ['PM2.5_bio', 'PM2.5_fossil']: @@ -328,7 +328,7 @@ def do_monthly_transformation(year): nc_month_factors = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE.replace('', sector))) month_factors = nc_month_factors.variables[sector][:] - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): data_aux = data * month_factors[month - 1, :, :] write_netcdf(os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant.lower(), year, str(month).zfill(2))), @@ -344,8 +344,8 @@ def do_monthly_transformation(year): def do_2010_monthly_transformation(): # TODO Documentation for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): - for month in xrange(1, 12 + 1, 1): + for ipcc in list(ipcc_to_sector_dict().keys()): + for month in range(1, 12 + 1, 1): file_path = os.path.join( INPUT_PATH, MONTHLY_INPUT_NAME.replace('', pollutant).replace('', @@ -353,7 +353,7 @@ def do_2010_monthly_transformation(): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') # print pollutant # print pollutant in ['PM2.5_bio', 'PM2.5_fossil'] diff --git a/preproc/edgarv432_voc_preproc.py b/preproc/edgarv432_voc_preproc.py index 091ce93..a300c80 100755 --- a/preproc/edgarv432_voc_preproc.py +++ b/preproc/edgarv432_voc_preproc.py @@ -138,7 +138,7 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr month=None): # TODO Documentation # Creating NetCDF & Dimensions - print output_name_path + print(output_name_path) nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) nc_output.createDimension('lon', center_lons.shape[0]) @@ -227,9 +227,9 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr def do_yearly_transformation(year): # TODO Documentation - print year + print(year) for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): + for ipcc in list(ipcc_to_sector_dict().keys()): file_path = os.path.join( INPUT_PATH, YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', @@ -237,7 +237,7 @@ def do_yearly_transformation(year): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') data = nc_in.variables['emi_{0}'.format(pollutant.lower())][:] @@ -250,7 +250,7 @@ def do_yearly_transformation(year): nc_in.close() sector = ipcc_to_sector_dict()[ipcc] - if pollutant in ['voc{0}'.format(x) for x in xrange(1, 9 + 1, 1)]: + if pollutant in ['voc{0}'.format(x) for x in range(1, 9 + 1, 1)]: pollutant_aux = pollutant.replace('voc', 'voc0') else: pollutant_aux = pollutant @@ -274,9 +274,9 @@ def do_yearly_transformation(year): def do_monthly_transformation(year): # TODO Documentation - print year + print(year) for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): + for ipcc in list(ipcc_to_sector_dict().keys()): file_path = os.path.join( INPUT_PATH, YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', @@ -284,7 +284,7 @@ def do_monthly_transformation(year): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') data = nc_in.variables['emi_{0}'.format(pollutant.lower())][:] @@ -298,7 +298,7 @@ def do_monthly_transformation(year): sector = ipcc_to_sector_dict()[ipcc] - if pollutant in ['voc{0}'.format(x) for x in xrange(1, 9 + 1, 1)]: + if pollutant in ['voc{0}'.format(x) for x in range(1, 9 + 1, 1)]: pollutant_aux = pollutant.replace('voc', 'voc0') else: pollutant_aux = pollutant @@ -314,7 +314,7 @@ def do_monthly_transformation(year): nc_month_factors = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE.replace('', sector))) month_factors = nc_month_factors.variables[sector][:] - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): data_aux = data * month_factors[month - 1, :, :] write_netcdf(os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format( pollutant_aux.lower(), year, str(month).zfill(2))), @@ -330,8 +330,8 @@ def do_monthly_transformation(year): def do_2010_monthly_transformation(): # TODO Documentation for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): - for month in xrange(1, 12 + 1, 1): + for ipcc in list(ipcc_to_sector_dict().keys()): + for month in range(1, 12 + 1, 1): file_path = os.path.join( INPUT_PATH, MONTHLY_INPUT_NAME.replace('', pollutant).replace('', @@ -339,7 +339,7 @@ def do_2010_monthly_transformation(): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') data = nc_in.variables['emi_{0}'.format(pollutant.lower())][:] @@ -353,7 +353,7 @@ def do_2010_monthly_transformation(): sector = ipcc_to_sector_dict()[ipcc] - if pollutant in ['voc{0}'.format(x) for x in xrange(1, 9 + 1, 1)]: + if pollutant in ['voc{0}'.format(x) for x in range(1, 9 + 1, 1)]: pollutant_aux = pollutant.replace('voc', 'voc0') else: pollutant_aux = pollutant diff --git a/preproc/emep_preproc.py b/preproc/emep_preproc.py index 15f44b0..72c2ff3 100755 --- a/preproc/emep_preproc.py +++ b/preproc/emep_preproc.py @@ -65,14 +65,14 @@ def calculate_grid_definition(in_path): # Longitudes lons = np.sort(np.unique(df.LONGITUDE)) lons_interval = lons[1:] - lons[:-1] - print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - df.LONGITUDE.min(), df.LONGITUDE.max(), lons_interval.min(), len(lons)) + print('Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( + df.LONGITUDE.min(), df.LONGITUDE.max(), lons_interval.min(), len(lons))) # Latitudes lats = np.sort(np.unique(df.LATITUDE)) lats_interval = lats[1:] - lats[:-1] - print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - df.LATITUDE.min(), df.LATITUDE.max(), lats_interval.min(), len(lats)) + print('Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( + df.LATITUDE.min(), df.LATITUDE.max(), lats_interval.min(), len(lats))) lats = np.arange(-90 + lats_interval.min()/2, 90, lats_interval.min(), dtype=np.float64) lons = np.arange(-180 + lons_interval.min()/2, 180, lons_interval.min(), dtype=np.float64) @@ -96,7 +96,7 @@ def do_transformation(year): INPUT_NAME.replace('', str(year)).replace('', sector).replace('', pollutant)) if os.path.exists(in_file): - print in_file + print(in_file) c_lats, c_lons, lat_interval, lon_interval = calculate_grid_definition(in_file) b_lats = create_bounds(c_lats, number_vertices=2) b_lons = create_bounds(c_lons, number_vertices=2) diff --git a/preproc/gfas12_h_preproc.py b/preproc/gfas12_h_preproc.py index 9cc93fe..e38d1d3 100755 --- a/preproc/gfas12_h_preproc.py +++ b/preproc/gfas12_h_preproc.py @@ -37,8 +37,8 @@ INPUT_PATH = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' INPUT_NAME = 'gfas_hourly_.grb' OUTPUT_PATH = '/esarchive/recon/ecmwf/gfas' -STARTING_DATE = datetime(year=2018, month=11, day=01) -ENDIND_DATE = datetime(year=2018, month=11, day=01) +STARTING_DATE = datetime(year=2018, month=11, day=0o1) +ENDIND_DATE = datetime(year=2018, month=11, day=0o1) PARAMETERS_FILE = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_hourly_Parameters.csv' # ============================================================== @@ -118,7 +118,7 @@ def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cel :type date: datetime.datetime """ - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -277,6 +277,6 @@ if __name__ == '__main__': if os.path.isfile(f): do_transformation(f, date_aux, OUTPUT_PATH, var_list) else: - print 'ERROR: file {0} not found'.format(f) + print('ERROR: file {0} not found'.format(f)) date_aux = date_aux + timedelta(days=1) diff --git a/preproc/gfas12_preproc.py b/preproc/gfas12_preproc.py index 328cbc0..86ffcf9 100755 --- a/preproc/gfas12_preproc.py +++ b/preproc/gfas12_preproc.py @@ -118,7 +118,7 @@ def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cel :type date: datetime.datetime """ - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -276,6 +276,6 @@ if __name__ == '__main__': if os.path.isfile(f): do_transformation(f, date_aux, OUTPUT_PATH, var_list) else: - print 'ERROR: file {0} not found'.format(f) + print('ERROR: file {0} not found'.format(f)) date_aux = date_aux + timedelta(days=1) diff --git a/preproc/htapv2_preproc.py b/preproc/htapv2_preproc.py index a81aebe..284fd6d 100755 --- a/preproc/htapv2_preproc.py +++ b/preproc/htapv2_preproc.py @@ -73,7 +73,7 @@ def do_transformation_annual(filename, out_path, pollutant, sector, year): """ from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds - print filename + print(filename) [c_lats, c_lons] = extract_vars(filename, ['lat', 'lon']) if pollutant == 'pm25': @@ -108,7 +108,7 @@ def do_transformation_annual(filename, out_path, pollutant, sector, year): os.makedirs(out_path) out_path = os.path.join(out_path, '{0}_{1}.nc'.format(pollutant, year)) - print out_path + print(out_path) write_netcdf(out_path, c_lats['data'], c_lons['data'], [data], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), cell_area=get_grid_area(filename), global_attributes=global_attributes,) @@ -139,8 +139,8 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds - for month in xrange(1, 13): - print filename_list[month - 1] + for month in range(1, 13): + print(filename_list[month - 1]) [c_lats, c_lons] = extract_vars(filename_list[month - 1], ['lat', 'lon']) if pollutant == 'pm25': @@ -239,7 +239,7 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): nmvoc_ratio_list = do_ratio_list() - print sector + print(sector) if sector == 'ENERGY': ratio_var = 'pow' @@ -263,16 +263,16 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): nmvoc_ratio_list.pop('voc24', None) nmvoc_ratio_list.pop('voc25', None) - print type(nmvoc_ratio_list), nmvoc_ratio_list + print(type(nmvoc_ratio_list), nmvoc_ratio_list) - for month in xrange(1, 13): - print filename_list[month - 1] + for month in range(1, 13): + print(filename_list[month - 1]) c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) [data] = extract_vars(filename_list[month - 1], ['emi_nmvoc']) - for voc, ratio_file in nmvoc_ratio_list.iteritems(): - print voc, ratio_file + for voc, ratio_file in nmvoc_ratio_list.items(): + print(voc, ratio_file) pollutant = voc [ratio] = extract_vars(ratio_file, [ratio_var]) @@ -303,7 +303,7 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): os.makedirs(out_path_aux) out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant, year, str(month).zfill(2))) - print out_path_aux + print(out_path_aux) write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) @@ -325,18 +325,18 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year nmvoc_ratio_list = do_ratio_list() - print sector + print(sector) - print type(nmvoc_ratio_list), nmvoc_ratio_list + print(type(nmvoc_ratio_list), nmvoc_ratio_list) - for month in xrange(1, 13): - print filename_list[month - 1] + for month in range(1, 13): + print(filename_list[month - 1]) c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) [ind, exf, sol] = extract_vars(filename_list[month - 1], ['emiss_ind', 'emiss_exf', 'emiss_sol']) - for voc, ratio_file in nmvoc_ratio_list.iteritems(): - print voc, ratio_file + for voc, ratio_file in nmvoc_ratio_list.items(): + print(voc, ratio_file) data = { 'name': voc, 'units': 'kg m-2 s-1', @@ -382,7 +382,7 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year os.makedirs(out_path_aux) out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) - print out_path_aux + print(out_path_aux) write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) @@ -439,7 +439,7 @@ def do_nmvoc_year_transformation(filename, out_path, sector, year): os.makedirs(out_path_aux) out_path_aux = os.path.join(out_path_aux, '{0}_{1}.nc'.format(pollutant, year)) - print out_path_aux + print(out_path_aux) write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), @@ -510,7 +510,7 @@ def check_vocs(year): :return: """ from hermesv3_gr.tools.netcdf_tools import extract_vars - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): for snap in ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT']: nmvoc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', 'nmvoc_{0}'.format(snap.lower()), 'nmvoc_{0}{1}.nc'.format(year, str(month).zfill(2))) @@ -518,26 +518,26 @@ def check_vocs(year): nmvoc_sum = new_voc['data'].sum() voc_sum = 0 - for voc in ['voc{0}'.format(str(x).zfill(2)) for x in xrange(1, 25 + 1, 1)]: + for voc in ['voc{0}'.format(str(x).zfill(2)) for x in range(1, 25 + 1, 1)]: voc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(voc, snap.lower()), '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) if os.path.exists(voc_path): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum) / nmvoc_sum, month) + print('{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum) / nmvoc_sum, month)) if __name__ == '__main__': for y in LIST_YEARS: - for pollutant_dict in get_pollutant_dict().iteritems(): + for pollutant_dict in get_pollutant_dict().items(): for current_sector in get_sector_dict()[pollutant_dict[0]]['month']: input_name_aux = INPUT_NAME.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) input_name_aux = input_name_aux.replace('', pollutant_dict[1]) file_list = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) - for aux_month in xrange(1, 13)] + for aux_month in range(1, 13)] do_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), pollutant_dict[0], current_sector, y) @@ -564,7 +564,7 @@ if __name__ == '__main__': input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) file_list = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) - for aux_month in xrange(1, 13)] + for aux_month in range(1, 13)] if current_sector == 'INDUSTRY_3subsectors': do_nmvoc_industry_month_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), @@ -580,5 +580,5 @@ if __name__ == '__main__': input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) input_name_aux = os.path.join(INPUT_PATH, input_name_aux) - print input_name_aux + print(input_name_aux) do_nmvoc_year_transformation(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean'), current_sector, y) diff --git a/preproc/tno_mac_iii_preproc.py b/preproc/tno_mac_iii_preproc.py index e96da9f..6c06388 100755 --- a/preproc/tno_mac_iii_preproc.py +++ b/preproc/tno_mac_iii_preproc.py @@ -73,14 +73,14 @@ def calculate_grid_definition(in_path): # Longitudes lons = np.sort(np.unique(dataframe.Lon)) lons_interval = lons[1:] - lons[:-1] - print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - dataframe.Lon.min(), dataframe.Lon.max(), lons_interval.min(), len(lons)) + print('Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( + dataframe.Lon.min(), dataframe.Lon.max(), lons_interval.min(), len(lons))) # Latitudes lats = np.sort(np.unique(dataframe.Lat)) lats_interval = lats[1:] - lats[:-1] - print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - dataframe.Lat.min(), dataframe.Lat.max(), lats_interval.min(), len(lats)) + print('Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( + dataframe.Lat.min(), dataframe.Lat.max(), lats_interval.min(), len(lats))) lats = np.arange(-90 + lats_interval.min() / 2, 90, lats_interval.min(), dtype=np.float64) lons = np.arange(-180 + lons_interval.min() / 2, 180, lons_interval.min(), dtype=np.float64) @@ -164,7 +164,7 @@ def do_transformation(year): dataframe = pd.concat([df_np, df_p]) for name, group in dataframe.groupby('SNAP'): - print 'snap', name + print('snap', name) pollutant_list = create_pollutant_empty_list(in_file, len(c_lats), len(c_lons)) # Other mobile sources ignoring sea cells (shipping emissions) @@ -174,7 +174,7 @@ def do_transformation(year): group = group.groupby(['row_lat', 'col_lon']).sum().reset_index() - for i in xrange(len(pollutant_list)): + for i in range(len(pollutant_list)): # pollutant_list[i]['data'][group.col_lon, group.row_lat] = group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'][group.row_lat, group.col_lon] += group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'] = pollutant_list[i]['data'].reshape((1,) + pollutant_list[i]['data'].shape) @@ -362,8 +362,8 @@ def check_vocs(year): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum) + print('{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum)) return True diff --git a/preproc/tno_mac_iii_preproc_voc_ratios.py b/preproc/tno_mac_iii_preproc_voc_ratios.py index 9aea926..80b41fe 100755 --- a/preproc/tno_mac_iii_preproc_voc_ratios.py +++ b/preproc/tno_mac_iii_preproc_voc_ratios.py @@ -101,7 +101,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, if not (regular_latlon or lcc or rotated): regular_latlon = True - print netcdf_path + print(netcdf_path) netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== @@ -116,7 +116,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat',) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -127,21 +127,21 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat',) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -215,7 +215,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lons.axis = "X" lons.long_name = "longitude coordinate" lons.standard_name = "longitude" - print 'lons:', lons[:].shape, center_longitudes.shape + print('lons:', lons[:].shape, center_longitudes.shape) lons[:] = center_longitudes if boundary_longitudes is not None: lons.bounds = "lon_bnds" @@ -287,7 +287,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, try: var[:] = variable['data'] except Exception: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -388,19 +388,19 @@ def create_voc_ratio(voc): import numpy as np [country_values, lat, lon] = extract_vars(TNO_WORLD_MASK, ['timezone_id', 'lat', 'lon']) country_values = country_values['data'].reshape((country_values['data'].shape[1], country_values['data'].shape[1])) - print OUTPUT_PATH + print(OUTPUT_PATH) if not os.path.exists(OUTPUT_PATH): os.makedirs(OUTPUT_PATH) complete_output_path = os.path.join(OUTPUT_PATH, 'ratio_{0}.nc'.format(voc)) if not os.path.exists(complete_output_path): - print 'Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path) + print('Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path)) data_list = [] for snap in get_sector_list(voc): - print snap + print(snap) mask_factor = np.zeros(country_values.shape) iso_codes = get_iso_codes() - for country_code, factor in get_country_code_and_factor(voc, snap).iteritems(): + for country_code, factor in get_country_code_and_factor(voc, snap).items(): try: mask_factor[country_values == iso_codes[country_code]] = factor except Exception: @@ -415,7 +415,7 @@ def create_voc_ratio(voc): }) write_netcdf(complete_output_path, lat['data'], lon['data'], data_list) else: - print 'Ratio file for {0} already created\npath: {1}'.format(voc, complete_output_path) + print('Ratio file for {0} already created\npath: {1}'.format(voc, complete_output_path)) return True @@ -468,7 +468,7 @@ def get_voc_list(): del df['ISO3'], df['snap'], df['output substance name'], df['fr'] df = df.drop_duplicates().dropna() voc_list = df.vcode.values - for i in xrange(len(voc_list)): + for i in range(len(voc_list)): voc_list[i] = voc_list[i].replace('v', 'voc') return df.vcode.values diff --git a/preproc/wiedinmyer_preproc.py b/preproc/wiedinmyer_preproc.py index 7ccdab7..fd0344b 100755 --- a/preproc/wiedinmyer_preproc.py +++ b/preproc/wiedinmyer_preproc.py @@ -81,7 +81,7 @@ def do_transformation(filename): :type filename: str """ import numpy as np - print filename + print(filename) from hermesv3_gr.tools.netcdf_tools import get_grid_area from cf_units import Unit @@ -101,10 +101,10 @@ def do_transformation(filename): for output_pollutant in LIST_POLLUTANTS: input_pollutant = out_pollutant_to_in_pollutant(output_pollutant) try: - print input_pollutant + print(input_pollutant) data = nc_in.variables[input_pollutant][:] except RuntimeWarning: - print 'ERROR reading {0}'.format(input_pollutant) + print('ERROR reading {0}'.format(input_pollutant)) data = np.nan_to_num(data) data = data/grid_area # To pass from Gg/year to Gg/m2.year data = data*factor @@ -119,7 +119,7 @@ def do_transformation(filename): if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) write_netcdf(os.path.join(out_path_aux, '{0}_{1}.nc'.format(output_pollutant, YEAR)), - data, data_attributes, lats, lons, grid_area, YEAR, 01) + data, data_attributes, lats, lons, grid_area, YEAR, 0o1) nc_in.close() @@ -157,7 +157,7 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr """ from hermesv3_gr.tools.coordinates_tools import create_bounds - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -243,4 +243,4 @@ if __name__ == '__main__': do_transformation(os.path.join(INPUT_PATH, INPUT_NAME)) - print 'Time(s):', timeit.default_timer() - starting_time + print('Time(s):', timeit.default_timer() - starting_time) -- GitLab From b19d04993d6e521c67ec24e260e2665d4edb4362 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 21 Oct 2019 12:51:28 +0200 Subject: [PATCH 02/28] Changed config.py replaces --- hermesv3_gr/config/config.py | 58 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index 4a73416..f8d1d4a 100755 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -32,7 +32,7 @@ class Config(ArgParser): def read_options(self): """ - Reads all the options from command line or from the configuration file. + Reads all the arguments from command line or from the configuration file. The value of an argument given by command line has high priority that the one that appear in the configuration file. @@ -76,17 +76,17 @@ class Config(ArgParser): p.add_argument('--vertical_description', required=True, help='Path to the file that contains the vertical description of the desired output.') - # Global options + # Global arguments p.add_argument('--inc_lat', required=False, help='Latitude resolution for a global domain.', type=float) p.add_argument('--inc_lon', required=False, help='Longitude resolution for a global domain.', type=float) - # Regular lat-lon options: + # Regular lat-lon arguments: p.add_argument('--lat_orig', required=False, help='Latitude of the corner of the first cell.', type=float) p.add_argument('--lon_orig', required=False, help='Longitude of the corner of the first cell.', type=float) p.add_argument('--n_lat', required=False, help='Number of latitude elements.', type=float) p.add_argument('--n_lon', required=False, help='Number of longitude elements.', type=float) - # Rotated options + # Rotated arguments p.add_argument('--centre_lat', required=False, help='Central geographic latitude of grid (non-rotated degrees). Corresponds to the TPH0D ' + 'parameter in NMMB-MONARCH.', type=float) @@ -106,7 +106,7 @@ class Config(ArgParser): help='Longitudinal grid resolution (rotated degrees). Corresponds to the DLMD parameter ' + 'in NMMB-MONARCH.', type=float) - # Lambert conformal conic options + # Lambert conformal conic arguments p.add_argument('--lat_1', required=False, help='Standard parallel 1 (in deg). Corresponds to the P_ALP parameter of the GRIDDESC file.', type=float) @@ -157,30 +157,30 @@ class Config(ArgParser): p.add_argument('--world_info', required=True, help='Path to the file that contains the world information like timezones, ISO codes, ...') - options = p.parse_args() - for item in vars(options): - is_str = False - exec ("is_str = str == type(options.{0})".format(item)) + arguments = p.parse_args() + for item in vars(arguments): + is_str = isinstance(arguments.__dict__[item], str) if is_str: - exec("options.{0} = options.{0}.replace('', options.input_dir)".format(item)) - exec("options.{0} = options.{0}.replace('', options.domain_type)".format(item)) - if options.domain_type == 'global' or options.domain_type == 'regular': - exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( - item, options.inc_lat, options.inc_lon)) - elif options.domain_type == 'rotated': - exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( - item, options.inc_rlat, options.inc_rlon)) - elif options.domain_type == 'lcc' or options.domain_type == 'mercator': - exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( - item, options.inc_x, options.inc_y)) - - options.start_date = self._parse_start_date(options.start_date) - options.end_date = self._parse_end_date(options.end_date, options.start_date) - - self.create_dir(options.output_dir) - self.create_dir(options.auxiliar_files_path) - - return options + arguments.__dict__[item] = arguments.__dict__[item].replace('', arguments.data_path) + arguments.__dict__[item] = arguments.__dict__[item].replace('', arguments.input_dir) + arguments.__dict__[item] = arguments.__dict__[item].replace('', arguments.domain_type) + if arguments.domain_type in ['global', 'regular']: + arguments.__dict__[item] = arguments.__dict__[item].replace('', '{1}_{2}'.format( + item, arguments.inc_lat, arguments.inc_lon)) + elif arguments.domain_type == 'rotated': + arguments.__dict__[item] = arguments.__dict__[item].replace('', '{1}_{2}'.format( + item, arguments.inc_rlat, arguments.inc_rlon)) + elif arguments.domain_type == 'lcc' or arguments.domain_type == 'mercator': + arguments.__dict__[item] = arguments.__dict__[item].replace('', '{1}_{2}'.format( + item, arguments.inc_x, arguments.inc_y)) + + arguments.start_date = self._parse_start_date(arguments.start_date) + arguments.end_date = self._parse_end_date(arguments.end_date, arguments.start_date) + + self.create_dir(arguments.output_dir) + self.create_dir(arguments.auxiliar_files_path) + + return arguments def get_output_name(self, date): """ @@ -277,7 +277,7 @@ class Config(ArgParser): date = datetime.strptime(str_date, date_format) break except ValueError as e: - if e.message == 'day is out of range for month': + if str(e) == 'day is out of range for month': raise ValueError(e) if date is None: -- GitLab From 165568eb3160f7113c3fc93cbc3fda7f06f5b5b9 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 22 Oct 2019 10:25:26 +0200 Subject: [PATCH 03/28] Updated timezone calculation --- .../temporal/tz_world_country_iso3166.csv | 15 ++++++++++++++- hermesv3_gr/modules/temporal/temporal.py | 16 ++++++++-------- hermesv3_gr/modules/writing/writer_wrf_chem.py | 12 +++++------- hermesv3_gr/tools/netcdf_tools.py | 4 ++-- setup.py | 2 +- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/data/profiles/temporal/tz_world_country_iso3166.csv b/data/profiles/temporal/tz_world_country_iso3166.csv index 4c82180..b6ddfd7 100755 --- a/data/profiles/temporal/tz_world_country_iso3166.csv +++ b/data/profiles/temporal/tz_world_country_iso3166.csv @@ -420,4 +420,17 @@ "Samoa";882;"Pacific/Apia";378;"WSM" "Yemen";887;"Asia/Aden";203;"YEM" "Zambia";894;"Africa/Lusaka";36;"ZMB" -"Namibia";"264";"Africa/Windhoek";52;"NAM" \ No newline at end of file +"Namibia";"264";"Africa/Windhoek";52;"NAM" +"Antarctica";10;"Antarctica/McMurdo";424;"ATA" +"Antarctica";10;"Antarctica/Mawson";425;"ATA" +"Antarctica";10;"Antarctica/Davis";426;"ATA" +"Antarctica";10;"Antarctica/Rothera";427;"ATA" +"Antarctica";10;"Antarctica/Troll";428;"ATA" +"Antarctica";10;"Antarctica/Syowa";429;"ATA" +"Antarctica";10;"Antarctica/Vostok";430;"ATA" +"Antarctica";10;"Antarctica/Casey";431;"ATA" +"Antarctica";10;"Antarctica/DumontDUrville";432;"ATA" +"Antarctica";10;"Antarctica/Palmer";433;"ATA" +"Myanmar";104;"Asia/Yangon";434;"MMR" +"Kazakhstan";398;"Asia/Qostanay";435;"KAZ" +"UTC";0;"Etc/UTC";999;"UTC" \ No newline at end of file diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 4fdc08e..457d0c8 100755 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -688,7 +688,7 @@ class TemporalDistribution(object): nc_in = Dataset(self.netcdf_timezones) timezones = nc_in.variables['timezone_id'][:, self.grid.x_lower_bound:self.grid.x_upper_bound, - self.grid.y_lower_bound:self.grid.y_upper_bound].astype(int) + self.grid.y_lower_bound:self.grid.y_upper_bound].astype(int) nc_in.close() tz_list = np.chararray(timezones.shape, itemsize=32) @@ -698,6 +698,7 @@ class TemporalDistribution(object): tz_list[timezones == id_aux] = timezone except IndexError: pass + settings.write_time('TemporalDistribution', 'calculate_timezones', timeit.default_timer() - st_time, level=3) return tz_list @@ -706,8 +707,8 @@ class TemporalDistribution(object): """ Calculate the temporal factor to correct the input data of the given date for each cell. - :param date: Date of the current timestep. - :type date: datetime.datetime + :param date_aux: Date of the current timestep. + :type date_aux: datetime.datetime :return: 2D array with the factors to correct the input data to the date of this timestep. :rtype: numpy.array @@ -720,14 +721,13 @@ class TemporalDistribution(object): df = pd.DataFrame(self.timezones_array.flatten(), columns=['tz']) - df['utc'] = pd.to_datetime(date_aux) + df['local'] = pd.to_datetime(date_aux, utc=True) try: - df['local'] = df.groupby('tz')['utc'].apply( - lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert(x.name).dt.tz_localize(None)) + df['local'] = df.groupby('tz')['local'].apply( + lambda x: x.dt.tz_convert(x.name.decode("utf-8")).dt.tz_localize(None)) except pytz.exceptions.UnknownTimeZoneError: df['local'] = df.groupby('tz')['utc'].apply( - lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert( - self.parse_tz(x.name)).dt.tz_localize(None)) + lambda x: x.dt.tz_convert(self.parse_tz(x.name.decode("utf-8"))).dt.tz_localize(None)) # ===== HOURLY PROFILES ===== df['weekday'] = df['local'].dt.weekday diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index 407e623..5f27d79 100755 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -299,16 +299,13 @@ class WriterWrfChem(Writer): :return: """ from datetime import timedelta - import netCDF4 - aux_times_list = [] + aux_times = np.chararray((len(self.hours), 19), itemsize=1) - for hour in self.hours: + for i, hour in enumerate(self.hours): aux_date = self.date + timedelta(hours=hour) - aux_times_list.append(aux_date.strftime("%Y-%m-%d_%H:%M:%S")) - - str_out = netCDF4.stringtochar(np.array(aux_times_list)) - return str_out + aux_times[i] = list(aux_date.strftime("%Y-%m-%d_%H:%M:%S")) + return aux_times def create_parallel_netcdf(self): # TODO Documentation @@ -402,6 +399,7 @@ class WriterWrfChem(Writer): # ===== Variables ===== settings.write_log("\t\tCreating NetCDF variables.", level=2) times = netcdf.createVariable('Times', 'S1', ('Time', 'DateStrLen', )) + print(times[:].shape, self.create_times_var().shape) times[:] = self.create_times_var() settings.write_log("\t\t\t'Times' variable created with size: {0}".format(times[:].shape), level=3) diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index a56842a..48869a1 100755 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -167,7 +167,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lat_dim = ('lat',) elif len(center_latitudes.shape) == 2: netcdf.createDimension('lat', center_latitudes.shape[0]) - lat_dim = ('lon', 'lat', ) + lat_dim = ('lat', 'lon', ) else: print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) @@ -178,7 +178,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lon_dim = ('lon',) elif len(center_longitudes.shape) == 2: netcdf.createDimension('lon', center_longitudes.shape[1]) - lon_dim = ('lon', 'lat', ) + lon_dim = ('lat', 'lon', ) else: print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) diff --git a/setup.py b/setup.py index 34e4e16..2050aed 100755 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ setup( 'ESMPy>=7.1.0.dev0', 'holidays', 'pytz', - 'timezonefinder<4.0.0', + 'timezonefinder>=4.0.0', 'mpi4py', 'pytest', ], -- GitLab From 028bc7f5fa8887b91a7294085dad486774a9cc61 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 22 Oct 2019 13:32:13 +0200 Subject: [PATCH 04/28] Added custom COMM Added first time Added erase auxiliary files --- conf/hermes.conf | 2 ++ hermesv3_gr/config/config.py | 33 ++++++++++++++++++------ hermesv3_gr/config/settings.py | 6 +++-- hermesv3_gr/hermes.py | 4 +-- hermesv3_gr/modules/temporal/temporal.py | 2 +- hermesv3_gr/tools/netcdf_tools.py | 5 ---- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index a9d1393..8797122 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -11,6 +11,8 @@ start_date = 2017/01/01 00:00:00 output_timestep_type = hourly output_timestep_num = 24 output_timestep_freq = 1 +first_time = 1 +erase_auxiliary_files = 1 [DOMAIN] diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index f8d1d4a..9199455 100755 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -20,13 +20,20 @@ from configargparse import ArgParser import sys +import os +from shutil import rmtree +from mpi4py import MPI class Config(ArgParser): """ Initialization of the arguments that the parser can handle. """ - def __init__(self): + def __init__(self, comm=None): + if comm is None: + comm = MPI.COMM_WORLD + self.comm = comm + super(Config, self).__init__() self.options = self.read_options() @@ -61,6 +68,10 @@ class Config(ArgParser): type=str, choices=['hourly', 'daily', 'monthly', 'yearly']) p.add_argument('--output_timestep_num', required=True, help='Number of timesteps to simulate.', type=int) p.add_argument('--output_timestep_freq', required=True, help='Frequency between timesteps.', type=int) + p.add_argument('--first_time', required=False, default='False', type=str, + help='Indicates if you want to run it for first time (only create auxiliary files).') + p.add_argument('--erase_auxiliary_files', required=False, default='False', type=str, + help='Indicates if you want to start from scratch removing the auxiliary files already created.') p.add_argument('--output_model', required=True, help='Name of the output model.', choices=['MONARCH', 'CMAQ', 'WRF_CHEM']) @@ -177,8 +188,16 @@ class Config(ArgParser): arguments.start_date = self._parse_start_date(arguments.start_date) arguments.end_date = self._parse_end_date(arguments.end_date, arguments.start_date) + arguments.first_time = self._parse_bool(arguments.first_time) + arguments.erase_auxiliary_files = self._parse_bool(arguments.erase_auxiliary_files) + self.create_dir(arguments.output_dir) - self.create_dir(arguments.auxiliar_files_path) + if arguments.erase_auxiliary_files: + if os.path.exists(arguments.auxiliary_files_path): + if self.comm.Get_rank() == 0: + rmtree(arguments.auxiliary_files_path) + self.comm.Barrier() + self.create_dir(arguments.auxiliary_files_path) return arguments @@ -207,8 +226,7 @@ class Config(ArgParser): full_path = os.path.join(self.options.output_dir, file_name) return full_path - @staticmethod - def create_dir(path): + def create_dir(self, path): """ Create the given folder if it is not created yet. @@ -217,8 +235,7 @@ class Config(ArgParser): """ import os from mpi4py import MPI - icomm = MPI.COMM_WORLD - comm = icomm.Split(color=0, key=0) + comm = self.comm.Split(color=0, key=0) rank = comm.Get_rank() if rank == 0: @@ -304,12 +321,12 @@ class Config(ArgParser): else: return self._parse_start_date(end_date) - def set_log_level(self): + def set_log_level(self, comm): """ Defines the log_level using the common script settings. """ from . import settings - settings.define_global_vars(self.options.log_level) + settings.define_global_vars(self.options.log_level, comm) if __name__ == '__main__': diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index 18b6a6a..46b10a6 100755 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -46,7 +46,7 @@ global log_file global df_times -def define_global_vars(in_log_level): +def define_global_vars(in_log_level, icomm): # TODO Documentation from mpi4py import MPI @@ -55,7 +55,9 @@ def define_global_vars(in_log_level): global rank global size - icomm = MPI.COMM_WORLD + if icomm is None: + icomm = MPI.COMM_WORLD + comm = icomm.Split(color=0, key=0) rank = comm.Get_rank() size = comm.Get_size() diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 52d46cb..00f6977 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -34,7 +34,7 @@ class Hermes(object): """ Interface class for HERMESv3. """ - def __init__(self, config, new_date=None): + def __init__(self, config, new_date=None, comm=None): from hermesv3_gr.modules.grids.grid import Grid from hermesv3_gr.modules.temporal.temporal import TemporalDistribution from hermesv3_gr.modules.writing.writer import Writer @@ -48,7 +48,7 @@ class Hermes(object): if new_date is not None: self.options.start_date = new_date - config.set_log_level() + config.set_log_level(comm) settings.define_log_file(self.options.output_dir, self.options.start_date) settings.define_times_file() diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 457d0c8..5606513 100755 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -688,7 +688,7 @@ class TemporalDistribution(object): nc_in = Dataset(self.netcdf_timezones) timezones = nc_in.variables['timezone_id'][:, self.grid.x_lower_bound:self.grid.x_upper_bound, - self.grid.y_lower_bound:self.grid.y_upper_bound].astype(int) + self.grid.y_lower_bound:self.grid.y_upper_bound].astype(int) nc_in.close() tz_list = np.chararray(timezones.shape, itemsize=32) diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index 48869a1..1709435 100755 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -23,11 +23,6 @@ from netCDF4 import Dataset from mpi4py import MPI from functools import reduce -ICOMM = MPI.COMM_WORLD -COMM = ICOMM.Split(color=0, key=0) -RANK = COMM.Get_rank() -SIZE = COMM.Get_size() - def open_netcdf(netcdf_path): """ -- GitLab From 8485dd7b5ee097ab6892f5488fe205068a871aac Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 22 Oct 2019 13:37:31 +0200 Subject: [PATCH 05/28] WIP --- hermesv3_gr/hermes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 00f6977..c2a92c3 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -41,6 +41,9 @@ class Hermes(object): global full_time st_time = full_time = timeit.default_timer() + if comm is None: + comm = MPI.COMM_WORLD + self.comm = comm self.config = config self.options = config.options @@ -48,7 +51,7 @@ class Hermes(object): if new_date is not None: self.options.start_date = new_date - config.set_log_level(comm) + config.set_log_level(self.comm) settings.define_log_file(self.options.output_dir, self.options.start_date) settings.define_times_file() @@ -95,6 +98,9 @@ class Hermes(object): """ from datetime import timedelta + if self.options.fisrt_time: + self.comm.Abort(0) + st_time = timeit.default_timer() settings.write_log('') settings.write_log('***** Starting HERMESv3 *****') @@ -139,10 +145,10 @@ class Hermes(object): return None -def run(): +def run(comm=None): date = Hermes(Config()).main() while date is not None: - date = Hermes(Config(), new_date=date).main() + date = Hermes(Config(comm), new_date=date, comm=comm).main() sys.exit(0) -- GitLab From 57abf198c060d962146714aff5c37a594a504ad6 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 22 Oct 2019 13:43:17 +0200 Subject: [PATCH 06/28] auxiliar to auxiliary --- conf/hermes.conf | 4 ++-- hermesv3_gr/config/config.py | 2 +- hermesv3_gr/config/settings.py | 6 ++++-- hermesv3_gr/hermes.py | 2 +- .../modules/emision_inventories/emission_inventory.py | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 8797122..58dae1a 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -12,7 +12,7 @@ output_timestep_type = hourly output_timestep_num = 24 output_timestep_freq = 1 first_time = 1 -erase_auxiliary_files = 1 +erase_auxiliary_files = 0 [DOMAIN] @@ -29,7 +29,7 @@ domain_type = global # domain_type = rotated # domain_type = mercator vertical_description = /data/profiles/vertical/Benchmark_15layers_vertical_description.csv -auxiliar_files_path = /data/auxiliar_files/_ +auxiliary_files_path = /data/auxiliar_files/_ # if domain_type == global: inc_lat = 1. diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index 9199455..7775770 100755 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -80,7 +80,7 @@ class Config(ArgParser): p.add_argument('--domain_type', required=True, help='Type of domain to simulate.', choices=['global', 'lcc', 'rotated', 'mercator', 'regular']) - p.add_argument('--auxiliar_files_path', required=True, + p.add_argument('--auxiliary_files_path', required=True, help='Path to the directory where the necessary auxiliary files will be created if them are ' + 'not created yet.') diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index 46b10a6..038f004 100755 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -46,7 +46,7 @@ global log_file global df_times -def define_global_vars(in_log_level, icomm): +def define_global_vars(in_log_level, mycomm): # TODO Documentation from mpi4py import MPI @@ -55,8 +55,10 @@ def define_global_vars(in_log_level, icomm): global rank global size - if icomm is None: + if mycomm is None: icomm = MPI.COMM_WORLD + else: + icomm = mycomm comm = icomm.Split(color=0, key=0) rank = comm.Get_rank() diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index c2a92c3..8d6ddda 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -68,7 +68,7 @@ class Hermes(object): self.grid = Grid.select_grid( self.options.domain_type, self.options.vertical_description, self.options.output_timestep_num, - self.options.auxiliar_files_path, self.options.inc_lat, self.options.inc_lon, self.options.lat_orig, + self.options.auxiliary_files_path, self.options.inc_lat, self.options.inc_lon, self.options.lat_orig, self.options.lon_orig, self.options.n_lat, self.options.n_lon, self.options.centre_lat, self.options.centre_lon, self.options.west_boundary, self.options.south_boundary, self.options.inc_rlat, self.options.inc_rlon, self.options.lat_1, self.options.lat_2, self.options.lon_0, self.options.lat_0, diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index 43e0db7..e03a3bd 100755 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -103,7 +103,7 @@ class EmissionInventory(object): # It will also create the WoldMasks necessaries self.masking = Masking( options.world_info, factors, regrid_mask, grid, - world_mask_file=os.path.join(os.path.dirname(options.auxiliar_files_path), + world_mask_file=os.path.join(os.path.dirname(options.auxiliary_files_path), '{0}_WorldMask.nc'.format(inventory_name))) self.input_pollutants = pollutants @@ -116,7 +116,7 @@ class EmissionInventory(object): if self.source_type == 'area': self.regrid = ConservativeRegrid( self.pollutant_dicts, - os.path.join(options.auxiliar_files_path, + os.path.join(options.auxiliary_files_path, "Weight_Matrix_{0}_{1}.nc".format(self.inventory_name, settings.size)), grid, masking=self.masking) @@ -136,7 +136,7 @@ class EmissionInventory(object): self.temporal = TemporalDistribution( current_date, options.output_timestep_type, options.output_timestep_num, options.output_timestep_freq, options.p_month, p_month, options.p_week, p_week, options.p_day, p_day, options.p_hour, p_hour, - options.world_info, options.auxiliar_files_path, grid) + options.world_info, options.auxiliary_files_path, grid) else: self.temporal = None settings.write_log('\t\tNone temporal profile set.', level=2) -- GitLab From dfa880540200a0420ab381521166579766b8bd8f Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 22 Oct 2019 13:53:22 +0200 Subject: [PATCH 07/28] WIP --- .codacy.yml | 2 +- .gitlab-ci.yml | 4 ++-- CHANGELOG | 5 +++++ environment.yml | 2 +- hermesv3_gr/hermes.py | 4 +++- run_test.py | 1 - 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.codacy.yml b/.codacy.yml index bf12fc2..0bc0d39 100755 --- a/.codacy.yml +++ b/.codacy.yml @@ -13,7 +13,7 @@ engines: enabled: true pylint: enabled: true - python_version: 2 + python_version: 3 exclude_paths: [ 'doc/**', diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08c1355..d4676c9 100755 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,11 +20,11 @@ prepare: test_python2: stage: test script: - - conda env update -f environment.yml -n hermesv3_gr python=2.7 + - conda env update -f environment.yml -n hermesv3_gr python=3.7 - source activate hermesv3_gr - python run_test.py - pip install codacy-coverage --upgrade - - python-codacy-coverage -r tests/report/python2/coverage.xml + - python-codacy-coverage -r tests/report/python3/coverage.xml #test_python3: # stage: test diff --git a/CHANGELOG b/CHANGELOG index 7d4bbaf..947c8d8 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +2.0.0 + - Python 3 + - first_time option + - erase_auxiliary_files option + 1.0.4 2019/07/19 - Specified version on timezonefinder to be less than 4.0.0 (not further support to python 2.7.X) diff --git a/environment.yml b/environment.yml index bebcce3..dc46ab7 100755 --- a/environment.yml +++ b/environment.yml @@ -6,7 +6,7 @@ channels: - conda-forge dependencies: - - python = 2 + - python = 3 - numpy - netcdf4 >= 1.3.1 - python-cdo >= 1.3.6 diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 8d6ddda..bfc8aec 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -98,7 +98,9 @@ class Hermes(object): """ from datetime import timedelta - if self.options.fisrt_time: + if self.options.first_time: + settings.write_log('FINISHED FIRST TIME RUN') + self.comm.Barrier() self.comm.Abort(0) st_time = timeit.default_timer() diff --git a/run_test.py b/run_test.py index 6eb2926..655465f 100755 --- a/run_test.py +++ b/run_test.py @@ -1,5 +1,4 @@ # coding=utf-8 -"""Script to run the tests for EarthDiagnostics and generate the code coverage report""" import os import sys -- GitLab From f9dfbfe62bb4f4de19fa1a74666806af435c3160 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 22 Oct 2019 14:02:41 +0200 Subject: [PATCH 08/28] PEP8 corrections --- hermesv3_gr/modules/speciation/speciation.py | 2 +- hermesv3_gr/tools/netcdf_tools.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hermesv3_gr/modules/speciation/speciation.py b/hermesv3_gr/modules/speciation/speciation.py index 9d0c1b8..efb3061 100755 --- a/hermesv3_gr/modules/speciation/speciation.py +++ b/hermesv3_gr/modules/speciation/speciation.py @@ -146,7 +146,7 @@ class Speciation(object): raise KeyError('ERROR: {0} pollutant is not in the molecular weights file {1} .'.format( emission['name'], self.molecular_weights_path)) sys.exit(1) - exec ("{0} = np.array(emission['data'], dtype=settings.precision)".format(emission['name'])) + exec("{0} = np.array(emission['data'], dtype=settings.precision)".format(emission['name'])) emission['units'] = '' input_pollutants.append(emission['name']) diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index 1709435..8678e9f 100755 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -601,9 +601,8 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # if variable['data'] is not 0: # print var[:].shape, variable['data'].shape, variable['data'].max() shape = tuple() - exec ("shape = (len(hours), {0}.size, {1}.size, {2}.size)".format(var_dim[0], var_dim[1], var_dim[2])) - # exit() - print(shape) + exec("shape = (len(hours), {0}.size, {1}.size, {2}.size)".format(var_dim[0], var_dim[1], var_dim[2])) + var[:] = np.zeros(shape) # Grid mapping -- GitLab From 3b558b8534242b03c74266b4dbaa5e9c035794d9 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 22 Oct 2019 17:48:01 +0200 Subject: [PATCH 09/28] WIP --- conf/hermes.conf | 4 +- hermesv3_gr/hermes.py | 6 +-- .../emision_inventories/emission_inventory.py | 36 ++++++------- .../gfas_emission_inventory.py | 10 ++-- .../point_gfas_emission_inventory.py | 11 ++-- hermesv3_gr/modules/grids/grid.py | 45 +++++++++++++---- hermesv3_gr/modules/grids/grid_global.py | 12 ++--- hermesv3_gr/modules/grids/grid_latlon.py | 12 ++--- hermesv3_gr/modules/grids/grid_lcc.py | 12 ++--- hermesv3_gr/modules/grids/grid_mercator.py | 8 +-- hermesv3_gr/modules/grids/grid_rotated.py | 16 ++---- .../modules/regrid/regrid_conservative.py | 2 +- hermesv3_gr/modules/speciation/speciation.py | 2 +- hermesv3_gr/modules/temporal/temporal.py | 50 ++++++++----------- 14 files changed, 103 insertions(+), 123 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 58dae1a..7a12597 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -11,8 +11,8 @@ start_date = 2017/01/01 00:00:00 output_timestep_type = hourly output_timestep_num = 24 output_timestep_freq = 1 -first_time = 1 -erase_auxiliary_files = 0 +first_time = 0 +erase_auxiliary_files = 1 [DOMAIN] diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index bfc8aec..8f67bef 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -67,6 +67,7 @@ class Hermes(object): self.levels = VerticalDistribution.get_vertical_output_profile(self.options.vertical_description) self.grid = Grid.select_grid( + self.comm, self.options.domain_type, self.options.vertical_description, self.options.output_timestep_num, self.options.auxiliary_files_path, self.options.inc_lat, self.options.inc_lon, self.options.lat_orig, self.options.lon_orig, self.options.n_lat, self.options.n_lon, self.options.centre_lat, @@ -98,11 +99,6 @@ class Hermes(object): """ from datetime import timedelta - if self.options.first_time: - settings.write_log('FINISHED FIRST TIME RUN') - self.comm.Barrier() - self.comm.Abort(0) - st_time = timeit.default_timer() settings.write_log('') settings.write_log('***** Starting HERMESv3 *****') diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index e03a3bd..1670424 100755 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -119,16 +119,16 @@ class EmissionInventory(object): os.path.join(options.auxiliary_files_path, "Weight_Matrix_{0}_{1}.nc".format(self.inventory_name, settings.size)), grid, masking=self.masking) - - # Creating Vertical Object - if p_vertical is not None: - self.vertical = VerticalDistribution( - self.get_profile(p_vertical), vertical_profile_path=options.p_vertical, - vertical_output_profile=vertical_output_profile) - else: - self.vertical = None - settings.write_log('\t\tNone vertical profile set.', level=2) - self.vertical_factors = None + if not options.first_time: + # Creating Vertical Object + if p_vertical is not None: + self.vertical = VerticalDistribution( + self.get_profile(p_vertical), vertical_profile_path=options.p_vertical, + vertical_output_profile=vertical_output_profile) + else: + self.vertical = None + settings.write_log('\t\tNone vertical profile set.', level=2) + self.vertical_factors = None # Creating Temporal Object # It will also create the necessaries timezone files @@ -136,18 +136,18 @@ class EmissionInventory(object): self.temporal = TemporalDistribution( current_date, options.output_timestep_type, options.output_timestep_num, options.output_timestep_freq, options.p_month, p_month, options.p_week, p_week, options.p_day, p_day, options.p_hour, p_hour, - options.world_info, options.auxiliary_files_path, grid) + options.world_info, options.auxiliary_files_path, grid, options.first_time) else: self.temporal = None settings.write_log('\t\tNone temporal profile set.', level=2) self.temporal_factors = None - - # Creating Speciation Object - if p_speciation is not None: - self.speciation = Speciation(p_speciation, options.p_speciation, options.molecular_weights) - else: - self.speciation = None - settings.write_log('\t\tNone speciation profile set.', level=2) + if not options.first_time: + # Creating Speciation Object + if p_speciation is not None: + self.speciation = Speciation(p_speciation, options.p_speciation, options.molecular_weights) + else: + self.speciation = None + settings.write_log('\t\tNone speciation profile set.', level=2) self.vertical_weights = None diff --git a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py index 33a5328..c062615 100755 --- a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py @@ -82,13 +82,13 @@ class GfasEmissionInventory(EmissionInventory): vertical_output_profile, reference_year=reference_year, factors=factors, regrid_mask=regrid_mask, p_vertical=None, p_month=p_month, p_week=p_week, p_day=p_day, p_hour=p_hour, p_speciation=p_speciation) + if not options.first_time: + self.approach = self.get_approach(p_vertical) + self.method = self.get_method(p_vertical) - self.approach = self.get_approach(p_vertical) - self.method = self.get_method(p_vertical) + self.altitude = self.get_altitude() - self.altitude = self.get_altitude() - - self.vertical = GfasVerticalDistribution(vertical_output_profile, self.approach, self.get_altitude()) + self.vertical = GfasVerticalDistribution(vertical_output_profile, self.approach, self.get_altitude()) settings.write_time('GFAS_EmissionInventory', 'Init', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py index 78167bb..10ca3a1 100755 --- a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py @@ -84,13 +84,14 @@ class PointGfasEmissionInventory(EmissionInventory): reference_year=reference_year, factors=factors, regrid_mask=regrid_mask, p_vertical=None, p_month=p_month, p_week=p_week, p_day=p_day, p_hour=p_hour, p_speciation=p_speciation) - # self.approach = self.get_approach(p_vertical) - self.method = self.get_method(p_vertical) + if not options.first_time: + # self.approach = self.get_approach(p_vertical) + self.method = self.get_method(p_vertical) - # self.altitude = self.get_altitude() + # self.altitude = self.get_altitude() - self.vertical = GfasVerticalDistribution(vertical_output_profile, self.get_approach(p_vertical), - self.get_altitude()) + self.vertical = GfasVerticalDistribution(vertical_output_profile, self.get_approach(p_vertical), + self.get_altitude()) settings.write_time('PointGfasEmissionInventory', 'Init', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index dca2013..b3acdb9 100755 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -40,10 +40,10 @@ class Grid(object): :type temporal_path: str """ - def __init__(self, grid_type, vertical_description_path, temporal_path): + def __init__(self, grid_type, vertical_description_path, temporal_path, comm): st_time = timeit.default_timer() # settings.write_log('Creating Grid...', level=1) - + self.comm = comm # Defining class atributes self.procs_array = None self.nrows = 0 @@ -66,7 +66,7 @@ class Grid(object): self.temporal_path = temporal_path self.shapefile_path = None - self.esmf_grid = None + # self.esmf_grid = None self.x_lower_bound = None self.x_upper_bound = None self.y_lower_bound = None @@ -78,6 +78,27 @@ class Grid(object): settings.write_time('Grid', 'Init', timeit.default_timer() - st_time, level=1) + def calculate_bounds(self): + x_min = 0 + y_min = 0 + if len(self.center_latitudes.shape) == 1: + x_max = self.center_latitudes.shape[0] + y_max = self.center_longitudes.shape[0] + else: + x_max = self.center_latitudes.shape[0] + y_max = self.center_longitudes.shape[1] + y_len = y_max // self.comm.Get_size() + exceeded_ranks = y_max % self.comm.Get_size() + self.x_lower_bound = x_min + self.x_upper_bound = x_max + if self.comm.Get_rank() < exceeded_ranks: + y_len = y_len + 1 + self.y_lower_bound = self.comm.Get_rank() * y_len + self.y_upper_bound = (self.comm.Get_rank() + 1) * y_len + else: + self.y_lower_bound = (self.comm.Get_rank() * y_len) + exceeded_ranks + self.y_upper_bound = ((self.comm.Get_rank() + 1) * y_len) + exceeded_ranks + @staticmethod def create_esmf_grid_from_file(file_name, sphere=True): import ESMF @@ -94,7 +115,7 @@ class Grid(object): return grid @staticmethod - def select_grid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, + def select_grid(comm, grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, lon_orig, n_lat, n_lon, centre_lat, centre_lon, west_boundary, south_boundary, inc_rlat, inc_rlon, lat_1, lat_2, lon_0, lat_0, nx, ny, inc_x, inc_y, x_0, y_0, lat_ts): # TODO describe better the rotated parameters @@ -180,26 +201,27 @@ class Grid(object): # Creating a different object depending on the grid type if grid_type == 'global': from hermesv3_gr.modules.grids.grid_global import GlobalGrid - grid = GlobalGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon) + grid = GlobalGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, + comm=comm) elif grid_type == 'regular': from hermesv3_gr.modules.grids.grid_latlon import LatLonGrid grid = LatLonGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, - lat_orig, lon_orig, n_lat, n_lon) + lat_orig, lon_orig, n_lat, n_lon, comm=comm) elif grid_type == 'rotated': from hermesv3_gr.modules.grids.grid_rotated import RotatedGrid grid = RotatedGrid(grid_type, vertical_description_path, timestep_num, temporal_path, - centre_lat, centre_lon, west_boundary, south_boundary, inc_rlat, inc_rlon) + centre_lat, centre_lon, west_boundary, south_boundary, inc_rlat, inc_rlon, comm=comm) elif grid_type == 'lcc': from hermesv3_gr.modules.grids.grid_lcc import LccGrid grid = LccGrid(grid_type, vertical_description_path, timestep_num, temporal_path, lat_1, lat_2, lon_0, - lat_0, nx, ny, inc_x, inc_y, x_0, y_0) + lat_0, nx, ny, inc_x, inc_y, x_0, y_0, comm=comm) elif grid_type == 'mercator': from hermesv3_gr.modules.grids.grid_mercator import MercatorGrid grid = MercatorGrid(grid_type, vertical_description_path, timestep_num, temporal_path, lat_ts, lon_0, - nx, ny, inc_x, inc_y, x_0, y_0) + nx, ny, inc_x, inc_y, x_0, y_0, comm=comm) else: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -389,8 +411,9 @@ class Grid(object): st_time = timeit.default_timer() settings.write_log('\t\tGetting 2D coordinates from ESMPy Grid', level=3) - lat = self.esmf_grid.get_coords(1, ESMF.StaggerLoc.CENTER).T - lon = self.esmf_grid.get_coords(0, ESMF.StaggerLoc.CENTER).T + esmf_grid = self.create_esmf_grid_from_file(self.coords_netcdf_file) + lat = esmf_grid.get_coords(1, ESMF.StaggerLoc.CENTER).T + lon = esmf_grid.get_coords(0, ESMF.StaggerLoc.CENTER).T settings.write_time('Grid', 'get_coordinates_2d', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/grids/grid_global.py b/hermesv3_gr/modules/grids/grid_global.py index a3ca236..2f63c3c 100755 --- a/hermesv3_gr/modules/grids/grid_global.py +++ b/hermesv3_gr/modules/grids/grid_global.py @@ -54,14 +54,13 @@ class GlobalGrid(Grid): """ def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, - center_longitude=float(0)): - import ESMF + center_longitude=float(0), comm=None): st_time = timeit.default_timer() settings.write_log('\tCreating Global grid.', level=2) # Initialize the class using parent - super(GlobalGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + super(GlobalGrid, self).__init__(grid_type, vertical_description_path, temporal_path, comm) self.center_lat = float(0) self.center_lon = center_longitude @@ -76,12 +75,7 @@ class GlobalGrid(Grid): super(GlobalGrid, self).write_coords_netcdf() settings.comm.Barrier() - self.esmf_grid = super(GlobalGrid, self).create_esmf_grid_from_file(self.coords_netcdf_file) - - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/grids/grid_latlon.py b/hermesv3_gr/modules/grids/grid_latlon.py index 0409d7b..df0a676 100755 --- a/hermesv3_gr/modules/grids/grid_latlon.py +++ b/hermesv3_gr/modules/grids/grid_latlon.py @@ -63,14 +63,13 @@ class LatLonGrid(Grid): """ def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, - lon_orig, n_lat, n_lon): - import ESMF + lon_orig, n_lat, n_lon, comm=None): st_time = timeit.default_timer() settings.write_log('\tCreating Regional regular lat-lon grid.', level=2) # Initialize the class using parent - super(LatLonGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + super(LatLonGrid, self).__init__(grid_type, vertical_description_path, temporal_path, comm) self.inc_lat = inc_lat self.inc_lon = inc_lon @@ -87,12 +86,7 @@ class LatLonGrid(Grid): self.write_coords_netcdf() settings.comm.Barrier() - self.esmf_grid = self.create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) - - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/grids/grid_lcc.py b/hermesv3_gr/modules/grids/grid_lcc.py index d360d74..42aaebc 100755 --- a/hermesv3_gr/modules/grids/grid_lcc.py +++ b/hermesv3_gr/modules/grids/grid_lcc.py @@ -77,13 +77,12 @@ class LccGrid(Grid): """ def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, lat_1, lat_2, lon_0, lat_0, - nx, ny, inc_x, inc_y, x_0, y_0, earth_radius=6370000.000): - import ESMF + nx, ny, inc_x, inc_y, x_0, y_0, earth_radius=6370000.000, comm=None): st_time = timeit.default_timer() settings.write_log('\tCreating Lambert Conformal Conic (LCC) grid.', level=2) # Initialises with parent class - super(LccGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + super(LccGrid, self).__init__(grid_type, vertical_description_path, temporal_path, comm) # Setting parameters self.lat_1 = lat_1 @@ -113,12 +112,7 @@ class LccGrid(Grid): self.write_coords_netcdf() settings.comm.Barrier() - self.esmf_grid = super(LccGrid, self).create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) - # - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/grids/grid_mercator.py b/hermesv3_gr/modules/grids/grid_mercator.py index 38bb722..d18c32d 100755 --- a/hermesv3_gr/modules/grids/grid_mercator.py +++ b/hermesv3_gr/modules/grids/grid_mercator.py @@ -69,7 +69,6 @@ class MercatorGrid(Grid): def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, lat_ts, lon_0, nx, ny, inc_x, inc_y, x_0, y_0, earth_radius=6370000.000): - import ESMF st_time = timeit.default_timer() settings.write_log('\tCreating Mercator grid.', level=2) @@ -101,12 +100,7 @@ class MercatorGrid(Grid): self.write_coords_netcdf() settings.comm.Barrier() - self.esmf_grid = super(MercatorGrid, self).create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) - # - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/grids/grid_rotated.py b/hermesv3_gr/modules/grids/grid_rotated.py index 24c0f9a..8c99036 100755 --- a/hermesv3_gr/modules/grids/grid_rotated.py +++ b/hermesv3_gr/modules/grids/grid_rotated.py @@ -40,14 +40,12 @@ class RotatedGrid(Grid): """ def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, centre_lat, centre_lon, - west_boundary, south_boundary, inc_rlat, inc_rlon): - import ESMF - + west_boundary, south_boundary, inc_rlat, inc_rlon, comm=None): st_time = timeit.default_timer() settings.write_log('\tCreating Rotated grid.', level=2) # Initialises with parent class - super(RotatedGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + super(RotatedGrid, self).__init__(grid_type, vertical_description_path, temporal_path, comm) # Setting parameters self.new_pole_longitude_degrees = -180 + centre_lon @@ -71,18 +69,10 @@ class RotatedGrid(Grid): if not os.path.exists(self.coords_netcdf_file): if settings.rank == 0: - # super(RotatedGrid, self).write_coords_netcdf() self.write_coords_netcdf() settings.comm.Barrier() - # self.write_coords_netcdf() - - self.esmf_grid = super(RotatedGrid, self).create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) - - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/regrid/regrid_conservative.py b/hermesv3_gr/modules/regrid/regrid_conservative.py index 35c718f..74c219e 100755 --- a/hermesv3_gr/modules/regrid/regrid_conservative.py +++ b/hermesv3_gr/modules/regrid/regrid_conservative.py @@ -48,7 +48,7 @@ class ConservativeRegrid(Regrid): src_field.read(filename=self.pollutant_dicts[0]['path'], variable=self.pollutant_dicts[0]['name'], timeslice=0) - dst_grid = self.grid.esmf_grid + dst_grid = self.grid.create_esmf_grid_from_file(self.grid.coords_netcdf_file) dst_field = ESMF.Field(dst_grid, name='my outut field') ESMF.Regrid(src_field, dst_field, filename=self.weight_matrix_file, regrid_method=ESMF.RegridMethod.CONSERVE,) diff --git a/hermesv3_gr/modules/speciation/speciation.py b/hermesv3_gr/modules/speciation/speciation.py index efb3061..713a484 100755 --- a/hermesv3_gr/modules/speciation/speciation.py +++ b/hermesv3_gr/modules/speciation/speciation.py @@ -182,7 +182,7 @@ class Speciation(object): raise AttributeError( "Error in speciation profile {0}: ".format(self.id) + "The output specie {0} cannot be calculated ".format(pollutant['name']) + - "with the expression {0} because{1}".format(formula, e.message)) + "with the expression {0} because{1}".format(formula, str(e))) else: sys.exit(1) speciated_emissions.append(dict_aux) diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 5606513..6f2c457 100755 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -70,7 +70,7 @@ class TemporalDistribution(object): """ def __init__(self, starting_date, timestep_type, timestep_num, timestep_freq, monthly_profile_path, monthly_profile_id, weekly_profile_path, weekly_profile_id, daily_profile_path, daily_profile_id, - hourly_profile_path, hourly_profile_id, world_info_path, auxiliar_files_dir, grid): + hourly_profile_path, hourly_profile_id, world_info_path, auxiliar_files_dir, grid, first_time=False): from timezonefinder import TimezoneFinder import pandas as pd @@ -80,40 +80,34 @@ class TemporalDistribution(object): self.grid = grid - # self.starting_date = starting_date + if not first_time: + self.date_array = self.caclulate_date_array(starting_date, timestep_type, timestep_num, timestep_freq) - self.date_array = self.caclulate_date_array(starting_date, timestep_type, timestep_num, timestep_freq) + if timestep_type in ['daily', 'hourly']: + self.daily_profile = self.get_daily_profile(daily_profile_id, daily_profile_path) + else: + self.daily_profile = None - # self.timestep_type = timestep_type - # self.timestep_num = timestep_num - # self.timestep_freq = timestep_freq - # - # self.ending_date = self.calculate_ending_date() - if timestep_type in ['daily', 'hourly']: - self.daily_profile = self.get_daily_profile(daily_profile_id, daily_profile_path) - else: - self.daily_profile = None + if self.daily_profile is None: + if timestep_type in ['monthly', 'daily', 'hourly']: + self.monthly_profile = self.get_monthly_profile(monthly_profile_id, monthly_profile_path) + else: + self.monthly_profile = None - if self.daily_profile is None: - if timestep_type in ['monthly', 'daily', 'hourly']: - self.monthly_profile = self.get_monthly_profile(monthly_profile_id, monthly_profile_path) + if timestep_type in ['daily', 'hourly']: + self.weekly_profile = self.get_weekly_profile(weekly_profile_id, weekly_profile_path) + else: + self.weekly_profile = None else: self.monthly_profile = None - - if timestep_type in ['daily', 'hourly']: - self.weekly_profile = self.get_weekly_profile(weekly_profile_id, weekly_profile_path) - else: self.weekly_profile = None - else: - self.monthly_profile = None - self.weekly_profile = None - if monthly_profile_id is not None or weekly_profile_id is not None: - warn('WARNING: Daily profile is set. Monthly or Weekly profiles will be ignored.') + if monthly_profile_id is not None or weekly_profile_id is not None: + warn('WARNING: Daily profile is set. Monthly or Weekly profiles will be ignored.') - if timestep_type in ['hourly']: - self.hourly_profile = self.get_hourly_profile(hourly_profile_id, hourly_profile_path) - else: - self.hourly_profile = None + if timestep_type in ['hourly']: + self.hourly_profile = self.get_hourly_profile(hourly_profile_id, hourly_profile_path) + else: + self.hourly_profile = None self.world_info = world_info_path self.netcdf_timezones = os.path.join(auxiliar_files_dir, 'timezones.nc') -- GitLab From a9e802fed48c4e64a3cd96d5ae1e48aa6938316e Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 23 Oct 2019 10:17:48 +0200 Subject: [PATCH 10/28] drop pandas warning --- hermesv3_gr/modules/grids/grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index b3acdb9..59f42be 100755 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -534,7 +534,7 @@ class Grid(object): # Make a list of list of tuples # [[(point_1.1), (point_1.2), (point_1.3), (point_1.4)], # [(point_2.1), (point_2.2), (point_2.3), (point_2.4)], ...] - list_points = df.as_matrix() + list_points = df.values del df['p1'], df['p2'], df['p3'], df['p4'] # List of polygons from the list of points -- GitLab From 17d596f304ae8bf75f5f42e21c8236c466c8ab83 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 23 Oct 2019 11:52:55 +0200 Subject: [PATCH 11/28] bug fix --- hermesv3_gr/hermes.py | 74 ++++++++++--------- .../point_gfas_emission_inventory.py | 11 ++- hermesv3_gr/modules/grids/grid.py | 42 +---------- 3 files changed, 45 insertions(+), 82 deletions(-) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 8f67bef..8718644 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -99,41 +99,45 @@ class Hermes(object): """ from datetime import timedelta - st_time = timeit.default_timer() - settings.write_log('') - settings.write_log('***** Starting HERMESv3 *****') - num = 1 - for ei in self.emission_list: - settings.write_log('Processing emission inventory {0} for the sector {1} ({2}/{3}):'.format( - ei.inventory_name, ei.sector, num, len(self.emission_list))) - num += 1 - - ei.do_regrid() - - if ei.vertical is not None: - settings.write_log("\tCalculating vertical distribution.", level=2) - if ei.source_type == 'area': - ei.vertical_factors = ei.vertical.calculate_weights() - elif ei.source_type == 'point': - ei.calculate_altitudes(self.options.vertical_description) - ei.point_source_by_cell() - # To avoid use point source as area source when is going to apply vertical factors while writing. - ei.vertical = None - else: - settings.write_log('ERROR: Check the .err file to get more info.') - if settings.rank == 0: - raise AttributeError('Unrecognized emission source type {0}'.format(ei.source_type)) - sys.exit(1) - - if ei.temporal is not None: - ei.temporal_factors = ei.temporal.calculate_3d_temporal_factors() - if ei.speciation is not None: - ei.emissions = ei.speciation.do_speciation(ei.emissions) - - self.writer.write(self.emission_list) - - settings.write_log("***** HERMESv3 simulation finished successfully *****") - settings.write_time('HERMES', 'main', timeit.default_timer() - st_time) + if self.options.first_time: + # Stop run + settings.write_log('***** HERMESv3_BU First Time finished successfully *****') + else: + st_time = timeit.default_timer() + settings.write_log('') + settings.write_log('***** Starting HERMESv3_GR *****') + num = 1 + for ei in self.emission_list: + settings.write_log('Processing emission inventory {0} for the sector {1} ({2}/{3}):'.format( + ei.inventory_name, ei.sector, num, len(self.emission_list))) + num += 1 + + ei.do_regrid() + + if ei.vertical is not None: + settings.write_log("\tCalculating vertical distribution.", level=2) + if ei.source_type == 'area': + ei.vertical_factors = ei.vertical.calculate_weights() + elif ei.source_type == 'point': + ei.calculate_altitudes(self.options.vertical_description) + ei.point_source_by_cell() + # To avoid use point source as area source when is going to apply vertical factors while writing. + ei.vertical = None + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('Unrecognized emission source type {0}'.format(ei.source_type)) + sys.exit(1) + + if ei.temporal is not None: + ei.temporal_factors = ei.temporal.calculate_3d_temporal_factors() + if ei.speciation is not None: + ei.emissions = ei.speciation.do_speciation(ei.emissions) + + self.writer.write(self.emission_list) + + settings.write_log("***** HERMESv3_GR simulation finished successfully *****") + settings.write_time('HERMES', 'main', timeit.default_timer() - st_time) settings.write_time('HERMES', 'TOTAL', timeit.default_timer() - full_time) settings.finish_logs(self.options.output_dir, self.options.start_date) diff --git a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py index 10ca3a1..78167bb 100755 --- a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py @@ -84,14 +84,13 @@ class PointGfasEmissionInventory(EmissionInventory): reference_year=reference_year, factors=factors, regrid_mask=regrid_mask, p_vertical=None, p_month=p_month, p_week=p_week, p_day=p_day, p_hour=p_hour, p_speciation=p_speciation) - if not options.first_time: - # self.approach = self.get_approach(p_vertical) - self.method = self.get_method(p_vertical) + # self.approach = self.get_approach(p_vertical) + self.method = self.get_method(p_vertical) - # self.altitude = self.get_altitude() + # self.altitude = self.get_altitude() - self.vertical = GfasVerticalDistribution(vertical_output_profile, self.get_approach(p_vertical), - self.get_altitude()) + self.vertical = GfasVerticalDistribution(vertical_output_profile, self.get_approach(p_vertical), + self.get_altitude()) settings.write_time('PointGfasEmissionInventory', 'Init', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index 59f42be..520b56b 100755 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -450,21 +450,13 @@ class Grid(object): settings.write_log('\t\tGrid shapefile not done. Lets try to create it.', level=3) # Create Shapefile - # Use the meters coordiantes to create the shapefile - y = self.boundary_latitudes x = self.boundary_longitudes - # sys.exit() if self.grid_type in ['global', 'regular']: x = x.reshape((x.shape[1], x.shape[2])) y = y.reshape((y.shape[1], y.shape[2])) - # x_aux = np.empty((x.shape[0], y.shape[0], 4)) - # x_aux[:, :, 0] = x[:, np.newaxis, 0] - # x_aux[:, :, 1] = x[:, np.newaxis, 1] - # x_aux[:, :, 2] = x[:, np.newaxis, 1] - # x_aux[:, :, 3] = x[:, np.newaxis, 0] aux_shape = (y.shape[0], x.shape[0], 4) x_aux = np.empty(aux_shape) x_aux[:, :, 0] = x[np.newaxis, :, 0] @@ -475,12 +467,6 @@ class Grid(object): x = x_aux del x_aux - # y_aux = np.empty((x.shape[0], y.shape[0], 4)) - # y_aux[:, :, 0] = y[np.newaxis, :, 0] - # y_aux[:, :, 1] = y[np.newaxis, :, 0] - # y_aux[:, :, 2] = y[np.newaxis, :, 1] - # y_aux[:, :, 3] = y[np.newaxis, :, 1] - y_aux = np.empty(aux_shape) y_aux[:, :, 0] = y[:, np.newaxis, 0] y_aux[:, :, 1] = y[:, np.newaxis, 0] @@ -490,8 +476,6 @@ class Grid(object): y = y_aux del y_aux - # exit() - if not full_grid: y = y[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound, :] x = x[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound, :] @@ -499,29 +483,12 @@ class Grid(object): aux_b_lats = y.reshape((y.shape[0] * y.shape[1], y.shape[2])) aux_b_lons = x.reshape((x.shape[0] * x.shape[1], x.shape[2])) - # The regular lat-lon projection has only 2 (laterals) points for each cell instead of 4 (corners) - # if aux_b_lats.shape[1] == 2: - # aux_b = np.empty((aux_b_lats.shape[0], 4)) - # aux_b[:, 0] = aux_b_lats[:, 0] - # aux_b[:, 1] = aux_b_lats[:, 0] - # aux_b[:, 2] = aux_b_lats[:, 1] - # aux_b[:, 3] = aux_b_lats[:, 1] - # aux_b_lats = aux_b - # - # if aux_b_lons.shape[1] == 2: - # aux_b = np.empty((aux_b_lons.shape[0], 4)) - # aux_b[:, 0] = aux_b_lons[:, 0] - # aux_b[:, 1] = aux_b_lons[:, 1] - # aux_b[:, 2] = aux_b_lons[:, 1] - # aux_b[:, 3] = aux_b_lons[:, 0] - # aux_b_lons = aux_b - # Create one dataframe with 8 columns, 4 points with two coordinates each one df_lats = pd.DataFrame(aux_b_lats, columns=['b_lat_1', 'b_lat_2', 'b_lat_3', 'b_lat_4']) df_lons = pd.DataFrame(aux_b_lons, columns=['b_lon_1', 'b_lon_2', 'b_lon_3', 'b_lon_4']) df = pd.concat([df_lats, df_lons], axis=1) - # Substituate 8 columns by 4 with the two coordinates + # Substitute 8 columns by 4 with the two coordinates df['p1'] = list(zip(df.b_lon_1, df.b_lat_1)) del df['b_lat_1'], df['b_lon_1'] df['p2'] = list(zip(df.b_lon_2, df.b_lat_2)) @@ -539,13 +506,6 @@ class Grid(object): # List of polygons from the list of points geometry = [Polygon(list(points)) for points in list_points] - # geometry = [] - # for point in list_points: - # print point - # geometry.append(Polygon(list(point))) - # print geometry[0] - # sys.exit() - # print len(geometry), len(df), gdf = gpd.GeoDataFrame(df, crs={'init': 'epsg:4326'}, geometry=geometry) gdf = gdf.to_crs(self.crs) -- GitLab From 71cdd3d9b8200bc667070a1bfc32292c9ec40220 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 23 Oct 2019 17:59:21 +0200 Subject: [PATCH 12/28] wip --- hermesv3_gr/hermes.py | 8 +++++--- hermesv3_gr/modules/writing/writer.py | 25 ++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 8718644..0dcf0e0 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -93,7 +93,7 @@ class Hermes(object): settings.write_time('HERMES', 'Init', timeit.default_timer() - st_time, level=1) # @profile - def main(self): + def main(self, return_emis=True): """ Main functionality of the model. """ @@ -133,8 +133,10 @@ class Hermes(object): ei.temporal_factors = ei.temporal.calculate_3d_temporal_factors() if ei.speciation is not None: ei.emissions = ei.speciation.do_speciation(ei.emissions) - - self.writer.write(self.emission_list) + if return_emis: + return self.writer.get_emis(self.emission_list) + else: + self.writer.write(self.emission_list) settings.write_log("***** HERMESv3_GR simulation finished successfully *****") settings.write_time('HERMES', 'main', timeit.default_timer() - st_time) diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index 8cb46f5..e12df93 100755 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -25,6 +25,7 @@ from mpi4py import MPI from netCDF4 import Dataset from hermesv3_gr.config import settings from functools import reduce +from pandas import DataFrame, MultiIndex class Writer(object): @@ -186,7 +187,7 @@ class Writer(object): return True - def calculate_data_by_var(self, variable, inventory_list, shape): + def calculate_data_by_var(self, variable, inventory_list, shape, change_units=True): """ Calculate the date of the given variable throw the inventory list. @@ -243,7 +244,8 @@ class Writer(object): settings.write_time('TemporalDistribution', 'calculate_data_by_var', timeit.default_timer() - temporal_time, level=2) # Unit changes - data = self.unit_change(variable, data) + if change_units: + data = self.unit_change(variable, data) if data is not None: data[data < 0] = 0 settings.write_time('Writer', 'calculate_data_by_var', timeit.default_timer() - st_time, level=3) @@ -253,7 +255,7 @@ class Writer(object): """ Implement on inner class """ - return np.array([0]) + return data @staticmethod def calculate_displacements(counts): @@ -603,3 +605,20 @@ class Writer(object): netcdf.setncatts(global_attributes) netcdf.close() + + def get_emis(self, emission_list): + self.set_variable_attributes(emission_list) + self.change_variable_attributes() + data = DataFrame(columns=self.variables_attributes.keys(), + index=['FID', 'layer', 'tstep']) + for var_name in self.variables_attributes.keys(): + print(var_name) + var_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape, change_units=False) + if var_data is None: + data[var_name] = 0 + else: + for tstep in range(data.shape[0]): + for layer in range(data.shape[1]): + + print(data.shape) + return None -- GitLab From 7ef41b3f8b266862140dbbba64ce82084ff8e4a8 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 25 Oct 2019 17:00:05 +0200 Subject: [PATCH 13/28] return_emis in memory done --- hermesv3_gr/hermes.py | 6 ++--- hermesv3_gr/modules/writing/writer.py | 38 ++++++++++++++++++++------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 0dcf0e0..c7f6053 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -93,7 +93,7 @@ class Hermes(object): settings.write_time('HERMES', 'Init', timeit.default_timer() - st_time, level=1) # @profile - def main(self, return_emis=True): + def main(self, return_emis=False): """ Main functionality of the model. """ @@ -149,10 +149,10 @@ class Hermes(object): return None -def run(comm=None): +def run(): date = Hermes(Config()).main() while date is not None: - date = Hermes(Config(comm), new_date=date, comm=comm).main() + date = Hermes(Config(), new_date=date).main() sys.exit(0) diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index e12df93..535d50e 100755 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -25,6 +25,7 @@ from mpi4py import MPI from netCDF4 import Dataset from hermesv3_gr.config import settings from functools import reduce +import pandas as pd from pandas import DataFrame, MultiIndex @@ -609,16 +610,35 @@ class Writer(object): def get_emis(self, emission_list): self.set_variable_attributes(emission_list) self.change_variable_attributes() - data = DataFrame(columns=self.variables_attributes.keys(), - index=['FID', 'layer', 'tstep']) + + data_list = [] + + fid_array = np.array(range(self.grid.full_shape[-2] * self.grid.full_shape[-1])) + fid_array = fid_array.reshape((self.grid.full_shape[-2], self.grid.full_shape[-1])) + fid_array = fid_array[self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound].flatten() + + zero_vars = [] for var_name in self.variables_attributes.keys(): - print(var_name) var_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape, change_units=False) if var_data is None: - data[var_name] = 0 + zero_vars.append(var_name) else: - for tstep in range(data.shape[0]): - for layer in range(data.shape[1]): - - print(data.shape) - return None + var_data_list = [] + for tstep in range(var_data.shape[0]): + for layer in range(var_data.shape[1]): + aux_data = DataFrame(data=var_data[tstep, layer, :, :].flatten(), columns=[var_name], + index=MultiIndex.from_product([fid_array, [layer], [tstep]], + names=['FID', 'layer', 'tstep'])) + aux_data = aux_data.replace(0, np.nan) + aux_data.dropna(how='all', inplace=True) + var_data_list.append(aux_data) + data_list.append(pd.concat(var_data_list, sort=False)) + + data = pd.concat(data_list, axis=1, sort=False) + data = data.replace(np.nan, 0) + + for var_name in zero_vars: + data[var_name] = 0 + + return data -- GitLab From 37a1d5f72dfedf73387bf08d72f2955e52095844 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 25 Oct 2019 17:12:53 +0200 Subject: [PATCH 14/28] PEP8 corrections --- hermesv3_gr/hermes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index c7f6053..bb047b9 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -121,7 +121,7 @@ class Hermes(object): elif ei.source_type == 'point': ei.calculate_altitudes(self.options.vertical_description) ei.point_source_by_cell() - # To avoid use point source as area source when is going to apply vertical factors while writing. + # To avoid use point source as area source when is going to apply vertical factors while writing ei.vertical = None else: settings.write_log('ERROR: Check the .err file to get more info.') -- GitLab From 29a3c37f3c7d114f589cf90224cc7f832c0c03c3 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 25 Oct 2019 17:23:16 +0200 Subject: [PATCH 15/28] PEP8 corrections --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 947c8d8..8a31e65 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ - Python 3 - first_time option - erase_auxiliary_files option + - Option to return emissions in memory 1.0.4 2019/07/19 -- GitLab From 83bfa6c01e9f5aa46d377e71c4a7168ffb463cb1 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 25 Oct 2019 17:23:56 +0200 Subject: [PATCH 16/28] Changelog updated --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 8a31e65..032fc29 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,5 @@ 2.0.0 + 2019/10/25 - Python 3 - first_time option - erase_auxiliary_files option -- GitLab From f0df366ab48a3fe06f38c83d6a4d0d3bab054beb Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 25 Oct 2019 17:24:31 +0200 Subject: [PATCH 17/28] Updated version --- hermesv3_gr/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_gr/__init__.py b/hermesv3_gr/__init__.py index 92192ee..8c0d5d5 100755 --- a/hermesv3_gr/__init__.py +++ b/hermesv3_gr/__init__.py @@ -1 +1 @@ -__version__ = "1.0.4" +__version__ = "2.0.0" -- GitLab From eb8c04dd541ff5da7fb646156ce0cac56d48fc22 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 29 Oct 2019 10:57:07 +0100 Subject: [PATCH 18/28] Added CMAQ vertical descriptions --- .../CMAQ_37layers_vertical_description.csv | 38 +++++++++++++++++++ ..._37layers_vertical_description_9layers.csv | 10 +++++ 2 files changed, 48 insertions(+) create mode 100644 data/profiles/vertical/CMAQ_37layers_vertical_description.csv create mode 100644 data/profiles/vertical/CMAQ_37layers_vertical_description_9layers.csv diff --git a/data/profiles/vertical/CMAQ_37layers_vertical_description.csv b/data/profiles/vertical/CMAQ_37layers_vertical_description.csv new file mode 100644 index 0000000..a920af5 --- /dev/null +++ b/data/profiles/vertical/CMAQ_37layers_vertical_description.csv @@ -0,0 +1,38 @@ +Ilayer;height_magl +1;40 +2;88 +3;145 +4;226 +5;324 +6;456 +7;623 +8;819 +9;1044 +10;1319 +11;1657 +12;2074 +13;2582 +14;3215 +15;3848 +16;4489 +17;5122 +18;5742 +19;6374 +20;6987 +21;7609 +22;8223 +23;8843 +24;9469 +25;10078 +26;10689 +27;11300 +28;11939 +29;12580 +30;13218 +31;13971 +32;14779 +33;15599 +34;16616 +35;17604 +36;18685 +37;18870 diff --git a/data/profiles/vertical/CMAQ_37layers_vertical_description_9layers.csv b/data/profiles/vertical/CMAQ_37layers_vertical_description_9layers.csv new file mode 100644 index 0000000..7f83db5 --- /dev/null +++ b/data/profiles/vertical/CMAQ_37layers_vertical_description_9layers.csv @@ -0,0 +1,10 @@ +Ilayer;height_magl +1;40 +2;88 +3;145 +4;226 +5;324 +6;456 +7;623 +8;819 +9;18870 -- GitLab From cfad9d3b1627c6c059f1a451e8796200bd636768 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 29 Oct 2019 13:06:19 +0100 Subject: [PATCH 19/28] Updating profiles --- data/profiles/speciation/MolecularWeights.csv | 4 ++ .../Speciation_profile_cb05_aero6_CMAQ.csv | 46 +++++++++++++++++++ .../temporal/TemporalProfile_Hourly.csv | 3 ++ .../temporal/TemporalProfile_Monthly.csv | 5 +- data/profiles/vertical/Vertical_profile.csv | 3 +- 5 files changed, 59 insertions(+), 2 deletions(-) diff --git a/data/profiles/speciation/MolecularWeights.csv b/data/profiles/speciation/MolecularWeights.csv index f005210..262b591 100755 --- a/data/profiles/speciation/MolecularWeights.csv +++ b/data/profiles/speciation/MolecularWeights.csv @@ -10,6 +10,9 @@ pm25_fossil;1.0 pm25_bio;1.0 oc;1.0 bc;1.0 +ec;1.0 +so4;1.0 +ash;1.0 c2h6s;62.13 hcl;36.46 c2h2;26.04 @@ -63,3 +66,4 @@ voc22;68.8 voc23;75.3 voc24;59.1 voc25;86.9 +nmvoc;1.0 \ No newline at end of file diff --git a/data/profiles/speciation/Speciation_profile_cb05_aero6_CMAQ.csv b/data/profiles/speciation/Speciation_profile_cb05_aero6_CMAQ.csv index 1546735..e546468 100755 --- a/data/profiles/speciation/Speciation_profile_cb05_aero6_CMAQ.csv +++ b/data/profiles/speciation/Speciation_profile_cb05_aero6_CMAQ.csv @@ -26,3 +26,49 @@ E022;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0; E023;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25;pm10-pm25 E024;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-bc;pm10-pm25 E086;0;0;0;0;so2;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E074;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.000008*nmvoc;0;0;0.000858*nmvoc;0.004177*nmvoc;0;0.018548*nmvoc;0;0;0;0.00104*nmvoc;0.011594*nmvoc;0;0;0;0.000893*nmvoc;0;0;0;0.02*pm25;0.01*pm25;0;0.15*pm25;0;pm25*0.00379;pm25*0.01152;pm25*0.03437;pm25*0.0844;pm25*0.00227;pm25*0.00021;pm25*0.00064;pm25*0.01573;pm25*0.05032;pm25*0.02394;pm25*0.00311;pm25*0.00438;pm25*0.58524;pm10-pm25 +E075;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.000013*nmvoc;0;0;0.003224*nmvoc;0.001662*nmvoc;0.000659*nmvoc;0.008038*nmvoc;0;0;0.000175*nmvoc;0.000496*nmvoc;0.025751*nmvoc;0;0;0;0.001726*nmvoc;0;0;0;0.03*pm25;0.01*pm25;0;0.1*pm25;pm25*0.02498;0;pm25*0.03098;0;0;0;0;0;0;0;0;0;0;pm25*0.80402;pm10-pm25 +E076;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.000047*nmvoc;0;0;0.002914*nmvoc;0.004187*nmvoc;0.000849*nmvoc;0.00186*nmvoc;0;0;0;0.002559*nmvoc;0.029992*nmvoc;0;0;0;0.001755*nmvoc;0;0;0;0.35*pm25;0.18*pm25;0;0.02*pm25;0;pm25*0.00254;pm25*0.3602;pm25*0.00767;pm25*0.00472;pm25*0.00197;pm25*0.00003;pm25*0.00074;pm25*0.01023;pm25*0.00446;pm25*0.00569;pm25*0.00026;pm25*0.00984;pm25*0.04158;pm10-pm25 +E077;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0.000006*nmvoc;0.005934*nmvoc;0;0.000026*nmvoc;0;0.000001*nmvoc;0;0.000373*nmvoc;0.055357*nmvoc;0;0;0;0.000048*nmvoc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25;pm10-pm25 +E078;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0.000087*nmvoc;0.000148*nmvoc;0.003563*nmvoc;0.000004*nmvoc;0;0.000001*nmvoc;0.001904*nmvoc;0.000161*nmvoc;0.036006*nmvoc;0;0;0;0.001686*nmvoc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E079;0.7*nox_no2;0.283*nox_no2;0.017*nox_no2;co;so2;nh3;0.000592*nmvoc;0;0;0.002423*nmvoc;0.001607*nmvoc;0;0.000899*nmvoc;0;0;0;0.002589*nmvoc;0.028079*nmvoc;0;0;0;0.003302*nmvoc;0;0;0;0.32*pm25;0.49*pm25;0;0.01*pm25;0;pm25*0.00006;pm25*0.02119;pm25*0.00013;pm25*0.00047;pm25*0.00002;0;pm25*0.00006;pm25*0.00116;pm25*0.00003;pm25*0.00018;0;0;pm25*0.00364;pm10-pm25 +E080;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0.002496*nmvoc;0.004456*nmvoc;0;0;0.001693*nmvoc;0;0;0;0.001111*nmvoc;0.030672*nmvoc;0;0;0.001132*nmvoc;0.000762*nmvoc;0;0;0;0.12*pm25;0.005*pm25;0;0.40*pm25;0;0;pm25*0.07404;pm25*0.00018;pm25*0.00074;0;0;0;0;0;pm25*0.00006;0;0;pm25*0.17496;pm10-pm25 +E081;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.001084*nmvoc;0.000756*nmvoc;0.000248*nmvoc;0.00622*nmvoc;0.000293*nmvoc;0;0.005437*nmvoc;0.000086*nmvoc;0;0;0.003296*nmvoc;0.011816*nmvoc;0;0;0.000192*nmvoc;0.000124*nmvoc;0;0;0;0.62*pm25;0.16*pm25;0;0.15*pm25;0;0;pm25*0.07404;pm25*0.00018;pm25*0.00074;0;0;0;0;0;pm25*0.00006;0;0;pm25*0.17496;pm10-pm25 +E082;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.000227*nmvoc;0.000311*nmvoc;0.000333*nmvoc;0.003921*nmvoc;0.000166*nmvoc;0;0.00206*nmvoc;0.00079*nmvoc;0;0;0.00111*nmvoc;0.039123*nmvoc;0;0;0.000245*nmvoc;0.000476*nmvoc;0;0;0;0.31*pm25;0.41*pm25;0;0.03*pm25;0;0;pm25*0.07404;pm25*0.00018;pm25*0.00074;0;0;0;0;0;pm25*0.00006;0;0;pm25*0.17496;pm10-pm25 +E083;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0.001867*nmvoc;0.008553*nmvoc;0.000529*nmvoc;0.005349*nmvoc;0;0.000169*nmvoc;0.000133*nmvoc;0.000435*nmvoc;0.025797*nmvoc;0;0;0;0.000742*nmvoc;0;0;0;0.31*pm25;0.2*pm25;0;0;0;pm25*0.08261;pm25*0.02011;pm25*0.01311;pm25*0.04112;0;pm25*0.00035;pm25*0.00484;pm25*0.0492;pm25*0.01191;pm25*0.01424;pm25*0.00143;pm25*0.01293;pm25*0.23809;pm10-pm25 +E084;nox_no2;0;0;co;0;nh3;0.001591*nmvoc;0.000103*nmvoc;0.000038*nmvoc;0;0;0.000022*nmvoc;0;0;0;0;0.000023*nmvoc;0.03941*nmvoc;0;0.000007*nmvoc;0.000237*nmvoc;0;0;0;0;0.318*pm25;0.0516*pm25;0;0.0446*pm25;pm25*0.00427;pm25*0.04911;pm25*0.17393;pm25*0.00506;pm25*0.00919;pm25*0.00042;pm25*0.00009;pm25*0.00463;pm25*0.01719;pm25*0.0031;pm25*0.00277;pm25*0.00011;pm25*0.04219;pm25*0.05788;pm10-pm25 +E085;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.002093*nmvoc;0.000525*nmvoc;0.000105*nmvoc;0.002929*nmvoc;0.001894*nmvoc;0.000023*nmvoc;0.002826*nmvoc;0.000116*nmvoc;0.000043*nmvoc;0.003664*nmvoc;0.003078*nmvoc;0.010261*nmvoc;0;0.000006*nmvoc;0.00013*nmvoc;0.000006*nmvoc;0;0;0;0.48*pm25;0.15*pm25;0;0;pm25*0.00427;pm25*0.04911;pm25*0.17393;pm25*0.00506;pm25*0.00919;pm25*0.00042;pm25*0.00009;pm25*0.00463;pm25*0.01719;pm25*0.0031;pm25*0.00277;pm25*0.00011;pm25*0.04219;pm25*0.05788;pm10-pm25 +E096;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E097;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E098;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E099;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E100;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0.01*voc18+0.3*voc19;0;0;0;0.5*voc01;0;0;0;0.5*voc01;0;7.5*voc06+2.2*voc17+4.11*voc18+4*voc19+4*voc23;0;0;voc14+0.2*voc17;voc15+voc17;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25;pm10-pm25 +E101;0.95*nox_no2;0.042*nox_no2;0.008*nox_no2;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+1.875*voc22+4*voc23;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E102;0.7*nox_no2;0.283*nox_no2;0.017*nox_no2;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+2.2*voc17+1.875*voc22+4*voc23;0;0;voc14+0.2*voc17;voc15+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E103;0.95*nox_no2;0.042*nox_no2;0.008*nox_no2;co;so2;nh3;0.625*voc22;0.375*voc22;0;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+voc08+2.2*voc17+1.875*voc22;0;0;0.2*voc17;voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E104;0;0;0;0;0;0;0;0;voc13;0;0;0;0;0.666*voc12;0;0;0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc13;0;0;voc14;voc15;0;0;0;oc;ec;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec;pm10-pm25 +E105;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+1.875*voc22+4*voc23;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E106;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+5*voc05+7.5*voc06+voc08+voc09+voc13+2.2*voc17+1.875*voc22+4*voc23;0;0;voc14+0.2*voc17;voc15+voc17;0;0;0;oc;0;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-so4;pm10-pm25 +E107;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E108;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec;pm10-pm25 +E109;0;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;oc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc;pm10-pm25 +E110;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec;pm10-pm25 +E902;0;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E903;0;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E906;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;oc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc;pm10-pm25 +E907;0.9*nox_no2;0.1*nox_no2;0;co;so2;0;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec;pm10-pm25 +E927;nox_no2;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E928;0;0;0;co;0;0;0.001591*nmvoc;0.000103*nmvoc;0.000038*nmvoc;0;0;0.000022*nmvoc;0;0;0;0;0.000023*nmvoc;0.03941*nmvoc;0;0.000007*nmvoc;0.000237*nmvoc;0;0;0;0;0.318*pm25;0.0516*pm25;0;0.0446*pm25;pm25*0.00427;pm25*0.04911;pm25*0.17393;pm25*0.00506;pm25*0.00919;pm25*0.00042;pm25*0.00009;pm25*0.00463;pm25*0.01719;pm25*0.0031;pm25*0.00277;pm25*0.00011;pm25*0.04219;pm25*0.05788;pm10-pm25 +E929;0;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E930;0.9*nox_no2;0.1*nox_no2;0;co;so2;0;0.002093*nmvoc;0.000525*nmvoc;0.000105*nmvoc;0.002929*nmvoc;0.001894*nmvoc;0.000023*nmvoc;0.002826*nmvoc;0.000116*nmvoc;0.000043*nmvoc;0.003664*nmvoc;0.003078*nmvoc;0.010261*nmvoc;0;0.000006*nmvoc;0.00013*nmvoc;0.000006*nmvoc;0;0;0;0.48*pm25;0.15*pm25;0;0;pm25*0.00427;pm25*0.04911;pm25*0.17393;pm25*0.00506;pm25*0.00919;pm25*0.00042;pm25*0.00009;pm25*0.00463;pm25*0.01719;pm25*0.0031;pm25*0.00277;pm25*0.00011;pm25*0.04219;pm25*0.05788;pm10-pm25 +E114;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E115;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;5.6*(oc+bc)-oc-bc;3.4*(oc+bc) +E116;0.9*nox_no;0.1*nox_no;0;co;so2;0;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E117;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;3.1*(oc+bc)-oc-bc;0.6*(oc+bc) +E118;nox_no;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E119;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;1.7*(oc+bc)-oc-bc;1.5*(oc+bc) +E120;0;0;0;0;0;0;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E121;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E122;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;3.4*(oc+bc)-oc-bc;0 +E123;0.7*nox_no;0.283*nox_no;0.017*nox_no;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;1.2*(oc+bc)-oc-bc;0 + diff --git a/data/profiles/temporal/TemporalProfile_Hourly.csv b/data/profiles/temporal/TemporalProfile_Hourly.csv index 0434817..6a65ae5 100755 --- a/data/profiles/temporal/TemporalProfile_Hourly.csv +++ b/data/profiles/temporal/TemporalProfile_Hourly.csv @@ -9,4 +9,7 @@ H007,0.6,0.6,0.6,0.6,0.6,0.65,0.75,0.9,1.1,1.35,1.45,1.6,1.65,1.75,1.7,1.55,1.35 H008,0.06,0.168,0.192,0.18,0.24,0.192,0.204,0.216,0.336,0.6,1.08,1.8,2.52,3.12,3.12,3.12,2.64,1.68,0.96,0.72,0.42,0.36,0.036,0.036 H009,0.597235,0.552995,0.497696,0.508756,0.575115,0.741014,0.917972,1.15023,1.32719,1.42673,1.45991,1.43779,1.39355,1.37143,1.36037,1.29401,1.24977,1.19447,1.09493,0.973272,0.829493,0.729954,0.685714,0.630415 H010,0.613383,0.557621,0.490706,0.468401,0.479554,0.568773,0.669145,0.814126,0.97026,1.12639,1.22677,1.27138,1.29368,1.3829,1.46097,1.47212,1.46097,1.46097,1.3829,1.23792,1.08178,0.936803,0.847584,0.724907 +H011,0.06,0.06,0.06,0.07,0.07,0.07,0.2,0.2,0.2,1.82,1.82,1.82,3.39,3.39,3.39,1.68,1.68,1.68,0.56,0.56,0.56,0.22,0.22,0.22 +H012,0.61,0.57,0.48,0.36,0.29,0.25,0.21,0.25,0.4,0.74,1.01,1.38,1.57,1.71,1.8,1.86,1.86,1.79,1.6,1.43,1.24,1.03,0.86,0.7 +H013,0.28,0.13,0.1,0.08,0.09,0.09,0.44,0.91,1.18,1.43,1.6,1.61,1.66,1.58,1.47,1.43,1.42,1.46,1.46,1.4,1.39,1.25,0.96,0.58 diff --git a/data/profiles/temporal/TemporalProfile_Monthly.csv b/data/profiles/temporal/TemporalProfile_Monthly.csv index 1c2108f..347292f 100755 --- a/data/profiles/temporal/TemporalProfile_Monthly.csv +++ b/data/profiles/temporal/TemporalProfile_Monthly.csv @@ -8,5 +8,8 @@ M006,0.95,0.96,1.02,1,1.01,1.03,1.03,1.01,1.04,1.03,1.01,0.91 M007,0.88,0.92,0.98,1.03,1.05,1.06,1.01,1.02,1.06,1.05,1.01,0.93 M008,0.88,0.92,0.98,1.03,1.05,1.06,1.01,1.02,1.06,1.05,1.01,0.93 M009,0.45,1.3,2.35,1.7,0.85,0.85,0.85,1,1.1,0.65,0.45,0.45 -M999,0,0,0,0,0,0,0,0,1,0,0,0 +M010,0.7,0.75,0.85,0.9,1,1.1,1.2,1.25,1.3,1.1,1,0.85 +M011,0,2,4.75,2.9,0.5,0.4,0.2,0.5,0.75,0,0,0 +M012,0.740123177,0.685722463,0.741070559,0.840611444,1.003188502,1.206099053,1.535051489,1.500726051,1.233704586,1.026921016,0.771651666,0.715129995 + diff --git a/data/profiles/vertical/Vertical_profile.csv b/data/profiles/vertical/Vertical_profile.csv index 2518a5a..3add62a 100755 --- a/data/profiles/vertical/Vertical_profile.csv +++ b/data/profiles/vertical/Vertical_profile.csv @@ -4,4 +4,5 @@ V002;36,72,108,144,218,292,366,440,514,588,724,896,1050,1204,1358;0,0.18,0.47,0. V003;10,1000;0.6,0.4 V004;1000,9000;0,1 V005;9000,12000;0,1 -V006;36,72,108,144,218,292,366,440,514,588,724,896,1050,1204,1358;0,0,0.02,0.26,0.56,0.13,0.03,0,0,0,0,0,0,0,0 \ No newline at end of file +V006;36,72,108,144,218,292,366,440,514,588,724,896,1050,1204,1358;0,0,0.02,0.26,0.56,0.13,0.03,0,0,0,0,0,0,0,0 +V007;20,92,184,324,522,781,1106;0,0,0.41,0.57,0.02,0,0 \ No newline at end of file -- GitLab From 215de72998a2a96bfb62c4707c88c7b49af8f2b8 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 31 Oct 2019 12:56:22 +0100 Subject: [PATCH 20/28] Changed get area function changed TFLAG creation Writting set to serial mode --- hermesv3_gr/config/settings.py | 2 +- hermesv3_gr/modules/grids/grid.py | 10 +------- hermesv3_gr/modules/writing/writer_cmaq.py | 30 +++++++++++++++------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index 038f004..8c9061d 100755 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -28,7 +28,7 @@ global precision precision = np.float64 global writing_serial -writing_serial = False +writing_serial = True global compressed_netcdf compressed_netcdf = True diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index 520b56b..ec43a21 100755 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -297,19 +297,11 @@ class Grid(object): :rtype: numpy.array """ from cdo import Cdo - from netCDF4 import Dataset - st_time = timeit.default_timer() settings.write_log('\t\tGetting cell area from {0}'.format(self.coords_netcdf_file), level=3) - # Initialises the CDO cdo = Cdo() - # Create a temporal file 's' with the cell area - s = cdo.gridarea(input=self.coords_netcdf_file) - # Get the cell area of the temporal file - nc_aux = Dataset(s, mode='r') - cell_area = nc_aux.variables['cell_area'][:] - nc_aux.close() + cell_area = cdo.gridarea(input=self.coords_netcdf_file, returnArray='cell_area') settings.write_time('Grid', 'get_cell_area', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 3c88760..5e238c5 100755 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -144,16 +144,27 @@ class WriterCmaq(Writer): :rtype: numpy.array """ from datetime import timedelta + t_flag = np.empty((len(hours_array), num_vars, 2)) - a = np.array([[[]]]) - - for inc_hours in hours_array: + for i_d, date in enumerate(hours_array): date = st_date + timedelta(hours=inc_hours) - b = np.array([[int(date.strftime('%Y%j'))], [int(date.strftime('%H%M%S'))]] * num_vars) - a = np.append(a, b) - - a.shape = (len(hours_array), 2, num_vars) - return a + y_d = int(date.strftime('%Y%j')) + hms = int(date.strftime('%H%M%S')) + for i_p in range(num_vars): + t_flag[i_d, i_p, 0] = y_d + t_flag[i_d, i_p, 1] = hms + + return t_flag + + # a = np.array([[[]]]) + # + # for inc_hours in hours_array: + # date = st_date + timedelta(hours=inc_hours) + # b = np.array([[int(date.strftime('%Y%j'))], [int(date.strftime('%H%M%S'))]] * num_vars) + # a = np.append(a, b) + # + # a.shape = (len(hours_array), 2, num_vars) + # return a @staticmethod def str_var_list(var_list): @@ -498,7 +509,8 @@ class WriterCmaq(Writer): tflag = netcdf.createVariable('TFLAG', 'i', ('TSTEP', 'VAR', 'DATE-TIME',)) tflag.setncatts({'units': "{:<16}".format(''), 'long_name': "{:<16}".format('TFLAG'), 'var_desc': "{:<80}".format('Timestep-valid flags: (1) YYYYDDD or (2) HHMMSS')}) - tflag[:] = self.create_tflag(self.date, self.hours, len(self.variables_attributes)) + var_tflag = self.create_tflag(self.date, self.hours, len(self.variables_attributes)) + tflag[:] = var_tflag settings.write_log("\t\t\t'TFLAG' variable created with size: {0}".format(tflag[:].shape), level=3) full_shape = None -- GitLab From 68f04cc963f4f6aa9214369fb59812874e85f101 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 31 Oct 2019 14:20:06 +0100 Subject: [PATCH 21/28] solved bug --- hermesv3_gr/modules/writing/writer_cmaq.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 5e238c5..71242c1 100755 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -146,7 +146,7 @@ class WriterCmaq(Writer): from datetime import timedelta t_flag = np.empty((len(hours_array), num_vars, 2)) - for i_d, date in enumerate(hours_array): + for i_d, inc_hours in enumerate(hours_array): date = st_date + timedelta(hours=inc_hours) y_d = int(date.strftime('%Y%j')) hms = int(date.strftime('%H%M%S')) @@ -156,16 +156,6 @@ class WriterCmaq(Writer): return t_flag - # a = np.array([[[]]]) - # - # for inc_hours in hours_array: - # date = st_date + timedelta(hours=inc_hours) - # b = np.array([[int(date.strftime('%Y%j'))], [int(date.strftime('%H%M%S'))]] * num_vars) - # a = np.append(a, b) - # - # a.shape = (len(hours_array), 2, num_vars) - # return a - @staticmethod def str_var_list(var_list): """ -- GitLab From d799bd96ccb99f83e2755b95b0df920f7e379d9d Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 6 Nov 2019 10:13:47 +0100 Subject: [PATCH 22/28] Correcting bugs --- hermesv3_gr/modules/masking/masking.py | 28 +++----------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/hermesv3_gr/modules/masking/masking.py b/hermesv3_gr/modules/masking/masking.py index f02f736..90670ca 100755 --- a/hermesv3_gr/modules/masking/masking.py +++ b/hermesv3_gr/modules/masking/masking.py @@ -85,29 +85,6 @@ class Masking(object): settings.write_time('Masking', 'get_country_codes', timeit.default_timer() - st_time, level=3) return countries_dict - @staticmethod - def partlst(lst, num): - """ - Split a Array in N balanced arrays. - - :param lst: Array to split - :type lst: numpy.array - - :param num: Number of mini arrays. - :type num: int - - :return: Array - :type: numpy.array - """ - import itertools - # Partition @lst in @n balanced parts, in given order - parts, rest = divmod(len(lst), num) - lstiter = iter(lst) - for j in range(num): - plen = len(lst) / num + (1 if rest > 0 else 0) - rest -= 1 - yield list(itertools.islice(lstiter, plen)) - def create_country_iso(self, in_nc): import numpy as np from hermesv3_gr.tools.netcdf_tools import extract_vars @@ -123,9 +100,10 @@ class Masking(object): dst_var = [] num = 0 - points = np.array(list(zip(lat.flatten(), lon.flatten()))) + aux_list = list(zip(lat.flatten(), lon.flatten())) + points = np.array(aux_list[:]) - points_list = list(self.partlst(points, settings.size)) + points_list = np.array_split(points, settings.size) for lat_aux, lon_aux in points_list[settings.rank]: num += 1 -- GitLab From 16209bd4020559465f423f7e0b2c6dbd7c04f914 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 7 Nov 2019 18:40:30 +0100 Subject: [PATCH 23/28] Solving Bugs (auto-CALIOPE): - HERMESv3_GR: temporal.py (isin) - HERMESv3_GR: World Mask - HERMESv3_GR: preproc emep SO2 - do not remove rundir --- hermesv3_gr/config/settings.py | 4 ++-- hermesv3_gr/modules/grids/grid.py | 4 ---- hermesv3_gr/modules/temporal/temporal.py | 2 +- hermesv3_gr/modules/writing/writer.py | 4 ++-- preproc/emep_preproc.py | 9 ++++++--- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index 8c9061d..893a739 100755 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -59,8 +59,8 @@ def define_global_vars(in_log_level, mycomm): icomm = MPI.COMM_WORLD else: icomm = mycomm - - comm = icomm.Split(color=0, key=0) + comm = icomm + #comm = icomm.Split(color=0, key=0) rank = comm.Get_rank() size = comm.Get_size() diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index ec43a21..a66f21f 100755 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -57,10 +57,6 @@ class Grid(object): self.boundary_longitudes = None self.cell_area = None - if settings.rank == 0: - if not os.path.exists(os.path.join(temporal_path)): - os.makedirs(os.path.join(temporal_path)) - settings.comm.Barrier() self.coords_netcdf_file = os.path.join(temporal_path, 'temporal_coords.nc') self.temporal_path = temporal_path diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 6f2c457..e1630aa 100755 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -195,7 +195,7 @@ class TemporalDistribution(object): if self.grid.grid_type == 'rotated': lat_name = 'rlat' lon_name = 'rlon' - elif self.grid.grid_type.isin(['mercator', 'lcc']): + elif self.grid.grid_type in ['mercator', 'lcc']: lat_name = 'x' lon_name = 'y' else: diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index 535d50e..9fb1d1a 100755 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -388,7 +388,7 @@ class Writer(object): lat_dim = ('lat',) elif len(center_latitudes.shape) == 2: netcdf.createDimension('lat', center_latitudes.shape[0]) - lat_dim = ('lon', 'lat',) + lat_dim = ('lat', 'lon',) else: print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) @@ -399,7 +399,7 @@ class Writer(object): lon_dim = ('lon',) elif len(center_longitudes.shape) == 2: netcdf.createDimension('lon', center_longitudes.shape[1]) - lon_dim = ('lon', 'lat',) + lon_dim = ('lat', 'lon',) else: print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( len(center_longitudes.shape))) diff --git a/preproc/emep_preproc.py b/preproc/emep_preproc.py index 72c2ff3..23bdf30 100755 --- a/preproc/emep_preproc.py +++ b/preproc/emep_preproc.py @@ -30,10 +30,11 @@ Besides citing HERMESv3_GR, users must also acknowledge the use of the correspon """ # ============== CONFIGURATION PARAMETERS ====================== -INPUT_PATH = '/esarchive/recon/ceip/emepv18/original_files' -OUTPUT_PATH = '/esarchive/recon/ceip/emepv18/yearly_mean' +INPUT_PATH = '/esarchive/recon/ceip/emepv19/original_files' +OUTPUT_PATH = '/esarchive/recon/ceip/emepv19/yearly_mean' INPUT_NAME = '__2018_GRID_.txt' -# list_years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016] +# list_years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, +# 2017] LIST_YEARS = [2015] LIST_POLLUTANTS = ['NOx', 'NMVOC', 'SOx', 'NH3', 'PM2_5', 'PM10', 'CO'] # ============================================================== @@ -105,6 +106,8 @@ def do_transformation(year): name = 'nox_no2' elif name == 'pm2_5': name = 'pm25' + elif name == 'sox': + name = 'so2' elif name == 'voc': name = 'nmvoc' -- GitLab From 448d4e99ee52f12d5d25a8ab9d5e2a3ebe92d8ee Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 8 Nov 2019 09:59:02 +0100 Subject: [PATCH 24/28] PEP8 corrections --- hermesv3_gr/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index 893a739..8a6966b 100755 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -60,7 +60,7 @@ def define_global_vars(in_log_level, mycomm): else: icomm = mycomm comm = icomm - #comm = icomm.Split(color=0, key=0) + # comm = icomm.Split(color=0, key=0) rank = comm.Get_rank() size = comm.Get_size() -- GitLab From ae14fb691d8958ce99e75ebcdb4b8c0dbb02e286 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 8 Nov 2019 10:09:59 +0100 Subject: [PATCH 25/28] Corrected bug on NetCDF creation --- hermesv3_gr/tools/netcdf_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index 8678e9f..9e0fa35 100755 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -429,7 +429,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lat_dim = ('lat',) elif len(center_latitudes.shape) == 2: netcdf.createDimension('lat', center_latitudes.shape[0]) - lat_dim = ('lon', 'lat', ) + lat_dim = ('lat', 'lon', ) else: print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) @@ -440,7 +440,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lon_dim = ('lon',) elif len(center_longitudes.shape) == 2: netcdf.createDimension('lon', center_longitudes.shape[1]) - lon_dim = ('lon', 'lat', ) + lon_dim = ('lat', 'lon', ) else: print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) -- GitLab From 12834c0f99a0dd6dfd9e4e4f0f0cd1009ba19564 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 13 Nov 2019 10:24:20 +0100 Subject: [PATCH 26/28] Setting WithCopyWarning removed --- preproc/tno_mac_iii_preproc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/preproc/tno_mac_iii_preproc.py b/preproc/tno_mac_iii_preproc.py index 6c06388..3b447d6 100755 --- a/preproc/tno_mac_iii_preproc.py +++ b/preproc/tno_mac_iii_preproc.py @@ -152,8 +152,8 @@ def do_transformation(year): dataframe = pd.read_table(in_file, sep=';') - df_np = dataframe[dataframe.SourceType != 'P'] - df_p = dataframe[dataframe.SourceType == 'P'] + df_np = dataframe[dataframe.SourceType != 'P'].copy() + df_p = dataframe[dataframe.SourceType == 'P'].copy() df_np.loc[:, 'row_lat'] = np.array((df_np.Lat - (-90 + lat_interval / 2)) / lat_interval, dtype=np.int32) df_np.loc[:, 'col_lon'] = np.array((df_np.Lon - (-180 + lon_interval / 2)) / lon_interval, dtype=np.int32) -- GitLab From 34489e0df5522b317d4363705d32d793e7420207 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 19 Nov 2019 11:18:06 +0100 Subject: [PATCH 27/28] Changed Hermes class name to HermesGr --- hermesv3_gr/hermes.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index bb047b9..1df2970 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -25,14 +25,13 @@ from hermesv3_gr.modules.emision_inventories.emission_inventory import EmissionI from hermesv3_gr.modules.vertical.vertical import VerticalDistribution from hermesv3_gr.tools.netcdf_tools import * -# import pyextrae.sequential as pyextrae global full_time -class Hermes(object): +class HermesGr(object): """ - Interface class for HERMESv3. + Interface class for HERMESv3_GR. """ def __init__(self, config, new_date=None, comm=None): from hermesv3_gr.modules.grids.grid import Grid @@ -150,9 +149,9 @@ class Hermes(object): def run(): - date = Hermes(Config()).main() + date = HermesGr(Config()).main() while date is not None: - date = Hermes(Config(), new_date=date).main() + date = HermesGr(Config(), new_date=date).main() sys.exit(0) -- GitLab From 7ecf34cd9d6e03cdfcdad68a4b880701d279fa8c Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 20 Nov 2019 13:16:42 +0100 Subject: [PATCH 28/28] Update date of version --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 032fc29..fdfe3ac 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,5 @@ 2.0.0 - 2019/10/25 + 2019/11/20 - Python 3 - first_time option - erase_auxiliary_files option -- GitLab