diff --git a/.codacy.yml b/.codacy.yml index 3e730e5f2f81b1a6fdeaf83ae87f1e092dfb3218..bf12fc2d77140a852ff1b9197370766a0ecec7c3 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -13,9 +13,9 @@ engines: enabled: true pylint: enabled: true - python_version: 3 + python_version: 2 exclude_paths: [ 'doc/**', - 'earthdiagnostics/cmor_tables/**', + 'data/**', ] diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87db4e1a3fe1d0407d90a920862f916c6cfd9edd..08c1355a02d158efce8b3c8438df86fdc3275b6e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,8 @@ test_python2: - conda env update -f environment.yml -n hermesv3_gr python=2.7 - source activate hermesv3_gr - python run_test.py + - pip install codacy-coverage --upgrade + - python-codacy-coverage -r tests/report/python2/coverage.xml #test_python3: # stage: test @@ -33,13 +35,6 @@ test_python2: # - source activate earthdiagnostics3 # - python run_test.py -report_codacy: - stage: report - script: - - source activate hermesv3_gr - - pip install codacy-coverage --upgrade - - python-codacy-coverage -r test/report/python2/coverage.xml - clean: stage: clean script: diff --git a/.pylintrc b/.pylintrc index 4094c1f4b05e85ba956aa1ff927fd5c0ae0c2646..db7741b994ec5e5720e803d9e73b639a55435ff8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -99,7 +99,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme [FORMAT] # Maximum number of characters on a single line. -max-line-length=79 +max-line-length=120 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ diff --git a/CHANGELOG b/CHANGELOG index 2409f3cb18e3cb457c5a6998578ebc5f254218d9..02048fdcba12b560db50a1b27b87a56b51af2ce6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,2 +1,4 @@ 0.0.0 - HERMESv3_GR first release \ No newline at end of file + 2018/09/18 + + HERMESv3_GR beta version first release \ No newline at end of file diff --git a/README.md b/README.md index ca8c8065db989e0081c45e7ec9af8f2034d0ba00..b393f1f503d737afb63e16c85f268c9cbc7ee09c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # HERMESv3 Global/Regional -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/34fc5d6c803444178034b99dd28c7e3c)](https://www.codacy.com/app/carlestena/hermesv3_gr?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/hermesv3_gr&utm_campaign=Badge_Grade) \ No newline at end of file +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/34fc5d6c803444178034b99dd28c7e3c)](https://www.codacy.com/app/carlestena/hermesv3_gr?utm_source=earth.bsc.es&utm_medium=referral&utm_content=gitlab/es/hermesv3_gr&utm_campaign=Badge_Grade) diff --git a/VERSION b/VERSION deleted file mode 100644 index bd52db81d0cdfa45b8ccaf4e811f178160eb261e..0000000000000000000000000000000000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.0 \ No newline at end of file diff --git a/conf/EI_configuration.csv b/conf/EI_configuration.csv index d3663317901b24b365d7e9d709c8189d27edea06..15e01cb4eca5afdab31acadb3e58ca17748c3572 100644 --- a/conf/EI_configuration.csv +++ b/conf/EI_configuration.csv @@ -1,17 +1,17 @@ ei;sector;ref_year;active;factor_mask;regrid_mask;pollutants;path;frequency;source_type;p_vertical;p_month;p_day;p_hour;p_speciation;comment HTAPv2;energy;2010;0;;;so2;/jrc/htapv2/monthly_mean;monthly;area;V001;;D002;H002;E998;added 05/2017 HTAPv2;industry;2010;0;;;so2;/jrc/htapv2/monthly_mean;monthly;area;V002;;D003;H004;E998;added 05/2017 -HTAPv2;residential;2010;1;;- FRA;so2;/jrc/htapv2/monthly_mean;monthly;area;;;;;E998;added 05/2017 -HTAPv2;residential;2010;1;FRA 50;+ FRA;so2;/jrc/htapv2/monthly_mean;monthly;area;;;;;E998;added 05/2017 +HTAPv2;residential;2010;0;;- FRA;so2;/jrc/htapv2/monthly_mean;monthly;area;;;;;E998;added 05/2017 +HTAPv2;residential;2010;0;FRA 50;+ FRA;so2;/jrc/htapv2/monthly_mean;monthly;area;;;;;E998;added 05/2017 HTAPv2;transport;2010;1;;;so2;/jrc/htapv2/monthly_mean;monthly;area;;;D001;weekday=H001, saturday=H002, sunday=H003;E998;added 05/2017 -HTAPv2;air_lto;2010;1;;;so2;/jrc/htapv2/yearly_mean;yearly;area;V003;M001;D001;H001;E998;added 05/2017 +HTAPv2;air_lto;2010;0;;;so2;/jrc/htapv2/yearly_mean;yearly;area;V003;M001;D001;H001;E998;added 05/2017 HTAPv2;air_cds;2010;0;;;so2;/jrc/htapv2/yearly_mean;yearly;area;V004;M001;D001;H001;E998;added 05/2017 HTAPv2;air_crs;2010;0;;;so2;/jrc/htapv2/yearly_mean;yearly;area;V005;M001;D001;H001;E998;added 05/2017 HTAPv2;ships;2010;0;;;so2;/jrc/htapv2/yearly_mean;yearly;area;;M001;D001;H001;E008;added 05/2017 wiedinmyer;;2014;0;;;so2;/ucar/wiedinmyer/yearly_mean;yearly;area;;M001;D001;H001;E998;added 05/2017 ECLIPSEv5a;flaring;2010;0;;;so2;/iiasa/eclipsev5a/yearly_mean;area;yearly;V006;M001;D001;H001;E998;added 11/2017 -GFASv12;;2015;1;;;so2,nox_no;/ecmwf/gfas/daily_mean;daily;area;method=sovief,approach=uniform;;;H001;E997;added 05/2017 +GFASv12;;2015;0;;;so2,nox_no;/ecmwf/gfas/daily_mean;daily;area;method=sovief,approach=uniform;;;H001;E997;added 05/2017 ECLIPSEv5a;transport;2010;0;;+ CHN,IND;so2;/iiasa/eclipsev5a/monthly_mean;monthly;area;;;D001;H001;E998;added 11/2017 ECLIPSEv5a;transport;2010;0;;;nox_no2;/iiasa/eclipsev5a/monthly_mean;monthly;area;;;D005;weekday=H006, saturday=H009, sunday=H010;E999;added 11/2017 -CARN;;2015;1;;;so2;/mtu/carnetal/yearly_mean;yearly;point;;M001;D001;H001;E998;added ... +CARN;;2015;0;;;so2;/mtu/carnetal/yearly_mean;yearly;point;;M001;D001;H001;E998;added ... Maestra;;2015;0;;;nox_no2;/home/Earth/ctena/Models/HERMESv3/;yearly;point;;M001;D001;H001;E999;added ... diff --git a/conf/hermes.conf b/conf/hermes.conf index 36f5cc2b6c98ac524b9afbf845eb49ce9914983f..c89314107b2cd85334f687b426adc9011d3574ba 100644 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -1,12 +1,12 @@ [GENERAL] log_level = 3 # input_dir = /gpfs/projects/bsc32/bsc32538/HERMESv3_GR_rotated/IN -input_dir = /home/Earth/ctena/Models/HERMESv3/IN +input_dir = /home/Earth/ctena/Models/hermesv3_gr # data_path = /gpfs/scratch/bsc32/bsc32538/HERMES_data data_path = /esarchive/recon #output_dir = /gpfs/projects/bsc32/bsc32538/HERMESv3_GR_rotated/OUT -output_dir = /home/Earth/ctena/Models/HERMESv3/OUT -output_name = HERMES_paralel_.nc +output_dir = /home/Earth/ctena/HERMES_out +output_name = HERMESv3_.nc start_date = 2014/09/02 00:00:00 # ***** end_date = start_date [DEFAULT] ***** # end_date = 2014/09/03 00:00:00 @@ -89,12 +89,7 @@ auxiliar_files_path = /data/auxiliar_files/_ [EMISSION_INVENTORY_CONFIGURATION] -# cross_table = /conf/EI_configuration_Scalability.csv -# cross_table = /conf/EI_configuration_gridded.csv -# cross_table = /conf/EI_configuration_test.csv -# cross_table = /conf/EI_configuration_publi.csv -# cross_table = /conf/EI_configuration_EU_aerosol_gas.csv -cross_table = /conf/EI_configuration_WRF_CHEM_Rene.csv +cross_table = /conf/EI_configuration.csv [EMISSION_INVENTORY_PROFILES] diff --git a/data/profiles/vertical/1layer_vertical_description.csv b/data/profiles/vertical/1layer_vertical_description.csv deleted file mode 100644 index defd16d075dab4ef66a4c76e4302035e86839a71..0000000000000000000000000000000000000000 --- a/data/profiles/vertical/1layer_vertical_description.csv +++ /dev/null @@ -1,2 +0,0 @@ -Ilayer;height_magl -1;1000 \ No newline at end of file diff --git a/environment.yml b/environment.yml index 168dd4e968dfae94e1af136ef359ebb80efd08ae..29166185767038062869839fff90a5db7c1ed2f4 100644 --- a/environment.yml +++ b/environment.yml @@ -6,9 +6,10 @@ channels: - conda-forge dependencies: + - python = 2 - numpy - netcdf4 >= 1.3.1 - - python-cdo >= 1.3.4 + - python-cdo >= 1.3.3 - geopandas - pyproj - configargparse @@ -17,6 +18,10 @@ dependencies: - pytz - timezonefinder - mpi4py + # Testing + - pytest + - pytest-cov + - pycodestyle - pip: - holidays diff --git a/hermesv3_gr/__init__.py b/hermesv3_gr/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..6c8e6b979c5f58121ac7ee2d9e024749da3a8ce1 100644 --- a/hermesv3_gr/__init__.py +++ b/hermesv3_gr/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.0" diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index c2b1ad21207c74efa6f6ee7c6a297f0ecc74674e..36d9db4dffb2971102fca7702ca350df2ab74aa0 100644 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -52,7 +52,7 @@ class Config(ArgParser): help="Name of the output file. You can add the string '' that will be substitute by the " + "starting date of the simulation day.") p.add_argument('--start_date', required=True, help='Starting Date to simulate (UTC)') - p.add_argument('--end_date', required=False, + p.add_argument('--end_date', required=False, default=None, help='If you want to simulate more than one day you have to specify the ending date of ' + 'simulation in this parameter. If it is not set end_date = start_date.') @@ -166,8 +166,8 @@ class Config(ArgParser): exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( item, options.inc_x, options.inc_y)) - options.start_date = self._parse_start_date(str(options.start_date)) - options.end_date = self._parse_end_date(str(options.end_date), options.start_date) + options.start_date = self._parse_start_date(options.start_date) + options.end_date = self._parse_end_date(options.end_date, options.start_date) self.create_dir(options.output_dir) self.create_dir(options.auxiliar_files_path) @@ -202,7 +202,7 @@ class Config(ArgParser): @staticmethod def create_dir(path): """ - Creates the given folder if it is not created yet. + Create the given folder if it is not created yet. :param path: Path to create. :type path: str @@ -248,7 +248,7 @@ class Config(ArgParser): @staticmethod def _parse_start_date(str_date): """ - Parses the date form string to datetime. + Parse the date form string to datetime. It accepts several ways to introduce the date: YYYYMMDD, YYYY/MM/DD, YYYYMMDDhh, YYYYYMMDD.hh, YYYY/MM/DD_hh:mm:ss, YYYY-MM-DD_hh:mm:ss, YYYY/MM/DD hh:mm:ss, YYYY-MM-DD hh:mm:ss, YYYY/MM/DD_hh, YYYY-MM-DD_hh. @@ -279,7 +279,7 @@ class Config(ArgParser): def _parse_end_date(self, end_date, start_date): """ - Parses the end date. + Parse the end date. If it's not defined it will be the same date that start_date (to do only one day). :param end_date: Date to the last day to simulate in string format. diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index d16ac27f8ed6f56efd71b48d42e28677e32935a0..1b93cfa1182ba517531a30737ae119b8c81f5942 100644 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -30,7 +30,7 @@ global writing_serial writing_serial = False global compressed_netcdf -compressed_netcdf = False +compressed_netcdf = True if not writing_serial: compressed_netcdf = False @@ -53,7 +53,7 @@ def define_global_vars(in_log_level): global comm global rank global size - + icomm = MPI.COMM_WORLD comm = icomm.Split(color=0, key=0) rank = comm.Get_rank() @@ -67,8 +67,9 @@ def define_log_file(log_path, date): # TODO Documentation log_path = os.path.join(log_path, 'logs') if not os.path.exists(log_path): - os.makedirs(log_path) - + if rank == 0: + os.makedirs(log_path) + comm.Barrier() log_path = os.path.join(log_path, 'HERMESv3_{0}_Rank{1}_Procs{2}.log'.format( date.strftime('%Y%m%d%H'), str(rank).zfill(4), str(size).zfill(4))) if os.path.exists(log_path): @@ -115,7 +116,8 @@ def finish_logs(output_dir, date): date.strftime('%Y%m%d%H'), str(size).zfill(4))) if os.path.exists(times_path): os.remove(times_path) - df_merged = reduce(lambda left, right: pd.merge(left, right, on=['Class', 'Function'], how='outer'), data_frames) + df_merged = reduce(lambda left, right: pd.merge(left, right, on=['Class', 'Function'], how='outer'), + data_frames) df_merged['min'] = df_merged.loc[:, range(size)].min(axis=1) df_merged['max'] = df_merged.loc[:, range(size)].max(axis=1) df_merged['mean'] = df_merged.loc[:, range(size)].mean(axis=1) diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 2ca7ab683cde2ad3c9cbf1e6e09928686f0030b3..f094111a0b08295c87d3c21d873e2e816251ca66 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -19,7 +19,6 @@ import timeit - from hermesv3_gr.config import settings from hermesv3_gr.config.config import Config from hermesv3_gr.modules.emision_inventories.emission_inventory import EmissionInventory @@ -58,7 +57,8 @@ class Hermes(object): if self.options.output_model in ['CMAQ', 'WRF_CHEM'] and self.options.domain_type == 'global': settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: - raise AttributeError('ERROR: Global domain is not aviable for {0} output model.'.format(self.options.output_model)) + raise AttributeError('ERROR: Global domain is not aviable for {0} output model.'.format( + self.options.output_model)) sys.exit(1) self.levels = VerticalDistribution.get_vertical_output_profile(self.options.vertical_description) @@ -71,7 +71,8 @@ class Hermes(object): self.options.nx, self.options.ny, self.options.inc_x, self.options.inc_y, self.options.x_0, self.options.y_0, self.options.lat_ts) - self.emission_list = EmissionInventory.make_emission_list(self.options, self.grid, self.levels, self.options.start_date) + self.emission_list = EmissionInventory.make_emission_list(self.options, self.grid, self.levels, + self.options.start_date) self.delta_hours = TemporalDistribution.calculate_delta_hours( self.options.start_date, self.options.output_timestep_type, self.options.output_timestep_num, @@ -79,7 +80,8 @@ class Hermes(object): self.writer = Writer.get_writer( self.options.output_model, self.config.get_output_name(self.options.start_date), self.grid, - self.levels, self.options.start_date, self.delta_hours, self.options.output_attributes, compress=settings.compressed_netcdf, + self.levels, self.options.start_date, self.delta_hours, self.options.output_attributes, + compress=settings.compressed_netcdf, parallel=not settings.writing_serial) settings.write_log('End of HERMESv3 initialization.') @@ -90,13 +92,7 @@ class Hermes(object): """ Main functionality of the model. """ - from multiprocessing import Process, Queue, cpu_count - from threading import Thread - import copy - import gc - import numpy as np from datetime import timedelta - from cf_units import Unit st_time = timeit.default_timer() settings.write_log('') @@ -142,8 +138,12 @@ class Hermes(object): return None -if __name__ == '__main__': +def run(): date = Hermes(Config()).main() while date is not None: date = Hermes(Config(), new_date=date).main() sys.exit(0) + + +if __name__ == '__main__': + run() diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index 325994900ec3fbb5b0e25cce5fe4a84ee11de0d2..3101a2fccdef07220c15f2b448d56b2971c302d3 100644 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -156,11 +156,11 @@ class EmissionInventory(object): def create_pollutants_dicts(self, pollutants): """ - Creates a list of dictionaries with the information of the name, paht and Dataset of each pollutant - + Create a list of dictionaries with the information of the name, paht and Dataset of each pollutant + :param pollutants: List of pollutants names :type pollutants: list - + :return: List of dictionaries :rtype: list """ @@ -262,14 +262,14 @@ class EmissionInventory(object): @staticmethod def make_emission_list(options, grid, vertical_output_profile, date): """ - Extracts the information of the cross table to read all the needed emissions. + Extract the information of the cross table to read all the needed emissions. :param options: Full list of parameters given by passing argument or in the configuration file. :type options: Namespace - + :param grid: Grid to use. :type grid: Grid - + :param vertical_output_profile: Path to eht file that contains the vertical profile. :type vertical_output_profile: str diff --git a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py index e53c8da2066e2724e266d788765b408cd29e035c..9801db83cb6f5d30125c8f53910faa65e4bdd7e0 100755 --- a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py @@ -116,10 +116,10 @@ class GfasEmissionInventory(EmissionInventory): def get_altitude(self): """ - Extracts the altitude values depending on the choosen method. + Extract the altitude values depending on the choosen method. :return: Array with the alittude of each fire. - :rtype: numpy.ndarray + :rtype: numpy.array """ from hermesv3_gr.tools.netcdf_tools import extract_vars @@ -144,7 +144,7 @@ class GfasEmissionInventory(EmissionInventory): @ staticmethod def get_approach(p_vertical): """ - Extracts the given approach value. + Extract the given approach value. :return: Approach value :rtype: str @@ -167,7 +167,7 @@ class GfasEmissionInventory(EmissionInventory): @ staticmethod def get_method(p_vertical): """ - Extracts the given method value. + Extract the given method value. :return: Method value :rtype: str diff --git a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py index 2238a735b9bcc8828b9b9cbd258069e97d95e1dd..8c22e114aa539794cd8a57109f9cb83820ab6e0d 100755 --- a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py @@ -142,7 +142,7 @@ class PointSourceEmissionInventory(EmissionInventory): def calculate_altitudes(self, vertical_description_path): """ - Calculates the number layer to allocate the point source. + Calculate the number layer to allocate the point source. :param vertical_description_path: Path to the file that contains the vertical description :type vertical_description_path: str diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index 250ef8d3b586bd1d77aa243f48bd01e736be8b04..0c424d1ecd90aa27f068164d514c93216535a649 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -39,6 +39,7 @@ class Grid(object): :param temporal_path: Path to the temporal folder. :type temporal_path: str """ + def __init__(self, grid_type, vertical_description_path, temporal_path): st_time = timeit.default_timer() # settings.write_log('Creating Grid...', level=1) @@ -97,7 +98,7 @@ class Grid(object): lat_1, lat_2, lon_0, lat_0, nx, ny, inc_x, inc_y, x_0, y_0, lat_ts): # TODO describe better the rotated parameters """ - Creates a Grid object depending on the grid type. + Create a Grid object depending on the grid type. :param grid_type: type of grid to create [global, rotated, lcc, mercator] :type grid_type: str @@ -208,7 +209,7 @@ class Grid(object): @staticmethod def set_vertical_levels(vertical_description_path): """ - Extracts the vertical levels. + Extract the vertical levels. :param vertical_description_path: path to the file that contain the vertical description of the required output file. @@ -246,16 +247,16 @@ class Grid(object): write_netcdf(self.coords_netcdf_file, self.center_latitudes, self.center_longitudes, [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, - RegularLatLon=True) + regular_latlon=True) - # Calculates the cell area of the auxiliary NetCDF file + # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() # Re-writes the NetCDF adding the cell area write_netcdf(self.coords_netcdf_file, self.center_latitudes, self.center_longitudes, [{'name': 'var_aux', 'units': '', 'data': 0}], cell_area=self.cell_area, boundary_latitudes=self.boundary_latitudes, - boundary_longitudes=self.boundary_longitudes, RegularLatLon=True) + boundary_longitudes=self.boundary_longitudes, regular_latlon=True) else: self.cell_area = self.get_cell_area() @@ -263,7 +264,7 @@ class Grid(object): def get_cell_area(self): """ - Calculates the cell area of the grid. + Calculate the cell area of the grid. :return: Area of each cell of the grid. :rtype: numpy.array @@ -276,7 +277,7 @@ class Grid(object): # Initialises the CDO cdo = Cdo() - # Creates a temporal file 's' with the cell area + # Create a temporal file 's' with the cell area s = cdo.gridarea(input=self.coords_netcdf_file) # Get the cell area of the temporal file nc_aux = Dataset(s, mode='r') @@ -290,7 +291,7 @@ class Grid(object): @staticmethod def create_regular_grid_1d_array(center, inc, boundary): """ - Creates a regular grid giving the center, boundary and increment. + Create a regular grid giving the center, boundary and increment. :param center: Center of the coordinates. :type center: float @@ -307,11 +308,11 @@ class Grid(object): st_time = timeit.default_timer() - # Calculates first center point. + # Calculate first center point. origin = center - abs(boundary) - # Calculates the quantity of cells. + # Calculate the quantity of cells. n = (abs(boundary) / inc) * 2 - # Calculates all the values + # Calculate all the values values = np.arange(origin + inc, origin + (n * inc) - inc + inc / 2, inc, dtype=np.float) settings.write_time('Grid', 'create_regular_grid_1d_array', timeit.default_timer() - st_time, level=3) @@ -321,7 +322,7 @@ class Grid(object): @staticmethod def create_bounds(coords, inc, number_vertices=2, inverse=False): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) :type coords: numpy.array @@ -342,17 +343,17 @@ class Grid(object): st_time = timeit.default_timer() settings.write_log('\t\t\tCreating boundaries.', level=3) - # Creates new arrays moving the centers half increment less and more. + # Create new arrays moving the centers half increment less and more. coords_left = coords - inc / 2 coords_right = coords + inc / 2 # Defining the number of corners needed. 2 to regular grids and 4 for irregular ones. if number_vertices == 2: - # Creates an array of N arrays of 2 elements to store the floor and the ceil values for each cell + # Create an array of N arrays of 2 elements to store the floor and the ceil values for each cell bound_coords = np.dstack((coords_left, coords_right)) bound_coords = bound_coords.reshape((len(coords), number_vertices)) elif number_vertices == 4: - # Creates an array of N arrays of 4 elements to store the corner values for each cell + # Create an array of N arrays of 4 elements to store the corner values for each cell # It can be stored in clockwise starting form the left-top element, or in inverse mode. if inverse: bound_coords = np.dstack((coords_left, coords_left, coords_right, coords_right)) diff --git a/hermesv3_gr/modules/grids/grid_lcc.py b/hermesv3_gr/modules/grids/grid_lcc.py index 43c535fff48ed4f2e76c8ddc2678a5e52958b8f8..96ea0ec75b35862e4a462aa98438d26eed3ddb51 100644 --- a/hermesv3_gr/modules/grids/grid_lcc.py +++ b/hermesv3_gr/modules/grids/grid_lcc.py @@ -145,10 +145,10 @@ class LccGrid(Grid): write_netcdf(self.coords_netcdf_file, self.center_latitudes, self.center_longitudes, [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, - LambertConformalConic=True, lcc_x=self.x, lcc_y=self.y, + lcc=True, lcc_x=self.x, lcc_y=self.y, lat_1_2="{0}, {1}".format(self.lat_1, self.lat_2), lon_0=self.lon_0, lat_0=self.lat_0) - # Calculates the cell area of the auxiliary NetCDF file + # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() # Re-writes the NetCDF adding the cell area @@ -156,7 +156,7 @@ class LccGrid(Grid): [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, cell_area=self.cell_area, - LambertConformalConic=True, lcc_x=self.x, lcc_y=self.y, + lcc=True, lcc_x=self.x, lcc_y=self.y, lat_1_2="{0}, {1}".format(self.lat_1, self.lat_2), lon_0=self.lon_0, lat_0=self.lat_0) else: self.cell_area = self.get_cell_area() @@ -173,7 +173,7 @@ class LccGrid(Grid): st_time = timeit.default_timer() settings.write_log('\t\tCreating lcc coordinates', level=3) - # Creates a regular grid in metres (Two 1D arrays) + # Create a regular grid in metres (Two 1D arrays) self.x = np.arange(self.x_0, self.x_0 + self.inc_x * self.nx, self.inc_x, dtype=np.float) if len(self.x)//2 < settings.size: settings.write_log('ERROR: Check the .err file to get more info.') @@ -191,7 +191,7 @@ class LccGrid(Grid): y_b = super(LccGrid, self).create_bounds(y, self.inc_y, number_vertices=4, inverse=True) x_b = super(LccGrid, self).create_bounds(x, self.inc_x, number_vertices=4) - # Creates the LCC projection + # Create the LCC projection projection = Proj( proj='lcc', ellps='WGS84', diff --git a/hermesv3_gr/modules/grids/grid_mercator.py b/hermesv3_gr/modules/grids/grid_mercator.py index f5371309a5565ae2ff52252d5a1426a66a36f923..f3104fbf6c2ba50452c63219260c457aef06986b 100644 --- a/hermesv3_gr/modules/grids/grid_mercator.py +++ b/hermesv3_gr/modules/grids/grid_mercator.py @@ -132,9 +132,9 @@ class MercatorGrid(Grid): write_netcdf(self.coords_netcdf_file, self.center_latitudes, self.center_longitudes, [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, - Mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) + mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) - # Calculates the cell area of the auxiliary NetCDF file + # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() # Re-writes the NetCDF adding the cell area @@ -146,7 +146,7 @@ class MercatorGrid(Grid): ], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, cell_area=self.cell_area, - Mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) + mercator=True, lcc_x=self.x, lcc_y=self.y, lon_0=self.lon_0, lat_ts=self.lat_ts) else: self.cell_area = self.get_cell_area() @@ -161,7 +161,7 @@ class MercatorGrid(Grid): st_time = timeit.default_timer() - # Creates a regular grid in metres (Two 1D arrays) + # Create a regular grid in metres (Two 1D arrays) self.x = np.arange(self.x_0, self.x_0 + self.inc_x * self.nx, self.inc_x, dtype=np.float) if len(self.x)//2 < settings.size: settings.write_log('ERROR: Check the .err file to get more info.') @@ -179,7 +179,7 @@ class MercatorGrid(Grid): y_b = super(MercatorGrid, self).create_bounds(y, self.inc_y, number_vertices=4, inverse=True) x_b = super(MercatorGrid, self).create_bounds(x, self.inc_x, number_vertices=4) - # Creates the LCC projection + # Create the LCC projection projection = Proj(self.crs) # UTM to Mercator diff --git a/hermesv3_gr/modules/grids/grid_rotated.py b/hermesv3_gr/modules/grids/grid_rotated.py index 02dc6e4b25156aceb8363d5ce630688d8cfa248b..856630075c7ed241f39dbe6818bf0dae9fe8b18c 100644 --- a/hermesv3_gr/modules/grids/grid_rotated.py +++ b/hermesv3_gr/modules/grids/grid_rotated.py @@ -17,9 +17,10 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import timeit + import sys import os +import timeit import hermesv3_gr.config.settings as settings from grid import Grid @@ -126,7 +127,7 @@ class RotatedGrid(Grid): def rotated2latlon(self, lon_deg, lat_deg, lon_min=-180): """ - Calculates the unrotated coordinates using the rotated ones. + Calculate the unrotated coordinates using the rotated ones. :param lon_deg: Rotated longitude coordinate. :type lon_deg: numpy.array @@ -211,11 +212,11 @@ class RotatedGrid(Grid): [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, - Rotated=True, rotated_lats=self.rlat, rotated_lons=self.rlon, + roated=True, rotated_lats=self.rlat, rotated_lons=self.rlon, north_pole_lat=self.new_pole_latitude_degrees, north_pole_lon=self.new_pole_longitude_degrees) - # Calculates the cell area of the auxiliary NetCDF file + # Calculate the cell area of the auxiliary NetCDF file self.cell_area = self.get_cell_area() # Re-writes the NetCDF adding the cell area @@ -223,7 +224,7 @@ class RotatedGrid(Grid): [{'name': 'var_aux', 'units': '', 'data': 0}], boundary_latitudes=self.boundary_latitudes, boundary_longitudes=self.boundary_longitudes, cell_area=self.cell_area, - Rotated=True, rotated_lats=self.rlat, rotated_lons=self.rlon, + roated=True, rotated_lats=self.rlat, rotated_lons=self.rlon, north_pole_lat=self.new_pole_latitude_degrees, north_pole_lon=self.new_pole_longitude_degrees) else: diff --git a/hermesv3_gr/modules/masking/masking.py b/hermesv3_gr/modules/masking/masking.py index 98c113cb77cef84d10f1dd5506114ecd5fc37f0e..36b1c93f85a709e9e5e7c771e9c39b9a3248ae09 100644 --- a/hermesv3_gr/modules/masking/masking.py +++ b/hermesv3_gr/modules/masking/masking.py @@ -17,14 +17,33 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import timeit -import hermesv3_gr.config.settings as settings import os +import timeit from warnings import warn as warning +import hermesv3_gr.config.settings as settings class Masking(object): + """ + Masking object to apply simple mask or factor mask. + + :param world_info: Path to the file that contains the ISO Codes and other relevant information. + :type world_info: str + + :param factors_mask_values: List of the factor mask values. + :type factors_mask_values: list + + :param regrid_mask_values: List of the mask values. + :type regrid_mask_values: list + + :param grid: Grid. + :type grid: Grid + + :param world_mask_file: + :type world_mask_file: str + """ + def __init__(self, world_info, factors_mask_values, regrid_mask_values, grid, world_mask_file=None): from timezonefinder import TimezoneFinder @@ -39,37 +58,53 @@ class Masking(object): self.regrid_mask_values = self.parse_masking_values(regrid_mask_values) self.regrid_mask = None self.scale_mask = None - self.tf = TimezoneFinder() + self.timezonefinder = TimezoneFinder() self.grid = grid settings.write_time('Masking', 'Init', timeit.default_timer() - st_time, level=3) def get_country_codes(self): + """ + Get the country code information. + + :return: Dictionary of country codes. + :rtype: dict + """ import pandas as pd st_time = timeit.default_timer() - # settings.write_log('\t\t\tGetting country codes.', level=3) - # df = pd.read_csv(self.world_info, sep=';', index_col=False, names=["country", "country_code"]) - df = pd.read_csv(self.world_info, sep=';') - del df['time_zone'], df['time_zone_code'] - df = df.drop_duplicates().dropna() - df = df.set_index('country_code_alpha') - countries_dict = df.to_dict() + dataframe = pd.read_csv(self.world_info, sep=';') + del dataframe['time_zone'], dataframe['time_zone_code'] + dataframe = dataframe.drop_duplicates().dropna() + dataframe = dataframe.set_index('country_code_alpha') + countries_dict = dataframe.to_dict() countries_dict = countries_dict['country_code'] settings.write_time('Masking', 'get_country_codes', timeit.default_timer() - st_time, level=3) return countries_dict @staticmethod - def partlst(lst, n): + def partlst(lst, num): + """ + Split a Array in N balanced arrays. + + :param lst: Array to split + :type lst: numpy.array + + :param num: Number of mini arrays. + :type num: int + + :return: Array + :type: numpy.array + """ import itertools - """Partition @lst in @n balanced parts, in given order""" - parts, rest = divmod(len(lst), n) + # Partition @lst in @n balanced parts, in given order + parts, rest = divmod(len(lst), num) lstiter = iter(lst) - for j in xrange(n): - plen = len(lst) / n + (1 if rest > 0 else 0) + for j in xrange(num): + plen = len(lst) / num + (1 if rest > 0 else 0) rest -= 1 yield list(itertools.islice(lstiter, plen)) @@ -112,7 +147,7 @@ class Masking(object): 'units': '', 'data': dst_var, }] - Writer.write_netcdf(self.world_mask_file, lat, lon, data, RegularLatLon=True) + Writer.write_netcdf(self.world_mask_file, lat, lon, data, regular_latlon=True) settings.comm.Barrier() settings.write_time('Masking', 'create_country_iso', timeit.default_timer() - st_time, level=3) @@ -128,7 +163,7 @@ class Masking(object): elif longitude > +180: longitude -= 360 - tz = self.tf.timezone_at(lng=longitude, lat=latitude) + tz = self.timezonefinder.timezone_at(lng=longitude, lat=latitude) settings.write_time('Masking', 'find_timezone', timeit.default_timer() - st_time, level=3) @@ -151,6 +186,12 @@ class Masking(object): return code[0] def parse_factor_values(self, values): + """ + + :param values: + :return: + :rtype: dict + """ import re st_time = timeit.default_timer() diff --git a/hermesv3_gr/modules/regrid/regrid.py b/hermesv3_gr/modules/regrid/regrid.py index 7304084ac827e2802f9c3846d8ffc38ced40330a..9d224a9b07148e36015bce72b4d3ae688d632c11 100644 --- a/hermesv3_gr/modules/regrid/regrid.py +++ b/hermesv3_gr/modules/regrid/regrid.py @@ -33,7 +33,7 @@ class Regrid(object): self.pollutant_dicts = pollutant_dicts self.weight_matrix_file = weight_matrix_file self.masking = masking - + if not self.is_created_weight_matrix(erase=False): settings.write_log("\t\t\tWeight matrix {0} is not created. ".format(weight_matrix_file) + "Trying to create it", level=1) @@ -51,7 +51,7 @@ class Regrid(object): def apply_weights(self, values): """ - Calculates the regridded values using the ESMF algorithm for a 3D array. + Calculate the regridded values using the ESMF algorithm for a 3D array. :param values: Input values to regrid :type values: numpy.array diff --git a/hermesv3_gr/modules/regrid/regrid_conservative.py b/hermesv3_gr/modules/regrid/regrid_conservative.py index ff8d5098acecffb444f97d887cfee4d275226a7e..f2de4d1ed0b96bac1bd6ea35928daf0d7be1e7bf 100644 --- a/hermesv3_gr/modules/regrid/regrid_conservative.py +++ b/hermesv3_gr/modules/regrid/regrid_conservative.py @@ -46,7 +46,7 @@ class ConservativeRegrid(Regrid): src_grid = self.grid.create_esmf_grid_from_file(self.pollutant_dicts[0]['path']) src_field = ESMF.Field(src_grid, name='my input field') src_field.read(filename=self.pollutant_dicts[0]['path'], variable=self.pollutant_dicts[0]['name'], - timeslice=[0]) + timeslice=0) dst_grid = self.grid.esmf_grid dst_field = ESMF.Field(dst_grid, name='my outut field') @@ -166,7 +166,7 @@ class ConservativeRegrid(Regrid): def apply_weights(self, values): """ - Calculates the regridded values using the ESMF algorithm for a 3D array specifically for a conservative regrid. + Calculate the regridded values using the ESMF algorithm for a 3D array specifically for a conservative regrid. :param values: Input values to regrid. :type values: numpy.array diff --git a/hermesv3_gr/modules/speciation/speciation.py b/hermesv3_gr/modules/speciation/speciation.py index ac6d17be529ce5a02a80da2fcb7a5c0aff56a9e4..6084c44978eb0db59489bd7fbf3862592a61fdd9 100644 --- a/hermesv3_gr/modules/speciation/speciation.py +++ b/hermesv3_gr/modules/speciation/speciation.py @@ -50,7 +50,7 @@ class Speciation(object): def get_speciation_profile(self, speciation_profile_path): """ - Extracts the speciation information as a dictionary with the destiny pollutant as key and the formula as value. + Extract the speciation information as a dictionary with the destiny pollutant as key and the formula as value. :param speciation_profile_path: :type speciation_profile_path: @@ -95,7 +95,7 @@ class Speciation(object): @staticmethod def extract_molecular_weights(molecular_weights_path): """ - Extracts the molecular weights for each pollutant as a dictionary with the name of the pollutant as key and the + Extract the molecular weights for each pollutant as a dictionary with the name of the pollutant as key and the molecular weight as value. :param molecular_weights_path: Path to the CSV that contains all the molecular weights. diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 98fd6d3a731520df3fa2ec256f6e2da66e4c54be..82e1e36e9f3316bf4bb004d97596bcbd8bde974e 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import timeit -import hermesv3_gr.config.settings as settings import os import sys +import timeit +import hermesv3_gr.config.settings as settings import numpy as np @@ -67,7 +67,6 @@ class TemporalDistribution(object): timezones. :type auxiliar_files_dir: str """ - def __init__(self, starting_date, timestep_type, timestep_num, timestep_freq, monthly_profile_path, month_profile_id, daily_profile_path, daily_profile_id, hourly_profile_path, hourly_profile_id, world_info_path, auxiliar_files_dir, grid): @@ -144,7 +143,7 @@ class TemporalDistribution(object): def calculate_ending_date(self): """ - Calculates the date of the last timestep. + Calculate the date of the last timestep. :return: Date of the last timestep :rtype: datetime.datetime @@ -174,7 +173,7 @@ class TemporalDistribution(object): def calculate_timedelta(self, date): """ - Calculates the difference of time to the next timestep. + Calculate the difference of time to the next timestep. :param date: Date of the current timestep. :type date: datetime.datetime @@ -208,7 +207,7 @@ class TemporalDistribution(object): def get_tz_from_id(self, tz_id): """ - Extracts the timezone (string format) for the given id (int). + Extract the timezone (string format) for the given id (int). :param tz_id: ID of the timezone. :type tz_id: int @@ -216,14 +215,13 @@ class TemporalDistribution(object): :return: Timezone :rtype: str """ - tz = self.world_info_df.time_zone[self.world_info_df.time_zone_code == tz_id].values return tz[0] def get_id_from_tz(self, tz): """ - Extracts the id (int) for the given timezone (string format). + Extract the id (int) for the given timezone (string format). :param tz: Timezone of the ID. :type tz: str @@ -231,7 +229,6 @@ class TemporalDistribution(object): :return: ID :rtype: int """ - tz_id = self.world_info_df.time_zone_code[self.world_info_df.time_zone == tz].values try: @@ -247,9 +244,11 @@ class TemporalDistribution(object): return tz_id @staticmethod - def parse_tz(tz): + def parse_tz(timezone): """ - Parses the timezone (string format). It is needed because some libraries have more timezones than others and it + Parse the timezone (string format). + + It is needed because some libraries have more timezones than others and it tries to simplify setting the strange ones into the nearest common one. Examples: 'America/Punta_Arenas': 'America/Santiago', @@ -262,8 +261,8 @@ class TemporalDistribution(object): 'Asia/Tomsk': 'Asia/Novokuznetsk', 'America/Fort_Nelson': 'America/Vancouver' - :param tz: Not parsed timezone. - :type tz: str + :param timezone: Not parsed timezone. + :type timezone: str :return: Parsed timezone :rtype: str @@ -281,14 +280,14 @@ class TemporalDistribution(object): 'Asia/Famagusta': 'Asia/Nicosia', } - if tz in tz_dict.iterkeys(): - tz = tz_dict[tz] + if timezone in tz_dict.iterkeys(): + timezone = tz_dict[timezone] - return tz + return timezone def find_closest_timezone(self, latitude, longitude): """ - Finds the closest timezone for the given coordinates. + Find the closest timezone for the given coordinates. :param latitude: Latitude coordinate to find timezone. :type latitude: float @@ -299,22 +298,21 @@ class TemporalDistribution(object): :return: Nearest timezone of the given coordinates. :rtype: str """ - st_time = timeit.default_timer() - dg = 0 - tz = None - while tz is None: - tz = self.tf.closest_timezone_at(lng=longitude, lat=latitude, delta_degree=dg) - dg += 1 + degrees = 0 + timezone = None + while timezone is None: + timezone = self.tf.closest_timezone_at(lng=longitude, lat=latitude, delta_degree=degrees) + degrees += 1 settings.write_time('TemporalDistribution', 'find_closest_timezone', timeit.default_timer() - st_time, level=3) - return tz + return timezone def is_created_netcdf_timezones(self): """ - Checks if the NetCDF of timezones is created + Check if the NetCDF of timezones is created :return: True if it is already created. :rtype: bool @@ -323,7 +321,7 @@ class TemporalDistribution(object): def create_netcdf_timezones(self, grid): """ - Creates a NetCDF with the timezones in the resolution of the given grid. + Create a NetCDF with the timezones in the resolution of the given grid. :param grid: Grid object with the coordinates. :type grid: Grid @@ -344,26 +342,23 @@ class TemporalDistribution(object): num = 0 points = zip(lat.flatten(), lon.flatten()) - # points = points[534000:] - # print len(points) + for lat_aux, lon_aux in points: num += 1 settings.write_log("\t\t\tlat:{0}, lon:{1} ({2}/{3})".format(lat_aux, lon_aux, num, len(points)), level=3) - tz = self.find_closest_timezone(lat_aux, lon_aux) - tz_id = self.get_id_from_tz(tz) + timezone = self.find_closest_timezone(lat_aux, lon_aux) + tz_id = self.get_id_from_tz(timezone) dst_var.append(tz_id) dst_var = np.array(dst_var) dst_var = dst_var.reshape((1,) + lat.shape) dst_var = settings.comm.gather(dst_var, root=0) if settings.rank == 0: - for var in dst_var: - print var.shape total_lat = np.concatenate(total_lat, axis=1) total_lon = np.concatenate(total_lon, axis=1) dst_var = np.concatenate(dst_var, axis=2) data = [{'name': 'timezone_id', 'units': '', 'data': dst_var}] - write_netcdf(self.netcdf_timezones, total_lat, total_lon, data, RegularLatLon=True) + write_netcdf(self.netcdf_timezones, total_lat, total_lon, data, regular_latlon=True) settings.comm.Barrier() settings.write_time('TemporalDistribution', 'create_netcdf_timezones', timeit.default_timer() - st_time, @@ -372,6 +367,13 @@ class TemporalDistribution(object): return True def read_gridded_profile(self, path, value): + # TODO Documentation + """ + + :param path: + :param value: + :return: + """ from netCDF4 import Dataset st_time = timeit.default_timer() @@ -391,7 +393,7 @@ class TemporalDistribution(object): def calculate_timezones(self): """ - Extracts the timezones ID's from the NetCDF and convert them to the timezone (str). + Calculate the timezones ID's from the NetCDF and convert them to the timezone (str). :return: Array with the timezone of each cell. :rtype: numpy.chararray @@ -408,9 +410,9 @@ class TemporalDistribution(object): tz_list = np.chararray(timezones.shape, itemsize=32) for id_aux in xrange(timezones.min(), timezones.max() + 1): try: - tz = self.get_tz_from_id(id_aux) - tz_list[timezones == id_aux] = tz - except: + timezone = self.get_tz_from_id(id_aux) + tz_list[timezones == id_aux] = timezone + except IndexError: pass settings.write_time('TemporalDistribution', 'calculate_timezones', timeit.default_timer() - st_time, level=3) @@ -418,7 +420,7 @@ class TemporalDistribution(object): def calculate_2d_temporal_factors(self, date): """ - Calculates the temporal factor to correct the input data of the given date for each cell. + Calculate the temporal factor to correct the input data of the given date for each cell. :param date: Date of the current timestep. :type date: datetime.datetime @@ -438,7 +440,7 @@ class TemporalDistribution(object): try: df['local'] = df.groupby('tz')['utc'].apply( lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert(x.name).dt.tz_localize(None)) - except: + except pytz.exceptions.UnknownTimeZoneError: df['local'] = df.groupby('tz')['utc'].apply( lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert( self.parse_tz(x.name)).dt.tz_localize(None)) @@ -449,7 +451,7 @@ class TemporalDistribution(object): df['hour'] = df.index.hour if self.hourly_profile is not None: - if type(self.hourly_profile) is dict: + if isinstance(self.hourly_profile, dict): df['hour_factor'] = df['hour'].map(self.hourly_profile) else: profile_ids = self.parse_hourly_profile_id() @@ -481,13 +483,13 @@ class TemporalDistribution(object): if self.monthly_profile is None: df['month_factor'] = 1 - elif type(self.monthly_profile) == dict: + elif isinstance(self.monthly_profile, dict): df['month_factor'] = df['month'].map(self.monthly_profile) - elif type(self.monthly_profile) == np.ndarray: + elif isinstance(self.monthly_profile, np.ndarray): for m, df_aux in df.groupby('month'): try: df.loc[df['month'] == m, 'month_factor'] = \ - self.monthly_profile[m-1, df.loc[df['month'] == m, 'i'].values] + self.monthly_profile[m - 1, df.loc[df['month'] == m, 'i'].values] except IndexError: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -511,7 +513,7 @@ class TemporalDistribution(object): def calculate_3d_temporal_factors(self): """ - Calculates the temporal factor to correct the input data of the given date for each cell. + Calculate the temporal factor to correct the input data of the given date for each cell. :return: 3D array with the factors to correct the input data to the date of this timestep. :rtype: numpy.array @@ -520,19 +522,19 @@ class TemporalDistribution(object): settings.write_log("\tCalculating temporal factors.", level=2) factors = [] - date_aux = self.starting_date + date = self.starting_date count = 0 - while date_aux <= self.ending_date: + while date <= self.ending_date: count += 1 settings.write_log("\t\t{0} temporal factor ({1}/{2}).".format( - date_aux.strftime('%Y/%m/%d %H:%M:%S'), count, self.timestep_num), level=3) + date.strftime('%Y/%m/%d %H:%M:%S'), count, self.timestep_num), level=3) - factors.append(self.calculate_2d_temporal_factors(date_aux)) + factors.append(self.calculate_2d_temporal_factors(date)) - d = date_aux - self.starting_date - self.hours_since.append(d.seconds / 3600 + d.days * 24) # 3600 seconds per hour - date_aux = date_aux + self.calculate_timedelta(date_aux) + date_aux = date - self.starting_date + self.hours_since.append(date_aux.seconds / 3600 + date_aux.days * 24) # 3600 seconds per hour + date = date + self.calculate_timedelta(date) factors = np.array(factors) @@ -542,7 +544,8 @@ class TemporalDistribution(object): def parse_hourly_profile_id(self): """ - Parses the hourly profile ID to get a dictionary with the ID for "weekday", "saturday" and "sunday" + Parse the hourly profile ID to get a dictionary with the ID for "weekday", "saturday" and "sunday" + :return: """ import re @@ -557,7 +560,8 @@ class TemporalDistribution(object): def get_temporal_hourly_profile(self, profile_id, date=None): """ - Extracts the hourly profile of the given ID in a dictionary format. + Extract the hourly profile of the given ID in a dictionary format. + The hour (0 to 23) is the key (int) and the value (float) is the factor. :param profile_id: ID of the hourly profile to use. @@ -572,8 +576,6 @@ class TemporalDistribution(object): import pandas as pd st_time = timeit.default_timer() - # settings.write_log("\t\t\tGetting temporal hourly profile '{0}' from {1} .".format( - # profile_id, self.hourly_profile_path), level=3) if date is None: df = pd.read_csv(self.hourly_profile_path) try: @@ -587,7 +589,6 @@ class TemporalDistribution(object): profile.pop('TP_H', None) profile = {int(k): float(v) for k, v in profile.items()} else: - # print self.hourly_profile profile = None settings.write_time('TemporalDistribution', 'get_temporal_hourly_profile', timeit.default_timer() - st_time, level=3) @@ -596,7 +597,8 @@ class TemporalDistribution(object): def get_temporal_daily_profile(self, date): """ - Extracts the daily profile of the given ID in a dictionary format. + Extract the daily profile of the given ID in a dictionary format. + The weekday (0 to 6) is the key (int) and the value (float) is the factor. :param date: Date of the timestep to simulate. @@ -633,8 +635,10 @@ class TemporalDistribution(object): def calculate_rebalance_factor(self, profile, date): """ - Calculates the necessary factor make consistent the full month data. This is needed for the months that if you - sum the daily factor of each day of the month it doesn't sum as the number of days of the month. + Calculate the necessary factor make consistent the full month data. + + This is needed for the months that if you sum the daily factor of each day of the month it doesn't sum as + the number of days of the month. :param profile: Daily profile. :type profile: dict @@ -657,8 +661,9 @@ class TemporalDistribution(object): @staticmethod def calculate_weekday_factor_full_month(profile, weekdays): + # TODO Documentation """ - Operates with all the days of the month to get the sum of daily factors of the full month. + Operate with all the days of the month to get the sum of daily factors of the full month. :param profile: :param weekdays: @@ -679,6 +684,12 @@ class TemporalDistribution(object): @staticmethod def calculate_weekdays(date): + # TODO Documentation + """ + + :param date: + :return: + """ from calendar import monthrange, weekday, MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY st_time = timeit.default_timer() @@ -698,7 +709,8 @@ class TemporalDistribution(object): @staticmethod def get_temporal_monthly_profile(profile_path, profile_id): """ - Extracts the monthly profile of the given ID in a dictionary format. + Extract the monthly profile of the given ID in a dictionary format. + The month (1 to 12) is the key (int) and the value (float) is the factor. :param profile_path: Path to the file that contains all the monthly profiles. @@ -739,6 +751,15 @@ class TemporalDistribution(object): @staticmethod def calculate_delta_hours(st_date, time_step_type, time_step_num, time_step_freq): + # TODO Documentation + """ + + :param st_date: + :param time_step_type: + :param time_step_num: + :param time_step_freq: + :return: + """ from datetime import timedelta from calendar import monthrange, isleap diff --git a/hermesv3_gr/modules/vertical/vertical.py b/hermesv3_gr/modules/vertical/vertical.py index de9041f4a14d1776abfc3490f23efbfc174cc94c..1b309ea4be040924ab3f08aa4078ac1ceb1aed42 100644 --- a/hermesv3_gr/modules/vertical/vertical.py +++ b/hermesv3_gr/modules/vertical/vertical.py @@ -50,7 +50,7 @@ class VerticalDistribution(object): def get_vertical_profile(self, path): """ - Extracts the vertical v_profile from the vertical v_profile file. + Extract the vertical v_profile from the vertical v_profile file. :param path: Path to the file that contains all the vertical profiles. :type path: str @@ -93,7 +93,7 @@ class VerticalDistribution(object): @staticmethod def get_vertical_output_profile(path): """ - Extracts the vertical description of the desired output. + Extract the vertical description of the desired output. :param path: Path to the file that contains the output vertical description. :type path: str @@ -118,7 +118,7 @@ class VerticalDistribution(object): @staticmethod def get_weights(prev_layer, layer, in_weight, output_vertical_profile): """ - Calculates the weights for the given layer. + Calculate the weights for the given layer. :param prev_layer: Altitude of the low layer. 0 if it's the first. :type prev_layer: float @@ -151,7 +151,7 @@ class VerticalDistribution(object): def calculate_weights(self): """ - Calculates the weights for all the vertical layers. + Calculate the weights for all the vertical layers. :return: Weights that goes to each layer. :rtype: list of float @@ -177,7 +177,7 @@ class VerticalDistribution(object): @staticmethod def apply_weights(data, weights): """ - Calculates the vertical distribution using the given data and weights. + Calculate the vertical distribution using the given data and weights. :param data: Emissions to be vertically distributed. :type data: numpy.array @@ -186,7 +186,7 @@ class VerticalDistribution(object): :type weights: numpy.array :return: Emissions already vertically distributed. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np diff --git a/hermesv3_gr/modules/vertical/vertical_gfas.py b/hermesv3_gr/modules/vertical/vertical_gfas.py index 08caf9f87f1332ede28525365a4c9e91fd1d9151..600ab7f6fef3559a064f0f37ea209847ada65260 100644 --- a/hermesv3_gr/modules/vertical/vertical_gfas.py +++ b/hermesv3_gr/modules/vertical/vertical_gfas.py @@ -45,7 +45,7 @@ class GfasVerticalDistribution(VerticalDistribution): @staticmethod def calculate_widths(heights_list): """ - Calculates the width of each vertical level. + Calculate the width of each vertical level. :param heights_list: List of the top altitude in meters of each level. :type heights_list: list @@ -67,7 +67,7 @@ class GfasVerticalDistribution(VerticalDistribution): def get_weights(self, heights_list): """ - Calculates the proportion (%) of emission to put on each layer. + Calculate the proportion (%) of emission to put on each layer. :param heights_list: List with the width of each vertical level. :type heights_list: list @@ -128,13 +128,13 @@ class GfasVerticalDistribution(VerticalDistribution): Allocates the fire emissions on their top level. :param values: 2D array with the fire emissions - :type values: numpy.ndarray + :type values: numpy.array :param altitude: 2D array with the altitude of the fires. - :type altitude: numpy.ndarray + :type altitude: numpy.array :return: Emissions already allocated on the top altitude of each fire. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np @@ -159,10 +159,10 @@ class GfasVerticalDistribution(VerticalDistribution): Manages all the process to do the vertical distribution. :param values: Emissions to be vertically distributed. - :type values: numpy.ndarray + :type values: numpy.array :return: Emissions already vertically distributed. - :rtype: numpy.ndarray + :rtype: numpy.array """ st_time = timeit.default_timer() diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index a110b6cdd0abbe5acf26797f4135495325ebe929..06e6f34b4136aa34365b2f02fee263ce579477ed 100644 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -18,12 +18,42 @@ # along with HERMESv3_GR. If not, see . +import sys import timeit +import numpy as np +from mpi4py import MPI +from netCDF4 import Dataset from hermesv3_gr.config import settings -import sys class Writer(object): + """ + Class to Write the output file. + + :param path: Path to the destination file. + :type path: str + + :param grid: Grid of the destination file. + :type grid: Grid + + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. + :type compress: bool + + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool + """ def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): @@ -42,11 +72,21 @@ class Writer(object): self.global_attributes_path = global_attributes_path def write(self, inventory_list): + """ + Write the netCDF4 file with the pollutants of the given list of inventories. + + :param inventory_list: List of inventories. + :type inventory_list: list + + :return: True at end + :rtype: bool + """ st_time = timeit.default_timer() settings.write_log('') settings.write_log("Writing netCDF output file {0} .".format(self.path)) self.set_variable_attributes(inventory_list) + self.change_variable_attributes() if self.parallel: if settings.rank == 0: self.create_parallel_netcdf() @@ -58,6 +98,9 @@ class Writer(object): settings.write_time('Writer', 'write', timeit.default_timer() - st_time) return True + def change_variable_attributes(self): + pass + def create_parallel_netcdf(self): """ Implemented on inner class. @@ -66,9 +109,50 @@ class Writer(object): def write_parallel_netcdf(self, emission_list): """ - Implemented on inner class. + Append the data to the netCDF4 file already created in parallel mode. + + :param emission_list: Data to append. + :type emission_list: list + + :return: True at end. + :rtype: bool """ - return None + + st_time = timeit.default_timer() + + settings.write_log("\tAppending data to parallel NetCDF file.", level=2) + if settings.size > 1: + netcdf = Dataset(self.path, mode='a', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) + else: + netcdf = Dataset(self.path, mode='a', format="NETCDF4") + settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) + index = 0 + # print "Rank {0} 2".format(rank) + for var_name in self.variables_attributes.iterkeys(): + + data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) + st_time = timeit.default_timer() + index += 1 + + var = netcdf.variables[var_name] + if settings.size > 1: + var.set_collective(True) + # Correcting NAN + if data is None: + data = 0 + var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] = data + + settings.write_log("\t\t\t'{0}' variable filled".format(var_name)) + + if 'cell_area' in netcdf.variables: + c_area = netcdf.variables['cell_area'] + c_area[self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] = self.grid.cell_area + + netcdf.close() + settings.write_time('Writer', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) + return True def write_serial_netcdf(self, emission_list): """ @@ -77,10 +161,19 @@ class Writer(object): return None def set_variable_attributes(self, inventory_list): + """ + Change the variables_attribute parameter of the Writer class. + + :param inventory_list: list of invenotries. + :type inventory_list: list + + :return: True at end. + :rtype: bool + """ st_time = timeit.default_timer() empty_dict = {} - for ei in inventory_list: - for emi in ei.emissions: + for inventory in inventory_list: + for emi in inventory.emissions: if not emi['name'] in empty_dict: dict_aux = emi.copy() dict_aux['data'] = None @@ -93,17 +186,21 @@ class Writer(object): return True def calculate_data_by_var(self, variable, inventory_list, shape): - # TODO Documentation """ + Calculate the date of the given variable throw the inventory list. - :param variable: - :param inventory_list: - :param shape: - :return: - """ - import timeit - import numpy as np + :param variable: Variable to calculate. + :type variable: str + + :param inventory_list: Inventory list + :type inventory_list: list + + :param shape: Output desired shape. + :type shape: tuple + :return: Data of the given variable. + :rtype: numpy.array + """ st_time = timeit.default_timer() settings.write_log("\t\t\t\tGetting data for '{0}' pollutant.".format(variable), level=3) @@ -128,6 +225,8 @@ class Writer(object): aux_data = np.zeros((shape[1], shape[2] * shape[3])) aux_data[ei.location['layer'], ei.location['FID']] = emission['data'] aux_data = aux_data.reshape((shape[1], shape[2], shape[3])) + else: + aux_data = None settings.write_time('VerticalDistribution', 'calculate_data_by_var', timeit.default_timer() - vertical_time, level=2) @@ -153,10 +252,19 @@ class Writer(object): """ Implement on inner class """ - return None + return np.array([0]) @staticmethod def calculate_displacements(counts): + """ + Calculate the index position of all the ranks. + + :param counts: Number of elements for rank + :type counts: list + + :return: Displacements + :rtype: list + """ st_time = timeit.default_timer() new_list = [0] @@ -170,15 +278,27 @@ class Writer(object): @staticmethod def tuple_to_index(tuple_list, bidimensional=False): + """ + Get the index for a list of shapes. + + :param tuple_list: List os shapes. + :type tuple_list: list + + :param bidimensional: Indicates if the tuple is bidimensional. + :type bidimensional: bool + + :return: List of index + :rtype: list + """ from operator import mul st_time = timeit.default_timer() new_list = [] - for tuple in tuple_list: + for my_tuple in tuple_list: if bidimensional: - new_list.append(tuple[-1] * tuple[-2]) + new_list.append(my_tuple[-1] * my_tuple[-2]) else: - new_list.append(reduce(mul, tuple)) + new_list.append(reduce(mul, my_tuple)) settings.write_time('Writer', 'tuple_to_index', timeit.default_timer() - st_time, level=3) return new_list @@ -190,17 +310,29 @@ class Writer(object): :param output_model: Name of the output model. Only accepted 'MONARCH, CMAQ or WRF_CHEM. :type output_model: str - :param path: Path to the output file. + :param path: Path to the destination file. :type path: str - :param grid: Grid object of the destination. + :param grid: Grid of the destination file. :type grid: Grid - :param compress: Indicates if you want a compressed NetCDF. + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. :type compress: bool - :param parallel: Indicates if you want to write the NetCDF in parallel. - :type parallel: bool + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool :return: Writing object of the desired output model. :rtype: Writer @@ -227,19 +359,23 @@ class Writer(object): def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, - RegularLatLon=False, - Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, - LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, - Mercator=False, lat_ts=None): + regular_latlon=False, + roated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, + lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, + mercator=False, lat_ts=None): + # TODO Deprecate + """ + Will be deprecated + """ from netCDF4 import Dataset from cf_units import Unit, encode_time - if not (RegularLatLon or LambertConformalConic or Rotated or Mercator): - RegularLatLon = True + if not (regular_latlon or lcc or roated or mercator): + regular_latlon = True netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -264,7 +400,7 @@ class Writer(object): print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( len(center_longitudes.shape)) sys.exit(1) - elif Rotated: + elif roated: var_dim = ('rlat', 'rlon',) # Rotated Latitude @@ -280,7 +416,7 @@ class Writer(object): sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic or Mercator: + elif lcc or mercator: var_dim = ('y', 'x',) netcdf.createDimension('y', len(lcc_y)) @@ -288,6 +424,10 @@ class Writer(object): netcdf.createDimension('x', len(lcc_x)) lon_dim = ('y', 'x',) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if levels is not None: @@ -318,11 +458,10 @@ class Writer(object): time[:] = [0.] else: time = netcdf.createVariable('time', 'd', ('time',), zlib=True) - u = Unit('hours') # print u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second)) # Unit('hour since 1970-01-01 00:00:00.0000000 UTC') - time.units = str( - u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second))) + time.units = str(Unit('hours').offset_by_time( + encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second))) time.standard_name = "time" time.calendar = "gregorian" time.long_name = "time" @@ -357,7 +496,7 @@ class Writer(object): # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes - if Rotated: + if roated: # Rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=True) rlat.long_name = "latitude in rotated pole grid" @@ -371,18 +510,18 @@ class Writer(object): rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = rotated_lons - if LambertConformalConic or Mercator: - x = netcdf.createVariable('x', 'd', ('x',), zlib=True) - x.units = Unit("km").symbol - x.long_name = "x coordinate of projection" - x.standard_name = "projection_x_coordinate" - x[:] = lcc_x - - y = netcdf.createVariable('y', 'd', ('y',), zlib=True) - y.units = Unit("km").symbol - y.long_name = "y coordinate of projection" - y.standard_name = "projection_y_coordinate" - y[:] = lcc_y + if lcc or mercator: + x_var = netcdf.createVariable('x', 'd', ('x',), zlib=True) + x_var.units = Unit("km").symbol + x_var.long_name = "x coordinate of projection" + x_var.standard_name = "projection_x_coordinate" + x_var[:] = lcc_x + + y_var = netcdf.createVariable('y', 'd', ('y',), zlib=True) + y_var.units = Unit("km").symbol + y_var.long_name = "y coordinate of projection" + y_var.standard_name = "projection_y_coordinate" + y_var[:] = lcc_y cell_area_dim = var_dim # Levels @@ -410,40 +549,40 @@ class Writer(object): var.coordinates = "lat lon" if cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif roated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' - elif Mercator: + elif mercator: var.grid_mapping = 'mercator' try: var[:] = variable['data'] - except: + except ValueError: print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: + elif roated: # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = north_pole_lat mapping.grid_north_pole_longitude = north_pole_lon - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" mapping.standard_parallel = lat_1_2 mapping.longitude_of_central_meridian = lon_0 mapping.latitude_of_projection_origin = lat_0 - elif Mercator: + elif mercator: # Mercator mapping = netcdf.createVariable('mercator', 'i') mapping.grid_mapping_name = "mercator" diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 7b23615e02564aa0070d33c88fe875b22d92147a..7b3480c780f442559400412192c7367b0a61b0e6 100644 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -18,17 +18,43 @@ # along with HERMESv3_GR. If not, see . -from hermesv3_gr.modules.writing.writer import Writer -import timeit -from hermesv3_gr.config import settings -import os import sys +import timeit import numpy as np from netCDF4 import Dataset from mpi4py import MPI +from hermesv3_gr.modules.writing.writer import Writer +from hermesv3_gr.config import settings class WriterCmaq(Writer): + """ + Class to Write the output file for CMAQ Chemical Transport Model CCTM. + + :param path: Path to the destination file. + :type path: str + + :param grid: Grid of the destination file. + :type grid: Grid + + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. + :type compress: bool + + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool + """ def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterCmaq, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) @@ -39,13 +65,20 @@ class WriterCmaq(Writer): 'XCELL', 'YCELL', 'VGTYP', 'VGTOP', 'VGLVLS', 'GDNAM', 'UPNAM', 'FILEDESC', 'HISTORY', 'VAR-LIST'] def unit_change(self, variable, data): + # TODO Documentation + """ + + :param variable: + :param data: + :return: + """ from cf_units import Unit if data is not None: units = None - for var in self.variables_attributes: - if var['name'] == variable: - units = var['units'] + for var_name in self.variables_attributes: + if var_name == variable: + units = self.variables_attributes[var_name]['units'] break if Unit(units).symbol == Unit('mol.s-1').symbol: @@ -60,35 +93,42 @@ class WriterCmaq(Writer): sys.exit(1) return data - @staticmethod - def change_variable_attributes(emission_list): + def change_variable_attributes(self): """ - Modifies the emission list to be consistent to use the output as input for CMAQ model. - - :param emission_list: List of emissions - :type emission_list: list + Modify the emission list to be consistent to use the output as input for CMAQ model. :return: Emission list ready for CMAQ - :rtype: list + :rtype: dict """ from cf_units import Unit - var_list = [] - for i in xrange(len(emission_list)): - emission_list[i]['var_desc'] = "{:<80}".format(emission_list[i]['long_name']) - emission_list[i]['long_name'] = "{:<16}".format(emission_list[i]['name']) + new_variable_dict = {} + for variable in self.variables_attributes: + if Unit(variable['units']).symbol == Unit('mol.s-1').symbol: + new_variable_dict[variable['name']] = { + 'units': "{:<16}".format('mole/s'), + 'var_desc': "{:<80}".format(variable['long_name']), + 'long_name': "{:<16}".format(variable['name']), + } + elif Unit(variable['units']).symbol == Unit('g.s-1').symbol: + new_variable_dict[variable['name']] = { + 'units': "{:<16}".format('g/s'), + 'var_desc': "{:<80}".format(variable['long_name']), + 'long_name': "{:<16}".format(variable['name']), + } + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise TypeError("The unit '{0}' of specie {1} is not ".format(variable['units'], variable['name']) + + "defined correctly. Should be 'mol.s-1' or 'g.s-1'") + sys.exit(1) - if Unit(emission_list[i]['units']).symbol == Unit('mol.s-1').symbol: - emission_list[i]['units'] = "{:<16}".format('mole/s') - elif Unit(emission_list[i]['units']).symbol == Unit('g.s-1').symbol: - emission_list[i]['units'] = "{:<16}".format('g/s') - var_list.append(emission_list[i]['name']) - return emission_list, var_list + self.variables_attributes = new_variable_dict @staticmethod def create_tflag(st_date, hours_array, num_vars): """ - Creates the content of the CMAQ variable TFLAG + Create the content of the CMAQ variable TFLAG :param st_date: Starting date :type st_date: datetime.datetime @@ -103,7 +143,6 @@ class WriterCmaq(Writer): :return: Array with the content of TFLAG :rtype: numpy.array """ - import numpy as np from datetime import timedelta a = np.array([[[]]]) @@ -127,7 +166,6 @@ class WriterCmaq(Writer): :return: List transformed on string. :rtype: str """ - str_var_list = "" for var in var_list: str_var_list += "{:<16}".format(var) @@ -135,6 +173,11 @@ class WriterCmaq(Writer): return str_var_list def read_global_attributes(self): + # TODO Documentation + """ + + :return: + """ import pandas as pd from warnings import warn as warning float_atts = ['VGTOP'] @@ -167,8 +210,8 @@ class WriterCmaq(Writer): atts_dict[att] = np.array(df.loc[df['attribute'] == att, 'value'].item().split(), dtype=np.float32) except ValueError: - settings.write_log('WARNING: The global attribute {0} is not defined; Using default value {1}'.format( - att, atts_dict[att])) + settings.write_log('WARNING: The global attribute {0} is not defined;'.format(att) + + ' Using default value {0}'.format(atts_dict[att])) if settings.rank == 0: warning('WARNING: The global attribute {0} is not defined; Using default value {1}'.format( att, atts_dict[att])) @@ -186,95 +229,14 @@ class WriterCmaq(Writer): def create_global_attributes(self, var_list): """ - Creates the global attributes and the order that they have to be filled. - - :param date: Starting date. - :type date: datetime.datetime - - :param nx: Number of elements on the x dimension. - :type nx: int - - :param ny: Number of elements on the y dimension. - :type ny: int - - :param nlays: Number of vertical layers. - :type nlays: int - - :param lat_1: Value of lat 1 of the Lambert Conformal Conic projection. - :type lat_1: float - - :param lat_2: Value of lat 2 of the Lambert Conformal Conic projection. - :type lat_2: float - - :param lon_0: Value of lon 0 of the Lambert Conformal Conic projection. - :type lon_0: float + Create the global attributes and the order that they have to be filled. - :param lat_0: Value of lat 0 of the Lambert Conformal Conic projection. - :type lat_0: float - - :param x_0: X value of the origin. - :type x_0: float - - :param y_0: Y value of the origin. - :type y_0: float - - :param inc_x: Increment (in meters) of the x values. - :type inc_x: float - - :param inc_y: Increment (in meters) of the y values. - :type inc_x: float - - :param var_list: List of variables. + :param var_list: List of variables :type var_list: list - :param exec_id: ID of the execution. - :type exec_id: str - - :param ftype: File data type = [CUSTOM3:1, GRDDED3:2, BNDARY3:3, IDDATA3:4, PROFIL3:5, or SMATRX3:6] - (Default = 1) - :type ftype: int - - :param tstep: time step, coded HHMMSS according to Models-3 conventions. - :type tstep: int - - :param nthik: For BNDARY3 files, perimeter thickness (cells), or for SMATRX3 files, number of matrix-columns - (unused for other file types) - :type nthik: int - - :param gdtyp: Map projection type - LATGRD3=1 (Lat-Lon), - LAMGRD3=2 (Lambert conformal conic), - MERGRD3=3 (general tangent Mercator), - STEGRD3=4 (general tangent stereographic), - UTMGRD3=5 (UTM, a special case of Mercator), - POLGRD3=6 (polar secant stereographic), - EQMGRD3=7 (equatorial secant Mercator), or - TRMGRD3=8 (transverse secant Mercator) - :type gdtyp: int - - :param vgtype: Vertical coordinate type - VGSGPH3=1 (hydrostatic sigma-P), - VGSGPN3=2 (nonhydrostatic sigma-P), - VGSIGZ3=3 (sigma-Z), - VGPRES3=4 (pressure (mb)), - VGZVAL3=5 (Z (m above sea lvl), or - VGHVAL3=6 (H (m above ground)) - :type vgtype: int - - :param vgtop: Model-top, for sigma vertical-coordinate types - :type vgtop: int - - :param vglvls: Array of vertical coordinate level values; level 1 of the grid goes from vertical coordinate - VGLEVELS[0] to VGLEVELS[1], etc. - :type vglvls: numpy.array - - :param gdnam: Grid Name - :type gdnam: str - :return: Dict of global attributes and a list with the keys ordered. :rtype: tuple """ - # TODO documentation from datetime import datetime global_attributes = self.read_global_attributes() @@ -323,6 +285,30 @@ class WriterCmaq(Writer): def create_cmaq_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, regular_lat_lon=False, rotated=False, nx=None, ny=None, lat_1=None, lat_2=None, lon_0=None, lat_0=None, x_0=None, y_0=None, inc_x=None, inc_y=None): + # TODO Documentation + """ + + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param regular_lat_lon: + :param rotated: + :param nx: + :param ny: + :param lat_1: + :param lat_2: + :param lon_0: + :param lat_0: + :param x_0: + :param y_0: + :param inc_x: + :param inc_y: + :return: + """ data_list, var_list = WriterCmaq.change_variable_attributes(data_list) @@ -346,10 +332,21 @@ class WriterCmaq(Writer): @staticmethod def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, global_attributes=None, regular_lat_lon=False, rotated=False): + # TODO Documentation + """ - import sys - from netCDF4 import Dataset - + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param global_attributes: + :param regular_lat_lon: + :param rotated: + :return: + """ if regular_lat_lon: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -394,6 +391,10 @@ class WriterCmaq(Writer): netcdf.close() def create_parallel_netcdf(self): + # TODO Documentation + """ + Create an empty netCDF + """ st_time = timeit.default_timer() settings.write_log("\tCreating parallel NetCDF file.", level=2) # netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) @@ -401,8 +402,8 @@ class WriterCmaq(Writer): # ===== Dimensions ===== settings.write_log("\t\tCreating NetCDF dimensions.", level=2) - # netcdf.createDimension('TSTEP', len(self.hours)) - netcdf.createDimension('TSTEP', None) + netcdf.createDimension('TSTEP', len(self.hours)) + # netcdf.createDimension('TSTEP', None) settings.write_log("\t\t\t'TSTEP' dimension: {0}".format('UNLIMITED ({0})'.format(len(self.hours))), level=3) netcdf.createDimension('DATE-TIME', 2) @@ -429,19 +430,17 @@ class WriterCmaq(Writer): settings.write_log("\t\t\t'TFLAG' variable created with size: {0}".format(tflag[:].shape), level=3) index = 0 - data_list, var_list = self.change_variable_attributes(self.variables_attributes) - for variable in self.variables_attributes: + # data_list, var_list = self.change_variable_attributes(self.variables_attributes) + for var_name in self.variables_attributes.iterkeys(): index += 1 - var = netcdf.createVariable(variable['name'], 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) - var.units = variable['units'] - var.long_name = str(variable['long_name']) - var.var_desc = str(variable['var_desc']) - settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(variable['name'], var[:].shape) + - "\n\t\t\t\t'{0}' variable will be filled later.".format(variable['name']), level=3) + var = netcdf.createVariable(var_name, 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) + var.setncatts(self.variables_attributes[var_name]) + settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(var_name, var[:].shape) + + "\n\t\t\t\t'{0}' variable will be filled later.".format(var_name), level=3) # ===== Global attributes ===== settings.write_log("\t\tCreating NetCDF metadata.", level=2) - global_attributes = self.create_global_attributes(var_list) + global_attributes = self.create_global_attributes(self.variables_attributes.keys()) for attribute in self.global_attributes_order: netcdf.setncattr(attribute, global_attributes[attribute]) @@ -449,34 +448,18 @@ class WriterCmaq(Writer): settings.write_time('WriterCmaq', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) - def write_parallel_netcdf(self, emission_list): - st_time = timeit.default_timer() - settings.write_log("\tAppending data to parallel NetCDF file.", level=2) - - if settings.size > 1: - netcdf = Dataset(self.path, mode='a', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) - else: - netcdf = Dataset(self.path, mode='a', format="NETCDF4") - settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) - index = 0 - for variable in self.variables_attributes: - data = self.calculate_data_by_var(variable['name'], emission_list, self.grid.shape) - - index += 1 + return True - var = netcdf.variables[variable['name']] - if settings.size > 1: - var.set_collective(True) - # Correcting NAN - if data is None: - data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] = data - settings.write_log("\t\t\t'{0}' variable filled".format(variable['name'])) + def write_serial_netcdf(self, emission_list): + """ + Write the netCDF in serial mode. - netcdf.close() - settings.write_time('WriterCmaq', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) + :param emission_list: List of the processed emissions for the different emission inventories + :type emission_list: list - def write_serial_netcdf(self, emission_list): + :return: True when it finish well. + :rtype: bool + """ st_time = timeit.default_timer() mpi_numpy = False @@ -484,7 +467,8 @@ class WriterCmaq(Writer): # Gathering the index if mpi_numpy or mpi_vector: - rank_position = np.array([self.grid.x_lower_bound, self.grid.x_upper_bound, self.grid.y_lower_bound, self.grid.y_upper_bound], dtype='i') + rank_position = np.array([self.grid.x_lower_bound, self.grid.x_upper_bound, self.grid.y_lower_bound, + self.grid.y_upper_bound], dtype='i') full_position = None if settings.rank == 0: full_position = np.empty([settings.size, 4], dtype='i') @@ -519,11 +503,11 @@ class WriterCmaq(Writer): full_shape = None index = 0 - data_list, var_list = self.change_variable_attributes(self.variables_attributes) - for variable in data_list: + # data_list, var_list = self.change_variable_attributes(self.variables_attributes) + for var_name in self.variables_attributes.iterkeys(): if settings.size != 1: - settings.write_log("\t\t\tGathering {0} data.".format(variable['name']), level=3) - rank_data = self.calculate_data_by_var(variable['name'], emission_list, self.grid.shape) + settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) + rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) if mpi_numpy or mpi_vector: if rank_data is not None: root_shape = settings.comm.bcast(rank_data.shape, root=0) @@ -577,15 +561,16 @@ class WriterCmaq(Writer): if settings.size != 1: try: data = np.concatenate(data, axis=3) - except: + except (UnboundLocalError, TypeError, IndexError): data = 0 st_time = timeit.default_timer() index += 1 - var = netcdf.createVariable(variable['name'], 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) - var.units = variable['units'] - var.long_name = str(variable['long_name']) - var.var_desc = str(variable['var_desc']) + var = netcdf.createVariable(var_name, 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) + var.setncatts(self.variables_attributes[var_name]) + # var.units = variable['units'] + # var.long_name = str(variable['long_name']) + # var.var_desc = str(variable['var_desc']) # var[:] = variable['data'] if mpi_numpy: @@ -597,9 +582,9 @@ class WriterCmaq(Writer): elif i == settings.size - 1: var[:, :, :, full_position[i][2]:] = recvbuf[i, :, :, :, :-1] else: - var[:, :, :, full_position[i][2]:full_position[i][3]] = recvbuf[i, :, :, :, - : full_shape[i][-1]] - except: + var[:, :, :, full_position[i][2]:full_position[i][3]] = \ + recvbuf[i, :, :, :, : full_shape[i][-1]] + except ValueError: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise TypeError("ERROR on i {0} ".format(i) + @@ -608,29 +593,29 @@ class WriterCmaq(Writer): sys.exit(1) elif mpi_vector: - var_time = timeit.default_timer() - - # data_list = []#np.empty(shape, dtype=np.float64) - if rank_data is not None: data = np.empty(var[:].shape, dtype=settings.precision) for i in xrange(settings.size): # print 'Resizeing {0}'.format(i) if not i == settings.size - 1: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) else: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) else: data = 0 var[:] = data else: var[:] = data - settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(variable['name'], var[:].shape), + settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(var_name, var[:].shape), level=3) settings.write_log("\t\tCreating NetCDF metadata.", level=2) if settings.rank == 0: # ===== Global attributes ===== - global_attributes = self.create_global_attributes(var_list) + global_attributes = self.create_global_attributes(self.variables_attributes.keys()) for attribute in self.global_attributes_order: netcdf.setncattr(attribute, global_attributes[attribute]) diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index f8239286a73f774d8ae52a12d476d4ddb9c3b14d..3321b06c0f0945da74668f8e8d50d6c6bb503663 100644 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -18,17 +18,43 @@ # along with HERMESv3_GR. If not, see . -from hermesv3_gr.modules.writing.writer import Writer -import timeit -from hermesv3_gr.config import settings -import os import sys +import timeit import numpy as np from netCDF4 import Dataset from mpi4py import MPI +from hermesv3_gr.modules.writing.writer import Writer +from hermesv3_gr.config import settings class WriterMonarch(Writer): + """ + Class to Write the output file in CF-1.6 conventions. + + :param path: Path to the destination file. + :type path: str + + :param grid: Grid of the destination file. + :type grid: Grid + + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. + :type compress: bool + + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool + """ def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterMonarch, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) @@ -38,14 +64,26 @@ class WriterMonarch(Writer): # } def unit_change(self, variable, data): + """ + Do the unit conversions of the data. + + :param variable: Variable to convert. + :type variable: dict + + :param data: Data to change. + :type data: numpy.array + + :return: Data with the new units. + :rtype: numpy.array + """ from cf_units import Unit st_time = timeit.default_timer() if data is not None: units = None - for var in self.variables_attributes: - if var['name'] == variable: - units = var['units'] + for var_name in self.variables_attributes: + if var_name == variable: + units = self.variables_attributes[var_name]['units'] break if Unit(units).symbol == Unit('mol.s-1.m-2').symbol: @@ -61,7 +99,27 @@ class WriterMonarch(Writer): settings.write_time('WriterMonarch', 'unit_change', timeit.default_timer() - st_time, level=3) return data + def change_variable_attributes(self): + """ + Modify the emission list to be consistent to use the output as input for CMAQ model. + + :return: Emission list ready for CMAQ + :rtype: dict + """ + new_variable_dict = {} + for variable in self.variables_attributes: + new_variable_dict[variable['name']] = variable + del new_variable_dict[variable['name']]['name'] + + self.variables_attributes = new_variable_dict + def create_parallel_netcdf(self): + """ + Create an empty netCDF4. + + :return: True at end. + :rtype: bool + """ from cf_units import Unit, encode_time st_time = timeit.default_timer() @@ -96,7 +154,8 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) lat_dim = ('lon', 'lat', ) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(self.grid.center_latitudes.shape)) + print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format( + len(self.grid.center_latitudes.shape)) sys.exit(1) # Longitude @@ -109,7 +168,8 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[1]), level=3) lon_dim = ('lon', 'lat', ) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(self.grid.center_longitudes.shape)) + print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( + len(self.grid.center_longitudes.shape)) sys.exit(1) elif Rotated: var_dim = ('rlat', 'rlon',) @@ -140,6 +200,10 @@ class WriterMonarch(Writer): netcdf.createDimension('x', len(self.grid.x)) settings.write_log("\t\t\t'x' dimension: {0}".format(len(self.grid.x)), level=3) lon_dim = ('y', 'x', ) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if self.levels is not None: @@ -171,10 +235,7 @@ class WriterMonarch(Writer): time[:] = [0.] else: time = netcdf.createVariable('time', 'd', ('time',)) - u = Unit('hours') - # print u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second)) - # Unit('hour since 1970-01-01 00:00:00.0000000 UTC') - time.units = str(u.offset_by_time(encode_time(self.date.year, self.date.month, self.date.day, + time.units = str(Unit('hours').offset_by_time(encode_time(self.date.year, self.date.month, self.date.day, self.date.hour, self.date.minute, self.date.second))) time.standard_name = "time" time.calendar = "gregorian" @@ -238,21 +299,21 @@ class WriterMonarch(Writer): rlon[:] = self.grid.rlon settings.write_log("\t\t\t'rlon' variable created with size: {0}".format(rlon[:].shape), level=3) if LambertConformalConic: - x = netcdf.createVariable('x', 'd', ('x',), zlib=self.compress) - x.units = Unit("km").symbol - x.long_name = "x coordinate of projection" - x.standard_name = "projection_x_coordinate" + x_var = netcdf.createVariable('x', 'd', ('x',), zlib=self.compress) + x_var.units = Unit("km").symbol + x_var.long_name = "x coordinate of projection" + x_var.standard_name = "projection_x_coordinate" if settings.rank == 0: - x[:] = self.grid.x - settings.write_log("\t\t\t'x' variable created with size: {0}".format(x[:].shape), level=3) + x_var[:] = self.grid.x + settings.write_log("\t\t\t'x' variable created with size: {0}".format(x_var[:].shape), level=3) - y = netcdf.createVariable('y', 'd', ('y',), zlib=self.compress) - y.units = Unit("km").symbol - y.long_name = "y coordinate of projection" - y.standard_name = "projection_y_coordinate" + y_var = netcdf.createVariable('y', 'd', ('y',), zlib=self.compress) + y_var.units = Unit("km").symbol + y_var.long_name = "y coordinate of projection" + y_var.standard_name = "projection_y_coordinate" if settings.rank == 0: - y[:] = self.grid.y - settings.write_log("\t\t\t'y' variable created with size: {0}".format(y[:].shape), level=3) + y_var[:] = self.grid.y + settings.write_log("\t\t\t'y' variable created with size: {0}".format(y_var[:].shape), level=3) cell_area_dim = var_dim # Levels @@ -272,10 +333,10 @@ class WriterMonarch(Writer): var[:] = 0 index = 0 - for variable in self.variables_attributes: + for var_name, variable in self.variables_attributes.iteritems(): index += 1 - var = netcdf.createVariable(variable['name'], 'f', ('time',) + var_dim, zlib=self.compress) + var = netcdf.createVariable(var_name, 'f', ('time',) + var_dim, zlib=self.compress) var.units = Unit(variable['units']).symbol if 'long_name' in variable: @@ -293,8 +354,8 @@ class WriterMonarch(Writer): var.grid_mapping = 'rotated_pole' elif LambertConformalConic: var.grid_mapping = 'Lambert_conformal' - settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(variable['name'], var[:].shape) + - "\n\t\t\t\t'{0}' variable will be filled later.".format(variable['name']), level=3) + settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(var_name, var[:].shape) + + "\n\t\t\t\t'{0}' variable will be filled later.".format(var_name), level=3) settings.write_log("\t\tCreating NetCDF metadata.", level=2) # Grid mapping @@ -333,43 +394,18 @@ class WriterMonarch(Writer): netcdf.close() settings.write_time('WriterMonarch', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) + return True - def write_parallel_netcdf(self, emission_list): - - st_time = timeit.default_timer() - - settings.write_log("\tAppending data to parallel NetCDF file.", level=2) - if settings.size > 1: - netcdf = Dataset(self.path, mode='a', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) - else: - netcdf = Dataset(self.path, mode='a', format="NETCDF4") - settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) - index = 0 - # print "Rank {0} 2".format(rank) - for variable in self.variables_attributes: - - data = self.calculate_data_by_var(variable['name'], emission_list, self.grid.shape) - st_time = timeit.default_timer() - index += 1 - - var = netcdf.variables[variable['name']] - if settings.size > 1: - var.set_collective(True) - # Correcting NAN - if data is None: - data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] = data - - settings.write_log("\t\t\t'{0}' variable filled".format(variable['name'])) - - if self.grid.cell_area is not None: - c_area = netcdf.variables['cell_area'] - c_area[self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] = self.grid.cell_area + def write_serial_netcdf(self, emission_list,): + """ + Write the netCDF4 file in serial mode. - netcdf.close() - settings.write_time('WriterMonarch', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) + :param emission_list: Data to append. + :type emission_list: list - def write_serial_netcdf(self, emission_list,): + :return: True at end. + :rtype: bool + """ from cf_units import Unit, encode_time st_time = timeit.default_timer() @@ -379,7 +415,8 @@ class WriterMonarch(Writer): # Gathering the index if mpi_numpy or mpi_vector: - rank_position = np.array([self.grid.x_lower_bound, self.grid.x_upper_bound, self.grid.y_lower_bound, self.grid.y_upper_bound], dtype='i') + rank_position = np.array([self.grid.x_lower_bound, self.grid.x_upper_bound, self.grid.y_lower_bound, + self.grid.y_upper_bound], dtype='i') full_position = None if settings.rank == 0: full_position = np.empty([settings.size, 4], dtype='i') @@ -387,31 +424,33 @@ class WriterMonarch(Writer): if settings.rank == 0: - RegularLatLon = False - Rotated = False - LambertConformalConic = False + regular_latlon = False + rotated = False + lcc = False if self.grid.grid_type == 'global': - RegularLatLon = True + regular_latlon = True elif self.grid.grid_type == 'rotated': - Rotated = True + rotated = True elif self.grid.grid_type == 'lcc': - LambertConformalConic = True + lcc = True settings.write_log("\tCreating NetCDF file.", level=2) netcdf = Dataset(self.path, mode='w', format="NETCDF4") # ===== Dimensions ===== settings.write_log("\t\tCreating NetCDF dimensions.", level=2) - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude if len(self.grid.center_latitudes.shape) == 1: - settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) + settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) netcdf.createDimension('lat', self.grid.center_latitudes.shape[0]) lat_dim = ('lat',) elif len(self.grid.center_latitudes.shape) == 2: - settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) + settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) netcdf.createDimension('lat', self.grid.center_latitudes.shape[0]) lat_dim = ('lon', 'lat', ) else: @@ -424,11 +463,13 @@ class WriterMonarch(Writer): # Longitude if len(self.grid.center_longitudes.shape) == 1: - settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[0]), level=3) + settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[0]), + level=3) netcdf.createDimension('lon', self.grid.center_longitudes.shape[0]) lon_dim = ('lon',) elif len(self.grid.center_longitudes.shape) == 2: - settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[0]), level=3) + settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[0]), + level=3) netcdf.createDimension('lon', self.grid.center_longitudes.shape[1]) lon_dim = ('lon', 'lat', ) else: @@ -438,10 +479,10 @@ class WriterMonarch(Writer): 'ERROR: Longitudes must be on a 1D or 2D array instead of {0} shape.'.format( len(self.grid.center_longitudes.shape))) sys.exit(1) - elif Rotated: + elif rotated: var_dim = ('rlat', 'rlon',) - # Rotated Latitude + # rotated Latitude if self.grid.rlat is None: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -451,7 +492,7 @@ class WriterMonarch(Writer): netcdf.createDimension('rlat', len(self.grid.rlat)) lat_dim = ('rlat', 'rlon',) - # Rotated Longitude + # rotated Longitude if self.grid.rlon is None: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -461,7 +502,7 @@ class WriterMonarch(Writer): netcdf.createDimension('rlon', len(self.grid.rlon)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic: + elif lcc: var_dim = ('y', 'x',) settings.write_log("\t\t\t'y' dimension: {0}".format(len(self.grid.y)), level=3) netcdf.createDimension('y', len(self.grid.y)) @@ -469,6 +510,10 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'x' dimension: {0}".format(len(self.grid.x)), level=3) netcdf.createDimension('x', len(self.grid.x)) lon_dim = ('y', 'x', ) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if self.levels is not None: @@ -496,9 +541,9 @@ class WriterMonarch(Writer): time[:] = [0.] else: time = netcdf.createVariable('time', 'd', ('time',)) - u = Unit('hours') - time.units = str(u.offset_by_time(encode_time( - self.date.year, self.date.month, self.date.day, self.date.hour, self.date.minute, self.date.second))) + time.units = str(Unit('hours').offset_by_time(encode_time( + self.date.year, self.date.month, self.date.day, self.date.hour, self.date.minute, + self.date.second))) time.standard_name = "time" time.calendar = "gregorian" time.long_name = "time" @@ -540,8 +585,8 @@ class WriterMonarch(Writer): settings.write_log( "\t\t\t'lon_bnds' variable created with size: {0}".format(lon_bnds[:].shape), level=3) - if Rotated: - # Rotated Latitude + if rotated: + # rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=self.compress) rlat.long_name = "latitude in rotated pole grid" rlat.units = Unit("degrees").symbol @@ -549,27 +594,27 @@ class WriterMonarch(Writer): rlat[:] = self.grid.rlat settings.write_log("\t\t\t'rlat' variable created with size: {0}".format(rlat[:].shape), level=3) - # Rotated Longitude + # rotated Longitude rlon = netcdf.createVariable('rlon', 'f', ('rlon',), zlib=self.compress) rlon.long_name = "longitude in rotated pole grid" rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = self.grid.rlon settings.write_log("\t\t\t'rlon' variable created with size: {0}".format(rlon[:].shape), level=3) - if LambertConformalConic: - x = netcdf.createVariable('x', 'd', ('x',), zlib=self.compress) - x.units = Unit("km").symbol - x.long_name = "x coordinate of projection" - x.standard_name = "projection_x_coordinate" - x[:] = self.grid.x - settings.write_log("\t\t\t'x' variable created with size: {0}".format(x[:].shape), level=3) - - y = netcdf.createVariable('y', 'd', ('y',), zlib=self.compress) - y.units = Unit("km").symbol - y.long_name = "y coordinate of projection" - y.standard_name = "projection_y_coordinate" - y[:] = self.grid.y - settings.write_log("\t\t\t'y' variable created with size: {0}".format(y[:].shape), level=3) + if lcc: + x_var = netcdf.createVariable('x', 'd', ('x',), zlib=self.compress) + x_var.units = Unit("km").symbol + x_var.long_name = "x coordinate of projection" + x_var.standard_name = "projection_x_coordinate" + x_var[:] = self.grid.x + settings.write_log("\t\t\t'x' variable created with size: {0}".format(x_var[:].shape), level=3) + + y_var = netcdf.createVariable('y', 'd', ('y',), zlib=self.compress) + y_var.units = Unit("km").symbol + y_var.long_name = "y coordinate of projection" + y_var.standard_name = "projection_y_coordinate" + y_var[:] = self.grid.y + settings.write_log("\t\t\t'y' variable created with size: {0}".format(y_var[:].shape), level=3) cell_area_dim = var_dim # Levels @@ -587,16 +632,17 @@ class WriterMonarch(Writer): full_shape = None index = 0 - for variable in self.variables_attributes: + for var_name in self.variables_attributes.iterkeys(): if settings.size != 1: - settings.write_log("\t\t\tGathering {0} data.".format(variable['name']), level=3) - rank_data = self.calculate_data_by_var(variable['name'], emission_list, self.grid.shape) + settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) + rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) if mpi_numpy or mpi_vector: if rank_data is not None: root_shape = settings.comm.bcast(rank_data.shape, root=0) if full_shape is None: full_shape = settings.comm.allgather(rank_data.shape) # print 'Rank {0} full_shape: {1}\n'.format(settings.rank, full_shape) + if mpi_numpy: if settings.size != 1: if settings.rank == 0: @@ -644,31 +690,31 @@ class WriterMonarch(Writer): if settings.size != 1: try: data = np.concatenate(data, axis=3) - except: + except (UnboundLocalError, TypeError, IndexError): data = 0 index += 1 - var = netcdf.createVariable(variable['name'], 'f', ('time',) + var_dim, zlib=self.compress) + var = netcdf.createVariable(var_name, 'f', ('time',) + var_dim, zlib=self.compress) - var.units = Unit(variable['units']).symbol + var.units = Unit(self.variables_attributes[var_name]['units']).symbol - if 'long_name' in variable: - var.long_name = str(variable['long_name']) + if 'long_name' in self.variables_attributes[var_name]: + var.long_name = str(self.variables_attributes[var_name]['long_name']) - if 'standard_name' in variable: - var.standard_name = str(variable['standard_name']) + if 'standard_name' in self.variables_attributes[var_name]: + var.standard_name = str(self.variables_attributes[var_name]['standard_name']) - if 'cell_method' in variable: - var.cell_method = str(variable['cell_method']) + if 'cell_method' in self.variables_attributes[var_name]: + var.cell_method = str(self.variables_attributes[var_name]['cell_method']) var.coordinates = "lat lon" if self.grid.cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif rotated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' if mpi_numpy: @@ -682,7 +728,7 @@ class WriterMonarch(Writer): else: var[:, :, :, full_position[i][2]:full_position[i][3]] = \ recvbuf[i, :, :, :, : full_shape[i][-1]] - except: + except ValueError: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise TypeError("ERROR on i {0} ".format(i) + @@ -695,32 +741,36 @@ class WriterMonarch(Writer): data = np.empty(var[:].shape, dtype=settings.precision) for i in xrange(settings.size): if not i == settings.size - 1: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) else: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) else: data = 0 var[:] = data else: var[:] = data - settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(variable['name'], var[:].shape), + settings.write_log("\t\t\t'{0}' variable created with size: {1}".format(var_name, var[:].shape), level=3) settings.write_log("\t\tCreating NetCDF metadata.", level=2) if settings.rank == 0: # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: - # Rotated pole + elif rotated: + # rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = 90 - self.grid.new_pole_latitude_degrees mapping.grid_north_pole_longitude = self.grid.new_pole_longitude_degrees - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" @@ -748,4 +798,3 @@ class WriterMonarch(Writer): if settings.rank == 0: netcdf.close() settings.write_time('WriterMonarch', 'write_serial_netcdf', timeit.default_timer() - st_time, level=3) - diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index b42a0db4e5bb8031300bb6ed3c4d203acc9a53fd..5cf01cf0dd1cebf6cd8743b1eb1aa4bbd0d7f47d 100644 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -18,17 +18,43 @@ # along with HERMESv3_GR. If not, see . -from hermesv3_gr.modules.writing.writer import Writer -import timeit -from hermesv3_gr.config import settings -import os import sys +import timeit import numpy as np from netCDF4 import Dataset from mpi4py import MPI +from hermesv3_gr.config import settings +from hermesv3_gr.modules.writing.writer import Writer class WriterWrfChem(Writer): + """ + Class to Write the output file for the WRF-CHEM Chemical Transport Model. + + :param path: Path to the destination file. + :type path: str + + :param grid: Grid of the destination file. + :type grid: Grid + + :param levels: List with the levels of the grid. + :type levels: list + + :param date: Date of the output file + :type date: datetime.datetime + + :param hours: List with the timestamp hours. + :type hours: list. + + :param global_attributes_path: Path to the file that contains the static global attributes. + :type global_attributes_path: str + + :param compress: Indicates if you want to compress the netCDF variable data. + :type compress: bool + + :param parallel: Indicates if you want to write in parallel mode. + :type parallel. bool + """ def __init__(self, path, grid, levels, date, hours, global_attributes_path, compress=True, parallel=False): super(WriterWrfChem, self).__init__(path, grid, levels, date, hours, global_attributes_path, compress, parallel) @@ -37,7 +63,7 @@ class WriterWrfChem(Writer): 'TITLE', 'START_DATE', 'WEST-EAST_GRID_DIMENSION', 'SOUTH-NORTH_GRID_DIMENSION', 'BOTTOM-TOP_GRID_DIMENSION', 'DX', 'DY', 'GRIDTYPE', 'DIFF_OPT', 'KM_OPT', 'DAMP_OPT', 'DAMPCOEF', 'KHDIF', 'KVDIF', 'MP_PHYSICS', 'RA_LW_PHYSICS', 'RA_SW_PHYSICS', 'SF_SFCLAY_PHYSICS', 'SF_SURFACE_PHYSICS', - 'BL_PBL_PHYSICS', 'CU_PHYSICS', 'SF_LAKE_PHYSICS', 'SURFACE_INPUT_SOURCE','SST_UPDATE', 'GRID_FDDA', + 'BL_PBL_PHYSICS', 'CU_PHYSICS', 'SF_LAKE_PHYSICS', 'SURFACE_INPUT_SOURCE', 'SST_UPDATE', 'GRID_FDDA', 'GFDDA_INTERVAL_M', 'GFDDA_END_H', 'GRID_SFDDA', 'SGFDDA_INTERVAL_M', 'SGFDDA_END_H', 'WEST-EAST_PATCH_START_UNSTAG', 'WEST-EAST_PATCH_END_UNSTAG', 'WEST-EAST_PATCH_START_STAG', 'WEST-EAST_PATCH_END_STAG', 'SOUTH-NORTH_PATCH_START_UNSTAG', 'SOUTH-NORTH_PATCH_END_UNSTAG', @@ -48,6 +74,13 @@ class WriterWrfChem(Writer): 'MAP_PROJ', 'MMINLU', 'NUM_LAND_CAT', 'ISWATER', 'ISLAKE', 'ISICE', 'ISURBAN', 'ISOILWATER'] def unit_change(self, variable, data): + # TODO Documentation + """ + + :param variable: + :param data: + :return: + """ from cf_units import Unit if data is not None: @@ -74,6 +107,11 @@ class WriterWrfChem(Writer): return data def change_variable_attributes(self): + # TODO Documentation + """ + + :return: + """ from cf_units import Unit new_variable_dict = {} @@ -102,9 +140,15 @@ class WriterWrfChem(Writer): raise TypeError("The unit '{0}' of specie {1} is not ".format(variable['units'], variable['name']) + "defined correctly. Should be 'mol.h-1.km-2' or 'ug.s-1.m-2'") sys.exit(1) + self.variables_attributes = new_variable_dict def read_global_attributes(self): + # TODO Documentation + """ + + :return: + """ import pandas as pd from warnings import warn as warning @@ -183,7 +227,8 @@ class WriterWrfChem(Writer): except ValueError: print 'A warning has occurred. Check the .err file to get more information.' if settings.rank == 0: - warning('The global attribute {0} is not defined; Using default value {1}'.format(att, atts_dict[att])) + warning('The global attribute {0} is not defined; Using default value {1}'.format( + att, atts_dict[att])) else: settings.write_log('WARNING: Check the .err file to get more information.') @@ -197,8 +242,9 @@ class WriterWrfChem(Writer): return atts_dict def create_global_attributes(self): + # TODO Documentation """ - Creates the global attributes that have to be filled. + Create the global attributes that have to be filled. """ global_attributes = self.read_global_attributes() @@ -247,6 +293,11 @@ class WriterWrfChem(Writer): return global_attributes def create_times_var(self): + # TODO Documentation + """ + + :return: + """ from datetime import timedelta import netCDF4 @@ -260,32 +311,38 @@ class WriterWrfChem(Writer): return str_out def create_parallel_netcdf(self): + # TODO Documentation + """ + + :return: + """ st_time = timeit.default_timer() settings.write_log("\tCreating parallel NetCDF file.", level=2) netcdf = Dataset(self.path, mode='w', format="NETCDF4") - if settings.rank == 0: - # ===== Dimensions ===== - settings.write_log("\t\tCreating NetCDF dimensions.", level=2) - netcdf.createDimension('Time', None) - settings.write_log("\t\t\t'Time' dimension: {0}".format('UNLIMITED ({0})'.format(len(self.hours))), level=3) - netcdf.createDimension('DateStrLen', 19) - settings.write_log("\t\t\t'DateStrLen' dimension: 19", level=3) - netcdf.createDimension('west_east', self.grid.center_longitudes.shape[1]) - settings.write_log("\t\t\t'west_east' dimension: {0}".format(len(self.hours)), level=3) - netcdf.createDimension('south_north', self.grid.center_latitudes.shape[0]) - settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) - netcdf.createDimension('emissions_zdim', len(self.levels)) - settings.write_log("\t\t\t'emissions_zdim' dimension: {0}".format(len(self.levels)), level=3) - - # ===== Variables ===== - settings.write_log("\t\tCreating NetCDF variables.", level=2) - times = netcdf.createVariable('Times', 'S1', ('Time', 'DateStrLen', )) - times[:] = self.create_times_var() - settings.write_log("\t\t\t'Times' variable created with size: {0}".format(times[:].shape), level=3) + # ===== Dimensions ===== + settings.write_log("\t\tCreating NetCDF dimensions.", level=2) + netcdf.createDimension('Time', len(self.hours)) + # netcdf.createDimension('Time', None) + settings.write_log("\t\t\t'Time' dimension: {0}".format('UNLIMITED ({0})'.format(len(self.hours))), + level=3) + netcdf.createDimension('DateStrLen', 19) + settings.write_log("\t\t\t'DateStrLen' dimension: 19", level=3) + netcdf.createDimension('west_east', self.grid.center_longitudes.shape[1]) + settings.write_log("\t\t\t'west_east' dimension: {0}".format(len(self.hours)), level=3) + netcdf.createDimension('south_north', self.grid.center_latitudes.shape[0]) + settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) + netcdf.createDimension('emissions_zdim', len(self.levels)) + settings.write_log("\t\t\t'emissions_zdim' dimension: {0}".format(len(self.levels)), level=3) + + # ===== Variables ===== + settings.write_log("\t\tCreating NetCDF variables.", level=2) + times = netcdf.createVariable('Times', 'S1', ('Time', 'DateStrLen', )) + times[:] = self.create_times_var() + settings.write_log("\t\t\t'Times' variable created with size: {0}".format(times[:].shape), level=3) index = 0 - self.change_variable_attributes() for var_name in self.variables_attributes.iterkeys(): index += 1 var = netcdf.createVariable(var_name, 'f', ('Time', 'emissions_zdim', 'south_north', 'west_east',), @@ -304,34 +361,15 @@ class WriterWrfChem(Writer): settings.write_time('WriterCmaq', 'create_parallel_netcdf', timeit.default_timer() - st_time, level=3) - def write_parallel_netcdf(self, emission_list): - st_time = timeit.default_timer() - settings.write_log("\tAppending data to parallel NetCDF file.", level=2) - - if settings.size > 1: - netcdf = Dataset(self.path, mode='a', format="NETCDF4", parallel=True, comm=settings.comm, info=MPI.Info()) - else: - netcdf = Dataset(self.path, mode='a', format="NETCDF4") - settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) - index = 0 - for var_name in self.variables_attributes.iterkeys(): - data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) - - index += 1 - - var = netcdf.variables[var_name] - if settings.size > 1: - var.set_collective(True) - # Correcting NAN - if data is None: - data = 0 - var[:, :, self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] = data - settings.write_log("\t\t\t'{0}' variable filled".format(var_name)) - - netcdf.close() - settings.write_time('WriterCmaq', 'write_parallel_netcdf', timeit.default_timer() - st_time, level=3) + return True def write_serial_netcdf(self, emission_list): + # TODO Documentation + """ + + :param emission_list: + :return: + """ st_time = timeit.default_timer() # Gathering the index @@ -356,7 +394,8 @@ class WriterWrfChem(Writer): netcdf.createDimension('west_east', self.grid.center_longitudes.shape[1]) settings.write_log("\t\t\t'west_east' dimension: {0}".format(len(self.hours)), level=3) netcdf.createDimension('south_north', self.grid.center_latitudes.shape[0]) - settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) + settings.write_log("\t\t\t'south_north' dimension: {0}".format(self.grid.center_latitudes.shape[0]), + level=3) netcdf.createDimension('emissions_zdim', len(self.levels)) settings.write_log("\t\t\t'emissions_zdim' dimension: {0}".format(len(self.levels)), level=3) @@ -369,7 +408,7 @@ class WriterWrfChem(Writer): full_shape = None index = 0 - self.change_variable_attributes() + # self.change_variable_attributes() for var_name in self.variables_attributes.iterkeys(): if settings.size != 1: @@ -404,12 +443,13 @@ class WriterWrfChem(Writer): if settings.size != 1: try: data = np.concatenate(data, axis=3) - except: + except (UnboundLocalError, TypeError, IndexError): data = 0 st_time = timeit.default_timer() index += 1 - var = netcdf.createVariable(var_name, 'f', ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=self.compress) + var = netcdf.createVariable(var_name, 'f', ('Time', 'emissions_zdim', 'south_north', 'west_east',), + zlib=self.compress) var.setncatts(self.variables_attributes[var_name]) var_time = timeit.default_timer() @@ -421,9 +461,13 @@ class WriterWrfChem(Writer): for i in xrange(settings.size): # print 'Resizeing {0}'.format(i) if not i == settings.size - 1: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]: displacements[i + 1]]).reshape(full_shape[i]) else: - data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) + data[:, :, full_position[i][0]:full_position[i][1], + full_position[i][2]:full_position[i][3]] = \ + np.array(recvbuf[0][displacements[i]:]).reshape(full_shape[i]) else: data = 0 var[:] = data @@ -439,6 +483,3 @@ class WriterWrfChem(Writer): netcdf.close() settings.write_time('WriterWrfChem', 'write_serial_netcdf', timeit.default_timer() - st_time, level=3) return True - - - diff --git a/hermesv3_gr/tools/coordinates_tools.py b/hermesv3_gr/tools/coordinates_tools.py index 4c1b767279fbe3f4e94c59ac73b46ebb6ba2def8..8dbc7cd496f0ea0ff1c9b92df8349599b0eea9ec 100644 --- a/hermesv3_gr/tools/coordinates_tools.py +++ b/hermesv3_gr/tools/coordinates_tools.py @@ -17,21 +17,16 @@ # You should have received a copy of the GNU General Public License # along with HERMESv3_GR. If not, see . -import os -import sys -# import numpy as np -# import math - - -# Global variables - def get_grid_area(filename): - # TODO Documentation """ + Calculate the area for each cell of the grid using CDO - :param filename: - :return: + :param filename: Path to the file to calculate the cell area + :type filename: str + + :return: Area of each cell of the grid. + :rtype: numpy.array """ from cdo import Cdo from netCDF4 import Dataset @@ -48,6 +43,7 @@ def get_grid_area(filename): def latlon2rotated(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): # TODO Documentation """ + Transform lat lon degrees into the rotated coordinates. :param lon_pole_deg: :param lat_pole_deg: @@ -61,16 +57,16 @@ def latlon2rotated(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): degrees_to_radians = math.pi / 180. radians_to_degrees = 180. / math.pi - lon_max = lon_min + 360 + # lon_max = lon_min + 360 # stlm=sin(tlm) sin_lat_pole_rad = math.sin(lat_pole_deg * degrees_to_radians) # ctlm=cos(tlm) cos_lat_pole_rad = math.cos(lat_pole_deg * degrees_to_radians) # stph=sin(tph) - sin_lon_pole_rad = math.sin(lon_pole_deg * degrees_to_radians) + # sin_lon_pole_rad = math.sin(lon_pole_deg * degrees_to_radians) # ctph=cos(tph) - cos_lon_pole_rad = math.cos(lon_pole_deg * degrees_to_radians) + # cos_lon_pole_rad = math.cos(lon_pole_deg * degrees_to_radians) # relm=(xlon-tlm0d)*dtr !distance from the centre lon (in rad) distance_from_center_lon = (lon_deg - lon_pole_deg) * degrees_to_radians @@ -107,6 +103,7 @@ def latlon2rotated(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): def rotated2latlon(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): # TODO Documentation """ + Transform rotated coordinates into lat lon degrees. :param lon_pole_deg: :param lat_pole_deg: @@ -119,7 +116,7 @@ def rotated2latlon(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): import math degrees_to_radians = math.pi / 180. - radians_to_degrees = 180. / math.pi + # radians_to_degrees = 180. / math.pi # Positive east to negative east lon_pole_deg -= 180 @@ -159,7 +156,7 @@ def rotated2latlon(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min=-180): # almd += 360 # elif almd > max_lon: # almd -= 360 - # TODO use lon_min + almd[almd > (lon_min + 360)] -= 360 almd[almd < lon_min] += 360 @@ -180,42 +177,9 @@ def rotated2latlon_single(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min= import math degrees_to_radians = math.pi / 180. - radians_to_degrees = 180. / math.pi + # radians_to_degrees = 180. / math.pi # lon_max = lon_min + 360 - # - # sin_lat_pole_rad = math.sin(lat_pole_deg*degrees_to_radians) - # cos_lat_pole_rad = math.cos(lat_pole_deg*degrees_to_radians) # - # sin_lon_pole_rad = math.sin(lon_pole_deg*degrees_to_radians) # stph - # cos_lon_pole_rad = math.cos(lon_pole_deg*degrees_to_radians) # ctph - # - # - # # relm=(xlon-tlm0d)*dtr !distance from the centre lon (in rad) - # distance_from_center_lon = (lon_deg - lon_pole_deg)*degrees_to_radians - # # ctlm=cos(relm) !cos of this distance - # cos_distance_from_center_lon = math.cos(distance_from_center_lon) - # # stlm=sin(relm) !sin of this distance - # sin_distance_from_center_lon = math.sin(distance_from_center_lon) - # # aph=xlat*dtr !lat in rad - # lat_rad = lat_deg*degrees_to_radians - # # ctph=cos(aph) !cos of lat - # cos_lat_rad = math.cos(lat_rad) - # # stph=sin(aph) !sin of lat - # sin_lat_rad = math.sin(lat_rad) - # - # - # - # # sph=ctph0*stph+stph0*ctph*ctlm - # sin_rotated_lat = (cos_lat_pole_rad*sin_lat_rad) + (sin_distance_from_center_lon*cos_lat_rad*cos_distance_from_center_lon) - # # sph=min(sph,1.) - # # sph=max(sph,-1.) - # if sin_rotated_lat > 1.: - # sin_rotated_lat = 1. - # if sin_rotated_lat < -1.: - # sin_rotated_lat = -1. - # # aph=asin(sph) - # real_latitude = math.asin(sin_rotated_lat) - # real_longitude = math.atan2(cos_lat_rad*sin_distance_from_center_lon, (cos_distance_from_center_lon*cos_lat_rad - sin_lat_pole_rad*sin_rotated_lat)/cos_lat_pole_rad) - math.pi # Positive east to negative east lon_pole_deg -= 180 @@ -258,17 +222,17 @@ def rotated2latlon_single(lon_pole_deg, lat_pole_deg, lon_deg, lat_deg, lon_min= def create_bounds(coords, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) - :type coords: numpy.ndarray + :type coords: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np @@ -298,7 +262,7 @@ def create_bounds_esmpy(coords, spheric=False): interval = coords[1] - coords[0] - bound_coords = coords - interval/2 + bound_coords = coords - interval / 2 if not spheric: bound_coords = np.append(bound_coords, [bound_coords[-1] + interval]) @@ -322,9 +286,6 @@ def create_regular_rotated(lat_origin, lon_origin, lat_inc, lon_inc, n_lat, n_lo center_latitudes = np.arange(lat_origin, lat_origin + (n_lat*lat_inc), lat_inc, dtype=np.float) center_longitudes = np.arange(lon_origin, lon_origin + (n_lon*lon_inc), lon_inc, dtype=np.float) - # print lat_origin + (n_lat*lat_inc) - # print n_lat*lat_inc - corner_latitudes = create_bounds_esmpy(center_latitudes) corner_longitudes = create_bounds_esmpy(center_longitudes) @@ -338,65 +299,15 @@ def create_regular_old(lat_origin, lon_origin, lat_inc, lon_inc, n_lat, n_lon): center_latitudes = np.arange(lat_origin, lat_origin + (n_lat*lat_inc), lat_inc, dtype=np.float) center_longitudes = np.arange(lon_origin, lon_origin + (n_lon*lon_inc), lon_inc, dtype=np.float) - # print lat_origin + (n_lat*lat_inc) - # print n_lat*lat_inc - corner_latitudes = create_bounds(center_latitudes) corner_longitudes = create_bounds(center_longitudes) return center_latitudes, center_longitudes, corner_latitudes, corner_longitudes -# def create_regular_grid(center_lat, center_lon, west_boundary, south_boundary, inc_lat, inc_lon): -# """ -# Creates a custom grid with the given parameters. The grid is divided in 4 arrays: -# - Center Latitudes -# - Center Longitudes -# - Boundary Latitudes (# latitudes +1) -# - Boundary Longitudes (# longitudes +1) -# -# :param center_lat: Latitude of the center of the grid (degrees). -# :type center_lat: float -# -# :param center_lon: Longitude of the center of the grid (degrees). -# :type center_lon: float -# -# :param west_boundary: Distance from de center to the western boundary (degrees) -# (not to the center of the first cell) -# :type west_boundary: float -# -# :param south_boundary: Distance from de center to the southern boundary (degrees) -# (not to the center of the first cell) -# :type south_boundary: float -# -# :param inc_lat: Vertical resolution of each cell (degrees). -# :type inc_lat: float -# -# :param inc_lon: Horizontal resolution of each cell (degrees) -# :type inc_lon: float -# -# :return: Arrays with the Center Latitudes, Center Longitudes, Boundary Latitudes, Boundary Longitudes. -# :rtype: tuple (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) -# """ -# lat_origin = center_lat - abs(south_boundary) + (inc_lat/2) -# lon_origin = center_lon - abs(west_boundary) + (inc_lon/2) -# n_lat = (abs(south_boundary)/inc_lat)*2 -# n_lon = (abs(west_boundary)/inc_lon)*2 -# -# center_latitudes = np.arange(lat_origin, lat_origin + (n_lat*inc_lat), inc_lat, dtype=np.float) -# center_longitudes = np.arange(lon_origin, lon_origin + (n_lon*inc_lon), inc_lon, dtype=np.float) -# -# corner_latitudes = create_bounds(center_latitudes) -# corner_longitudes = create_bounds(center_longitudes) -# -# # print center_latitudes -# -# return center_latitudes, center_longitudes, corner_latitudes, corner_longitudes - - def create_regular_grid(center_lat, center_lon, west_boundary, south_boundary, inc_lat, inc_lon): """ - Creates a custom grid with the given parameters. The grid is divided in 4 arrays: + Create a custom grid with the given parameters. The grid is divided in 4 arrays: - Center Latitudes - Center Longitudes - Boundary Latitudes (# latitudes +1) @@ -423,7 +334,7 @@ def create_regular_grid(center_lat, center_lon, west_boundary, south_boundary, i :type inc_lon: float :return: Arrays with the Center Latitudes, Center Longitudes, Boundary Latitudes, Boundary Longitudes. - :rtype: tuple (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) + :rtype: tuple (numpy.array, numpy.array, numpy.array, numpy.array) """ import numpy as np @@ -432,58 +343,44 @@ def create_regular_grid(center_lat, center_lon, west_boundary, south_boundary, i n_lat = (abs(south_boundary)/inc_lat)*2 n_lon = (abs(west_boundary)/inc_lon)*2 - center_latitudes = np.arange(lat_origin + inc_lat, lat_origin + (n_lat*inc_lat) - inc_lat + inc_lat/2, inc_lat, + center_latitudes = np.arange(lat_origin + inc_lat, lat_origin + (n_lat*inc_lat) - inc_lat + inc_lat / 2, inc_lat, dtype=np.float) - center_longitudes = np.arange(lon_origin + inc_lon, lon_origin + (n_lon*inc_lon) - inc_lon + inc_lon/2, inc_lon, + center_longitudes = np.arange(lon_origin + inc_lon, lon_origin + (n_lon*inc_lon) - inc_lon + inc_lon / 2, inc_lon, dtype=np.float) corner_latitudes = create_bounds(center_latitudes) corner_longitudes = create_bounds(center_longitudes) center_latitudes = np.concatenate([ - [lat_origin + inc_lat/2 - inc_lat/4], + [lat_origin + inc_lat / 2 - inc_lat / 4], center_latitudes, - [lat_origin + (n_lat*inc_lat) - inc_lat/2 + inc_lat/4]]) + [lat_origin + (n_lat * inc_lat) - inc_lat / 2 + inc_lat / 4]]) center_longitudes = np.concatenate([ - [lon_origin + inc_lon/2 - inc_lon/4], + [lon_origin + inc_lon / 2 - inc_lon / 4], center_longitudes, - [lon_origin + (n_lon*inc_lon) - inc_lon/2 + inc_lon/4]]) + [lon_origin + (n_lon * inc_lon) - inc_lon / 2 + inc_lon / 4]]) corner_latitudes = np.concatenate([ - [[[lat_origin, lat_origin + inc_lat/2]]], + [[[lat_origin, lat_origin + inc_lat / 2]]], corner_latitudes, - [[[lat_origin + (n_lat*inc_lat) - inc_lat/2, lat_origin + (n_lat*inc_lat)]]]], axis=1) + [[[lat_origin + (n_lat * inc_lat) - inc_lat / 2, lat_origin + (n_lat * inc_lat)]]]], axis=1) corner_longitudes = np.concatenate([ - [[[lon_origin, lon_origin + inc_lon/2]]], + [[[lon_origin, lon_origin + inc_lon / 2]]], corner_longitudes, - [[[lon_origin + (n_lon*inc_lon) - inc_lon/2, lon_origin + (n_lon*inc_lon)]]]], axis=1) + [[[lon_origin + (n_lon * inc_lon) - inc_lon / 2, lon_origin + (n_lon * inc_lon)]]]], axis=1) return center_latitudes, center_longitudes, corner_latitudes, corner_longitudes if __name__ == '__main__': import numpy as np - new_pole_longitude_degrees = 20.0 # lonpole tlm0d - new_pole_latitude_degrees = 35.0 # latpole tph0d - # - print latlon2rotated(new_pole_longitude_degrees, new_pole_latitude_degrees, 20.0, 35.0) - print latlon2rotated(new_pole_longitude_degrees, new_pole_latitude_degrees, -20.2485, -9.9036) - # - print rotated2latlon_single(new_pole_longitude_degrees, new_pole_latitude_degrees, 0, 0) - print rotated2latlon_single(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -35.) - # # print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -34.9) - # # print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -34.8) - # # print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, -51., -34.7) - print rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, np.array([-51., -51., -51., -51.]), - np.array([-35., -34.9, -34.8, -34.7])) - # - # lat, lon = rotated2latlon(new_pole_longitude_degrees, new_pole_latitude_degrees, np.array([0]), np.array([0])) - # print lat - - # lat, lon, b_lat, b_lon = create_regular_grid(0, 0, -180, -90, 1., 1.) - # print lat - # print lon - # print b_lat - # print b_lon + new_pole_lon_d = 20.0 # lonpole tlm0d + new_pole_lat_d = 35.0 # latpole tph0d + print latlon2rotated(new_pole_lon_d, new_pole_lat_d, 20.0, 35.0) + print latlon2rotated(new_pole_lon_d, new_pole_lat_d, -20.2485, -9.9036) + print rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, 0, 0) + print rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, -51., -35.) + print rotated2latlon(new_pole_lon_d, new_pole_lat_d, np.array([-51., -51., -51., -51.]), + np.array([-35., -34.9, -34.8, -34.7])) diff --git a/hermesv3_gr/tools/custom_calendar.py b/hermesv3_gr/tools/custom_calendar.py index 256bda95e0d8c9f9bd89c6b1ee5739c25fba8d59..c221328de61f4fc73ea6fa0c4c252aa9629f29bd 100644 --- a/hermesv3_gr/tools/custom_calendar.py +++ b/hermesv3_gr/tools/custom_calendar.py @@ -26,7 +26,7 @@ import holidays def custom_holidays(zone, year): """ - Calculates the festivity days that appear in the library holidays adding the Maundy Thursday and the God Friday + Calculate the festivity days that appear in the library holidays adding the Maundy Thursday and the God Friday :param zone: Name of the country. It has to appear and has to have the same format (capital letters) of the library holidays: https://pypi.python.org/pypi/holidays @@ -69,7 +69,7 @@ def get_holidays(zone, year): def pascua(year): """ - Calculates the "Pascua" date + Calculate the "Pascua" date :param year: Year to found the Pascua. :type year: int diff --git a/hermesv3_gr/tools/lcc_LatLon_to_m.py b/hermesv3_gr/tools/lcc_LatLon_to_m.py deleted file mode 100644 index 5fbe9d96b664c4f0ef1b21d7885e3208adabce95..0000000000000000000000000000000000000000 --- a/hermesv3_gr/tools/lcc_LatLon_to_m.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -from pyproj import Proj -if __name__ == '__main__': - projection = Proj( - proj='lcc', - ellps='WGS84', - R=6370000.000, - lat_1=37, - lat_2=43, - lon_0=-3, - lat_0=40, - to_meter=1, - x_0=0, - y_0=0, - a=6370000.000, - k_0=1.0) - lon_array = [-11.5488, -11.5066, 7.1104] - lat_array = [32.5108, 32.5142, 46.6579] - UTMx, UTMy = projection(lon_array, lat_array) - - print UTMx, UTMy \ No newline at end of file diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index d09423357f764f0dadc39894ae838ff5319e464c..f0d72dede20951049303d4de871368074ac39090 100644 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -22,46 +22,76 @@ import sys from netCDF4 import Dataset from mpi4py import MPI -icomm = MPI.COMM_WORLD -comm = icomm.Split(color=0, key=0) -rank = comm.Get_rank() -size = comm.Get_size() +ICOMM = MPI.COMM_WORLD +COMM = ICOMM.Split(color=0, key=0) +RANK = COMM.Get_rank() +SIZE = COMM.Get_size() def open_netcdf(netcdf_path): - from netCDF4 import Dataset - nc_out = Dataset(netcdf_path, mode='a') - return nc_out + """ + Open a netCDF file. + + :param netcdf_path: Path to the netCDF file. + :type netcdf_path: str + + :return: netCDF + :rtype: Dataset + """ + netcdf = Dataset(netcdf_path, mode='a') + return netcdf -def close_netcdf(nc): - nc.close() +def close_netcdf(netcdf): + """ + Close the netCDF. + + :param netcdf: netCDF + :type netcdf: Dataset + :return: + """ + netcdf.close() def get_grid_area(filename): """ - Calculates the area of each cell. + Calculate the area of each cell. :param filename: Full path to the NetCDF to calculate the cell areas. :type filename: str :return: Returns the area of each cell. - :rtype: numpy.ndarray + :rtype: numpy.array """ from cdo import Cdo cdo = Cdo() - s = cdo.gridarea(input=filename) - nc_aux = Dataset(s, mode='r') - grid_area = nc_aux.variables['cell_area'][:] - nc_aux.close() + src = cdo.gridarea(input=filename) + netcdf = Dataset(src, mode='r') + grid_area = netcdf.variables['cell_area'][:] + netcdf.close() return grid_area -def extract_vars(netcdf_path, variables_list, attributes_list=list()): +def extract_vars(netcdf_path, variables_list, attributes_list=()): + """ + Get the data from the list of variabbles. + + :param netcdf_path: Path to the netCDF file + :type netcdf_path: str + + :param variables_list: List of the names of the variables to get. + :type variables_list: list + + :param attributes_list: List of the names of the variable attributes to get. + :type attributes_list: list + + :return: List of the variables from the netCDF as a dictionary with data as values and with the other keys their + attributes. + :rtype: list. + """ data_list = [] - # print netcdf_path netcdf = Dataset(netcdf_path, mode='r') for var in variables_list: if var == 'emi_nox_no2': @@ -85,19 +115,49 @@ def extract_vars(netcdf_path, variables_list, attributes_list=list()): def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, - RegularLatLon=False, - Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, - LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, - Mercator=False, lat_ts=None): + regular_latlon=False, + rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, + lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None, + mercator=False, lat_ts=None): + # TODO Documentation + """ + + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param boundary_latitudes: + :param boundary_longitudes: + :param cell_area: + :param global_attributes: + :param regular_latlon: + :param rotated: + :param rotated_lats: + :param rotated_lons: + :param north_pole_lat: + :param north_pole_lon: + :param lcc: + :param lcc_x: + :param lcc_y: + :param lat_1_2: + :param lon_0: + :param lat_0: + :param mercator: + :param lat_ts: + :return: + """ from cf_units import Unit, encode_time - if not (RegularLatLon or LambertConformalConic or Rotated or Mercator): - RegularLatLon = True + if not (regular_latlon or lcc or rotated or mercator): + regular_latlon = True netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -121,7 +181,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, else: print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) sys.exit(1) - elif Rotated: + elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude @@ -137,7 +197,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic or Mercator: + elif lcc or mercator: var_dim = ('y', 'x',) netcdf.createDimension('y', len(lcc_y)) @@ -145,6 +205,10 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('x', len(lcc_x)) lon_dim = ('y', 'x', ) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if levels is not None: @@ -152,15 +216,11 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # Bounds if boundary_latitudes is not None: - # print boundary_latitudes.shape - # print len(boundary_latitudes[0, 0]) try: netcdf.createDimension('nv', len(boundary_latitudes[0, 0])) except TypeError: netcdf.createDimension('nv', boundary_latitudes.shape[1]) - # sys.exit() - # Time netcdf.createDimension('time', None) @@ -178,7 +238,8 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, u = Unit('hours') # print u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second)) # Unit('hour since 1970-01-01 00:00:00.0000000 UTC') - time.units = str(u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second))) + time.units = str(u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, + date.second))) time.standard_name = "time" time.calendar = "gregorian" time.long_name = "time" @@ -213,7 +274,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes - if Rotated: + if rotated: # Rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=True) rlat.long_name = "latitude in rotated pole grid" @@ -227,7 +288,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = rotated_lons - if LambertConformalConic or Mercator: + if lcc or mercator: x = netcdf.createVariable('x', 'd', ('x',), zlib=True) x.units = Unit("km").symbol x.long_name = "x coordinate of projection" @@ -266,41 +327,41 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, var.coordinates = "lat lon" if cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif rotated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' - elif Mercator: + elif mercator: var.grid_mapping = 'mercator' try: var[:] = variable['data'] - except: + except ValueError: print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: + elif rotated: # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = north_pole_lat mapping.grid_north_pole_longitude = north_pole_lon - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" mapping.standard_parallel = lat_1_2 mapping.longitude_of_central_meridian = lon_0 mapping.latitude_of_projection_origin = lat_0 - elif Mercator: - #Mercator + elif mercator: + # Mercator mapping = netcdf.createVariable('mercator', 'i') mapping.grid_mapping_name = "mercator" mapping.longitude_of_projection_origin = lon_0 @@ -322,23 +383,49 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, - levels=None, date=None, hours=None, - boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, - RegularLatLon=False, - Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, - LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): + levels=None, date=None, hours=None, + boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, + regular_latlon=False, + rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, + lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): + # TODO Documentation + """ + + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param boundary_latitudes: + :param boundary_longitudes: + :param cell_area: + :param global_attributes: + :param regular_latlon: + :param rotated: + :param rotated_lats: + :param rotated_lons: + :param north_pole_lat: + :param north_pole_lon: + :param lcc: + :param lcc_x: + :param lcc_y: + :param lat_1_2: + :param lon_0: + :param lat_0: + :return: + """ from cf_units import Unit, encode_time - import sys - from netCDF4 import Dataset import numpy as np - if not (RegularLatLon or LambertConformalConic or Rotated): - RegularLatLon = True + if not (regular_latlon or lcc or rotated): + regular_latlon = True netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -362,7 +449,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, else: print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) sys.exit(1) - elif Rotated: + elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude @@ -379,7 +466,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic: + elif lcc: var_dim = ('y', 'x',) netcdf.createDimension('y', len(lcc_y)) @@ -387,6 +474,10 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('x', len(lcc_x)) lon_dim = ('y', 'x', ) + else: + lat_dim = None + lon_dim = None + var_dim = None # Levels if levels is not None: @@ -416,7 +507,8 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, u = Unit('hours') # print u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second)) # Unit('hour since 1970-01-01 00:00:00.0000000 UTC') - time.units = str(u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, date.second))) + time.units = str(u.offset_by_time(encode_time(date.year, date.month, date.day, date.hour, date.minute, + date.second))) time.standard_name = "time" time.calendar = "gregorian" time.long_name = "time" @@ -450,7 +542,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes - if Rotated: + if rotated: # Rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=True) rlat.long_name = "latitude in rotated pole grid" @@ -464,7 +556,7 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = rotated_lons - if LambertConformalConic: + if lcc: x = netcdf.createVariable('x', 'd', ('x',), zlib=True) x.units = Unit("km").symbol x.long_name = "x coordinate of projection" @@ -504,35 +596,35 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, var.coordinates = "lat lon" if cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif rotated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' # print 'HOURSSSSSSSSSSSSSSSSSSSSS:', hours # if variable['data'] is not 0: # print var[:].shape, variable['data'].shape, variable['data'].max() - shape = None + shape = tuple() exec ("shape = (len(hours), {0}.size, {1}.size, {2}.size)".format(var_dim[0], var_dim[1], var_dim[2])) # exit() print shape var[:] = np.zeros(shape) # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: + elif rotated: # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = north_pole_lat mapping.grid_north_pole_longitude = north_pole_lon - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" @@ -555,17 +647,30 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, def tuple_to_index(tuple_list, bidimensional=False): + # TODO Documentation + """ + + :param tuple_list: + :param bidimensional: + :return: + """ from operator import mul new_list = [] - for tuple in tuple_list: + for my_tuple in tuple_list: if bidimensional: - new_list.append(tuple[-1] * tuple[-2]) + new_list.append(my_tuple[-1] * my_tuple[-2]) else: - new_list.append(reduce(mul, tuple)) + new_list.append(reduce(mul, my_tuple)) return new_list def calculate_displacements(counts): + # TODO Documentation + """ + + :param counts: + :return: + """ new_list = [0] accum = 0 for counter in counts[:-1]: @@ -576,7 +681,3 @@ def calculate_displacements(counts): if __name__ == '__main__': pass - - - - diff --git a/hermesv3_gr/tools/sample_files.py b/hermesv3_gr/tools/sample_files.py new file mode 100644 index 0000000000000000000000000000000000000000..07c642eca5a99cda1739bb9729f896be23ba3981 --- /dev/null +++ b/hermesv3_gr/tools/sample_files.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +# Copyright 2018 Earth Sciences Department, BSC-CNS +# +# This file is part of HERMESv3_GR. +# +# HERMESv3_GR is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# HERMESv3_GR is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with HERMESv3_GR. If not, see . + + +import sys +import os + + +def make_conf_file_list(): + main_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir) + + file_list = [ + {'conf': [ + os.path.join(main_dir, 'conf', 'hermes.conf'), + os.path.join(main_dir, 'conf', 'EI_configuration.csv'), + ]}, + ] + + return file_list + + +def make_profiles_file_list(): + main_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir) + + file_list = [ + {'data': [ + os.path.join(main_dir, 'data', 'global_attributes.csv'), + {'profiles': [{ + 'speciation': [ + os.path.join(main_dir, 'data', 'profiles', 'speciation', 'MolecularWeights.csv'), + os.path.join(main_dir, 'data', 'profiles', 'speciation', 'Speciation_profile_cb05_aero5_CMAQ.csv'), + os.path.join(main_dir, 'data', 'profiles', 'speciation', + 'Speciation_profile_cb05_aero5_MONARCH_aerosols.csv'), + os.path.join(main_dir, 'data', 'profiles', 'speciation', + 'Speciation_profile_cb05_aero5_MONARCH_fullchem.csv'), + os.path.join(main_dir, 'data', 'profiles', 'speciation', + 'Speciation_profile_radm2_madesorgam_WRF_CHEM.csv'), + ]}, + {'temporal': [ + os.path.join(main_dir, 'data', 'profiles', 'temporal', 'TemporalProfile_Daily.csv'), + os.path.join(main_dir, 'data', 'profiles', 'temporal', 'TemporalProfile_Hourly.csv'), + os.path.join(main_dir, 'data', 'profiles', 'temporal', 'TemporalProfile_Monthly.csv'), + os.path.join(main_dir, 'data', 'profiles', 'temporal', 'tz_world_country_iso3166.csv'), + ]}, + {'vertical': [ + os.path.join(main_dir, 'data', 'profiles', 'vertical', 'CMAQ_15layers_vertical_description.csv'), + os.path.join(main_dir, 'data', 'profiles', 'vertical', + 'MONARCH_Global_48layers_vertical_description.csv'), + os.path.join(main_dir, 'data', 'profiles', 'vertical', + 'MONARCH_regional_48layers_vertical_description.csv'), + os.path.join(main_dir, 'data', 'profiles', 'vertical', 'Vertical_profile.csv'), + ]}, + ]}, + ]}, + ] + + return file_list + + +def make_preproc_file_list(): + main_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir) + + file_list = [ + os.path.join(main_dir, 'preproc', 'ceds_preproc.py'), + os.path.join(main_dir, 'preproc', 'eclipsev5a_preproc.py'), + os.path.join(main_dir, 'preproc', 'edgarv432_ap_preproc.py'), + os.path.join(main_dir, 'preproc', 'edgarv432_voc_preproc.py'), + os.path.join(main_dir, 'preproc', 'emep_preproc.py'), + os.path.join(main_dir, 'preproc', 'gfas12_preproc.py'), + os.path.join(main_dir, 'preproc', 'htapv2_preproc.py'), + os.path.join(main_dir, 'preproc', 'tno_mac_iii_preproc.py'), + os.path.join(main_dir, 'preproc', 'tno_mac_iii_preproc_voc_ratios.py'), + os.path.join(main_dir, 'preproc', 'wiedinmyer_preproc.py'), + ] + + return file_list + + +def query_yes_no(question, default="yes"): + valid = {"yes": True, "y": True, "1": True, 1: True, + "no": False, "n": False, "0": False, 0: False} + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while True: + sys.stdout.write(question + prompt) + choice = raw_input().lower() + if default is not None and choice == '': + return valid[default] + elif choice in valid: + return valid[choice] + else: + sys.stdout.write("Please respond with 'yes' or 'no' (or 'y' or 'n').\n") + + +def check_args(args, exe_str): + if len(args) == 0: + print("Missing destination path after '{0}'. e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str)) + sys.exit(1) + elif len(args) > 1: + print("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str)) + sys.exit(1) + else: + dir_path = args[0] + + if not os.path.exists(dir_path): + if query_yes_no("'{0}' does not exist. Do you want to create it? ".format(dir_path)): + os.makedirs(dir_path) + else: + sys.exit(0) + + return dir_path + + +def copy_files(file_list, directory): + from shutil import copy2 + + if not os.path.exists(directory): + os.makedirs(directory) + + for element in file_list: + if dict == type(element): + copy_files(element.values()[0], os.path.join(directory, element.keys()[0])) + else: + copy2(element, directory) + return True + + +def copy_config_files(): + argv = sys.argv[1:] + + parent_dir = check_args(argv, 'hermesv3_gr_copy_config_files') + + copy_files(make_conf_file_list(), parent_dir) + copy_files(make_profiles_file_list(), parent_dir) + + +def copy_preproc_files(): + argv = sys.argv[1:] + + parent_dir = check_args(argv, 'hermesv3_gr_copy_preproc_files') + + copy_files(make_preproc_file_list(), parent_dir) + + +if __name__ == '__main__': + copy_config_files() diff --git a/preproc/ceds_preproc.py b/preproc/ceds_preproc.py old mode 100644 new mode 100755 index 068c26155f6d89cb5fae61fcd969449e0afc32be..c458dff25fcb67e2f3e98ae95963ae86d3932ca5 --- a/preproc/ceds_preproc.py +++ b/preproc/ceds_preproc.py @@ -23,20 +23,21 @@ import sys # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/jgcri/ceds/original_files' -output_path = '/esarchive/recon/jgcri/ceds' -list_pollutants = ['BC', 'CO', 'NH3', 'NMVOC', 'NOx', 'OC', 'SO2'] -voc_pollutants = ['VOC01', 'VOC02', 'VOC03', 'VOC04', 'VOC05', 'VOC06', 'VOC07', 'VOC08', 'VOC09', 'VOC12', 'VOC13', +INPUT_PATH = '/esarchive/recon/jgcri/ceds/original_files' +OUTPUT_PATH = '/esarchive/recon/jgcri/ceds' +LIST_POLLUTANTS = ['BC', 'CO', 'NH3', 'NMVOC', 'NOx', 'OC', 'SO2'] +VOC_POLLUTANTS = ['VOC01', 'VOC02', 'VOC03', 'VOC04', 'VOC05', 'VOC06', 'VOC07', 'VOC08', 'VOC09', 'VOC12', 'VOC13', 'VOC14', 'VOC15', 'VOC16', 'VOC17', 'VOC18', 'VOC19', 'VOC20', 'VOC21', 'VOC22', 'VOC23', 'VOC24', 'VOC25'] -list_sectors = ['agriculture', 'energy', 'industry', 'transport', 'residential', 'solvents', 'waste', 'ships'] -# list_years = from 1950 to 2014 -list_years = [2010] -input_name = '-em-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim_gr_01-12.nc' -voc_input_name = '-em-speciated-VOC_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim-supplemental-data_gr_01-12.nc' -do_air = True -air_input_name = '-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26_gr_01-12.nc' +LIST_SECTORS = ['agriculture', 'energy', 'industry', 'transport', 'residential', 'solvents', 'waste', 'ships'] +# LIST_YEARS = from 1950 to 2014 +LIST_YEARS = [2010] +INPUT_NAME = '-em-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sectorDim_gr_01-12.nc' +VOC_INPUT_NAME = '-em-speciated-VOC_input4MIPs_emissions_CMIP_CEDS-v2016-07-26-sector' + \ + 'Dim-supplemental-data_gr_01-12.nc' +DO_AIR = True +AIR_INPUT_NAME = '-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-v2016-07-26_gr_01-12.nc' # ============================================================== @@ -120,11 +121,11 @@ def get_input_name(pollutant, year, air=False): :rtype: str """ if air: - file_name = air_input_name.replace('', pollutant) - elif pollutant in list_pollutants: - file_name = input_name.replace('', pollutant) - elif pollutant in voc_pollutants: - file_name = voc_input_name.replace('', '{0}-{1}'.format(pollutant, voc_to_vocname(pollutant))) + file_name = AIR_INPUT_NAME.replace('', pollutant) + elif pollutant in LIST_POLLUTANTS: + file_name = INPUT_NAME.replace('', pollutant) + elif pollutant in VOC_POLLUTANTS: + file_name = VOC_INPUT_NAME.replace('', '{0}-{1}'.format(pollutant, voc_to_vocname(pollutant))) else: raise ValueError('Pollutant {0} not in pollutant list or voc list'.format(pollutant)) @@ -139,7 +140,7 @@ def get_input_name(pollutant, year, air=False): else: file_name = file_name.replace('', str(2000)).replace('', str(2014)) - return os.path.join(input_path, file_name) + return os.path.join(INPUT_PATH, file_name) def get_full_year_data(file_name, pollutant, sector, year, air=False): @@ -179,9 +180,9 @@ def get_full_year_data(file_name, pollutant, sector, year, air=False): i_time = np.where(time_array == datetime(year=year, month=1, day=1))[0][0] if air: data = nc.variables['AIR'][i_time:i_time + 12, :, :, :] - elif pollutant in list_pollutants: + elif pollutant in LIST_POLLUTANTS: data = nc.variables['{0}_em_anthro'.format(pollutant)][i_time:i_time+12, sector_to_index(sector), :, :] - elif pollutant in voc_pollutants: + elif pollutant in VOC_POLLUTANTS: data = nc.variables['{0}-{1}_em_speciated_VOC'.format( pollutant, voc_to_vocname(pollutant).replace('-', '_'))][i_time:i_time+12, sector_to_index(sector), :, :] else: @@ -222,14 +223,14 @@ def do_transformation(year): """ from datetime import datetime from hermesv3_gr.tools.netcdf_tools import extract_vars, get_grid_area, write_netcdf - for pollutant in list_pollutants + voc_pollutants: + for pollutant in LIST_POLLUTANTS + VOC_POLLUTANTS: file_name = get_input_name(pollutant, year) if os.path.exists(file_name): c_lats, c_lons, b_lats, b_lons = extract_vars(file_name, ['lat', 'lon', 'lat_bnds', 'lon_bnds']) cell_area = get_grid_area(file_name) global_attributes = get_global_attributes(file_name) - for sector in list_sectors: + for sector in LIST_SECTORS: data = get_full_year_data(file_name, pollutant, sector, year) if pollutant == 'NOx': @@ -237,7 +238,7 @@ def do_transformation(year): else: pollutant_name = pollutant.lower() - file_path = os.path.join(output_path, 'monthly_mean', '{0}_{1}'.format(pollutant_name, sector)) + file_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(pollutant_name, sector)) if not os.path.exists(file_path): os.makedirs(file_path) @@ -267,7 +268,7 @@ def do_air_transformation(year): from datetime import datetime from hermesv3_gr.tools.netcdf_tools import extract_vars, get_grid_area, write_netcdf - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: file_name = get_input_name(pollutant, year, air=True) if os.path.exists(file_name): c_lats, c_lons, b_lats, b_lons = extract_vars(file_name, ['lat', 'lon', 'lat_bnds', 'lon_bnds']) @@ -283,7 +284,7 @@ def do_air_transformation(year): pollutant_name = pollutant.lower() for sector in ['air_lto', 'air_cds', 'air_crs']: - file_path = os.path.join(output_path, 'monthly_mean', '{0}_{1}'.format(pollutant_name, sector)) + file_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(pollutant_name, sector)) if not os.path.exists(file_path): os.makedirs(file_path) @@ -314,7 +315,7 @@ def do_air_transformation(year): if __name__ == '__main__': - for y in list_years: + for y in LIST_YEARS: # do_transformation(y) - if do_air: + if DO_AIR: do_air_transformation(y) diff --git a/preproc/eclipsev5a_preproc.py b/preproc/eclipsev5a_preproc.py old mode 100644 new mode 100755 index 6a359b5bb24d95097c1ce13187b04a1ac6325d34..df523d20aeb7703a2d237a766dd2f01046e545c7 --- a/preproc/eclipsev5a_preproc.py +++ b/preproc/eclipsev5a_preproc.py @@ -26,25 +26,25 @@ from cf_units import Unit # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/iiasa/eclipsev5a/original_files' -output_path = '/esarchive/recon/iiasa/eclipsev5a/original_files/test' -input_name = 'ECLIPSE_base_CLE_V5a_.nc' -input_name_flaring = 'ECLIPSE_V5a_baseline_CLE_flaring.nc' -input_name_ship = "ship_CLE_emis_.nc" -monthly_pattern_file = 'ECLIPSEv5_monthly_patterns.nc' -list_years = [1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030, 2040, 2050] -list_pollutants = ['BC', 'CH4', 'CO', 'NH3', 'NOx', 'OC', 'OM', 'PM10', 'PM25', 'SO2', 'VOC'] +INPUT_PATH = '/esarchive/recon/iiasa/eclipsev5a/original_files' +OUTPUT_PATH = '/esarchive/recon/iiasa/eclipsev5a/original_files/test' +INPUT_NAME = 'ECLIPSE_base_CLE_V5a_.nc' +INPUT_NAME_FLARING = 'ECLIPSE_V5a_baseline_CLE_flaring.nc' +INPUT_NAME_SHIPS = "ship_CLE_emis_.nc" +MONTHLY_PATTERN_FILE = 'ECLIPSEv5_monthly_patterns.nc' +LIST_YEARS = [1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030, 2040, 2050] +LIST_POLLUTANTS = ['BC', 'CH4', 'CO', 'NH3', 'NOx', 'OC', 'OM', 'PM10', 'PM25', 'SO2', 'VOC'] # ============================================================== -month_factor = 1000000. / (30. * 24. * 3600.) # To pass from kt/month to Kg/s -year_factor = 1000000. / (365. * 24. * 3600.) # To pass from kt/year to Kg/s -var_units = 'kg.m-2.s-1' +MONTH_FACTOR = 1000000. / (30. * 24. * 3600.) # To pass from kt/month to Kg/s +YEAR_FACTOR = 1000000. / (365. * 24. * 3600.) # To pass from kt/year to Kg/s +VAR_UNITS = 'kg.m-2.s-1' def get_grid_area(filename): """ - Calculates the area for each cell of the grid using CDO + Calculate the area for each cell of the grid using CDO :param filename: Path to the file to calculate the cell area :type filename: str @@ -53,11 +53,10 @@ def get_grid_area(filename): :rtype: numpy.array """ from cdo import Cdo - from netCDF4 import Dataset cdo = Cdo() - s = cdo.gridarea(input=filename) - nc_aux = Dataset(s, mode='r') + src = cdo.gridarea(input=filename) + nc_aux = Dataset(src, mode='r') grid_area = nc_aux.variables['cell_area'][:] nc_aux.close() @@ -66,17 +65,17 @@ def get_grid_area(filename): def create_bounds(coordinates, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coordinates: Coordinates in degrees (latitude or longitude) - :type coordinates: numpy.ndarray + :type coordinates: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ interval = coordinates[1] - coordinates[0] @@ -164,7 +163,7 @@ def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cel nc_output.setncattr('source', 'IIASA', ) nc_output.setncattr('history', 'Re-writing of the ECLIPSEv5a input to follow the CF-1.6 conventions;\n' + '2017-11-28: Creating;\n') - nc_output.setncattr('http://www.iiasa.ac.at/web/home/research/researchPrograms/air/ECLIPSEv5a.html') + nc_output.setncattr('web', 'http://www.iiasa.ac.at/web/home/research/researchPrograms/air/ECLIPSEv5a.html') nc_output.setncattr('comment', 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)', ) @@ -207,7 +206,7 @@ def extract_month_profile_by_sector(sector, month, pollutant=None): else: profile_name = sector_dict[sector] - nc_profiles = Dataset(os.path.join(input_path, monthly_pattern_file), mode='r') + nc_profiles = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE), mode='r') profile = nc_profiles.variables[profile_name][month, :, :] @@ -219,7 +218,7 @@ def extract_month_profile_by_sector(sector, month, pollutant=None): def get_output_name(pollutant, sector, year, month): # TODO Docuemtnation - output_path_aux = os.path.join(output_path, 'monthly_mean', '{0}_{1}'.format(pollutant, sector), ) + output_path_aux = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(pollutant, sector), ) if not(os.path.exists(output_path_aux)): os.makedirs(output_path_aux) @@ -229,7 +228,7 @@ def get_output_name(pollutant, sector, year, month): def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area): # TODO Docuemtnation - for i in xrange(len(list_years)): + for i in xrange(len(LIST_YEARS)): for month in xrange(12): # print i, list_years[i], month + 1 @@ -239,11 +238,11 @@ def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area) pollutant_name = 'nmvoc' else: pollutant_name = pollutant.lower() - output_name = get_output_name(pollutant_name.lower(), sector.lower(), list_years[i], month + 1) + output_name = get_output_name(pollutant_name.lower(), sector.lower(), LIST_YEARS[i], month + 1) profile = extract_month_profile_by_sector(sector, month, pollutant) data_aux = data[i, :, :] * profile # print factor - data_aux = (data_aux * month_factor) / cell_area + data_aux = (data_aux * MONTH_FACTOR) / cell_area # #data_aux = data_aux / cell_area # print 'original: ', data[i, 192, 404] # print 'factor: ', profile[192, 404] @@ -253,16 +252,16 @@ def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area) 'name': pollutant_name, 'long_name': pollutant_name, 'data': data_aux, - 'units': Unit(var_units), + 'units': Unit(VAR_UNITS), }] write_netcdf(output_name, data_list, c_lats, c_lons, cell_area, - datetime(year=list_years[i], month=month + 1, day=1)) + datetime(year=LIST_YEARS[i], month=month + 1, day=1)) def do_transformation(): # TODO Documentation - for pollutant in list_pollutants: - file_name = os.path.join(input_path, input_name.replace('', pollutant)) + for pollutant in LIST_POLLUTANTS: + file_name = os.path.join(INPUT_PATH, INPUT_NAME.replace('', pollutant)) print file_name nc = Dataset(file_name, mode='r') c_lats = nc.variables['lat'][:] @@ -278,7 +277,7 @@ def do_transformation(): def get_flaring_output_name(pollutant, sector, year): # TODO Docuemtnation - output_path_aux = os.path.join(output_path, 'yearly_mean', '{0}_{1}'.format(pollutant, sector), ) + output_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', '{0}_{1}'.format(pollutant, sector), ) if not(os.path.exists(output_path_aux)): os.makedirs(output_path_aux) @@ -309,34 +308,34 @@ def get_flaring_var_name(nc_var): def do_flaring_transformation(): # TODO Documentation - nc_in = Dataset(os.path.join(input_path, input_name_flaring), mode='r') + nc_in = Dataset(os.path.join(INPUT_PATH, INPUT_NAME_FLARING), mode='r') c_lats = nc_in.variables['lat'][:] c_lons = nc_in.variables['lon'][:] - cell_area = get_grid_area(os.path.join(input_path, input_name_flaring)) + cell_area = get_grid_area(os.path.join(INPUT_PATH, INPUT_NAME_FLARING)) for var in nc_in.variables: var_name = get_flaring_var_name(var) if var_name is not None: data = nc_in.variables[var][:] data = np.nan_to_num(data) - for i in xrange(len(list_years)): - output_name = get_flaring_output_name(var_name, 'flaring', list_years[i]) + for i in xrange(len(LIST_YEARS)): + output_name = get_flaring_output_name(var_name, 'flaring', LIST_YEARS[i]) data_aux = data[i, :, :] - data_aux = (data_aux * year_factor) / cell_area + data_aux = (data_aux * YEAR_FACTOR) / cell_area data_aux = data_aux.reshape((1,) + data_aux.shape) data_list = [{ 'name': var_name, 'long_name': var_name, 'data': data_aux, - 'units': Unit(var_units), + 'units': Unit(VAR_UNITS), }] write_netcdf(output_name, data_list, c_lats, c_lons, cell_area, - datetime(year=list_years[i], month=1, day=1)) + datetime(year=LIST_YEARS[i], month=1, day=1)) nc_in.close() def get_ship_output_name(pollutant, sector, year): # TODO Docuemntation - output_path_aux = os.path.join(output_path, 'yearly_mean', '{0}_{1}'.format(pollutant, sector), ) + output_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', '{0}_{1}'.format(pollutant, sector), ) if not(os.path.exists(output_path_aux)): os.makedirs(output_path_aux) @@ -366,8 +365,8 @@ def get_ship_var_name(nc_var): def do_ship_transformation(): # TODO Documentation - for year in list_years: - in_path = os.path.join(input_path, input_name_ship.replace('', str(year))) + for year in LIST_YEARS: + in_path = os.path.join(INPUT_PATH, INPUT_NAME_SHIPS.replace('', str(year))) nc_in = Dataset(in_path, mode='r') c_lats = nc_in.variables['lat'][:] c_lons = nc_in.variables['lon'][:] @@ -380,13 +379,13 @@ def do_ship_transformation(): data = nc_in.variables[var][0, :, :] data = np.nan_to_num(data) - data = (data * year_factor) / cell_area + data = (data * YEAR_FACTOR) / cell_area data = data.reshape((1,) + data.shape) data_list = [{ 'name': var_name, 'long_name': var_name, 'data': data, - 'units': Unit(var_units), + 'units': Unit(VAR_UNITS), }] write_netcdf(get_ship_output_name(var_name, 'ship', year), data_list, c_lats, c_lons, cell_area, diff --git a/preproc/edgarv432_ap_preproc.py b/preproc/edgarv432_ap_preproc.py index 34f77affbfd160a9612c428895cc49a2d5d88d00..3dbf1df13557e95d93b4df4e6e57b46cb9e6ea31 100755 --- a/preproc/edgarv432_ap_preproc.py +++ b/preproc/edgarv432_ap_preproc.py @@ -25,22 +25,22 @@ from warnings import warn as warning # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/jrc/edgarv432_ap/original_files/' -output_path = '/esarchive/recon/jrc/edgarv432_ap' -list_pollutants = ['BC', 'CO', 'NH3', 'NOx', 'OC', 'PM10', 'PM2.5_bio', 'PM2.5_fossil', 'SO2', 'NMVOC'] -# list_years = [1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, +INPUT_PATH = '/esarchive/recon/jrc/edgarv432_ap/original_files/' +OUTPUT_PATH = '/esarchive/recon/jrc/edgarv432_ap' +LIST_POLLUTANTS = ['BC', 'CO', 'NH3', 'NOx', 'OC', 'PM10', 'PM2.5_bio', 'PM2.5_fossil', 'SO2', 'NMVOC'] +# LIST_YEARS = [1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, # 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, # 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012] -list_years = [2012] +LIST_YEARS = [2012] # To do yearly emissions -process_yearly = True -yearly_input_name = 'yearly/v432___.0.1x0.1.nc' +PROCESS_YEARLY = True +YEARLY_INPUT_NAME = 'yearly/v432___.0.1x0.1.nc' # To process monthly emissions, 2010 directly from monthly_input_name and other years calculated using bla bla bla -process_monthly = True -monthly_input_name = 'monthly/v432__2010__.0.1x0.1.nc' -monthly_pattern_file = 'temporal_profiles/v432_FM_.0.1x0.1.nc' +PROCESS_MONTHLY = True +MONTHLY_INPUT_NAME = 'monthly/v432__2010__.0.1x0.1.nc' +MONTHLY_PATTERN_FILE = 'temporal_profiles/v432_FM_.0.1x0.1.nc' # ============================================================== """ @@ -87,7 +87,7 @@ def ipcc_to_sector_dict(): def create_bounds(coordinates, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coordinates: Coordinates in degrees (latitude or longitude) :type coordinates: numpy.array @@ -115,7 +115,7 @@ def create_bounds(coordinates, number_vertices=2): def get_grid_area(filename): """ - Calculates the area for each cell of the grid using CDO + Calculate the area for each cell of the grid using CDO :param filename: Path to the file to calculate the cell area :type filename: str @@ -225,11 +225,11 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr def do_yearly_transformation(year): # TODO Documentation - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): file_path = os.path.join( - input_path, - yearly_input_name.replace('', pollutant).replace('', str(year)).replace('', + INPUT_PATH, + YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', ipcc)) if os.path.exists(file_path): @@ -263,7 +263,7 @@ def do_yearly_transformation(year): 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'yearly_mean', pollutant.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', pollutant.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) write_netcdf(os.path.join(out_path_aux, '{0}_{1}.nc'.format(pollutant.lower(), year)), @@ -277,11 +277,11 @@ def do_yearly_transformation(year): def do_monthly_transformation(year): # TODO Documentation - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): file_path = os.path.join( - input_path, - yearly_input_name.replace('', pollutant).replace('', str(year)).replace('', + INPUT_PATH, + YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', ipcc)) if os.path.exists(file_path): @@ -315,11 +315,11 @@ def do_monthly_transformation(year): 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'monthly_mean', pollutant.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'monthly_mean', pollutant.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) - nc_month_factors = Dataset(os.path.join(input_path, monthly_pattern_file.replace('', sector))) + nc_month_factors = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE.replace('', sector))) month_factors = nc_month_factors.variables[sector][:] for month in xrange(1, 12 + 1, 1): data_aux = data * month_factors[month - 1, :, :] @@ -336,12 +336,12 @@ def do_monthly_transformation(year): def do_2010_monthly_transformation(): # TODO Documentation - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): for month in xrange(1, 12 + 1, 1): file_path = os.path.join( - input_path, - monthly_input_name.replace('', pollutant).replace('', + INPUT_PATH, + MONTHLY_INPUT_NAME.replace('', pollutant).replace('', str(month)).replace('', ipcc)) if os.path.exists(file_path): @@ -374,7 +374,7 @@ def do_2010_monthly_transformation(): 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'monthly_mean', pollutant.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'monthly_mean', pollutant.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) write_netcdf(os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant.lower(), 2010, @@ -389,12 +389,12 @@ def do_2010_monthly_transformation(): if __name__ == '__main__': - if process_yearly: - for y in list_years: + if PROCESS_YEARLY: + for y in LIST_YEARS: do_yearly_transformation(y) - if process_monthly: - for y in list_years: + if PROCESS_MONTHLY: + for y in LIST_YEARS: if y == 2010: do_2010_monthly_transformation() else: diff --git a/preproc/edgarv432_voc_preproc.py b/preproc/edgarv432_voc_preproc.py index a42d8aaa09f2e427cf1610cfaaf4b54bff1d5445..8f8e6ccbc57115dc007fb99c821045a818e231be 100755 --- a/preproc/edgarv432_voc_preproc.py +++ b/preproc/edgarv432_voc_preproc.py @@ -25,24 +25,24 @@ from warnings import warn as warning # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/jrc/edgarv432_voc/original_files/' -output_path = '/esarchive/recon/jrc/edgarv432_voc' -list_pollutants = ['voc1', 'voc2', 'voc3', 'voc4', 'voc5', 'voc6', 'voc7', 'voc8', 'voc9', 'voc10', 'voc11', 'voc12', +INPUT_PATH = '/esarchive/recon/jrc/edgarv432_voc/original_files/' +OUTPUT_PATH = '/esarchive/recon/jrc/edgarv432_voc' +LIST_POLLUTANTS = ['voc1', 'voc2', 'voc3', 'voc4', 'voc5', 'voc6', 'voc7', 'voc8', 'voc9', 'voc10', 'voc11', 'voc12', 'voc13', 'voc14', 'voc15', 'voc16', 'voc17', 'voc18', 'voc19', 'voc20', 'voc21', 'voc22', 'voc23', 'voc24', 'voc25'] # list_years = [1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, # 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, # 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012] -list_years = [2010] +LIST_YEARS = [2010] # To do yearly emissions -process_yearly = True -yearly_input_name = 'yearly/v432_VOC_spec___.0.1x0.1.nc' +PROCESS_YEARLY = True +YEARLY_INPUT_NAME = 'yearly/v432_VOC_spec___.0.1x0.1.nc' # To process monthly emissions, 2010 directly from monthly_input_name and other years calculated using bla bla bla -process_monthly = False -monthly_input_name = 'monthly/v432_VOC_spec__2010__.0.1x0.1.nc' -monthly_pattern_file = 'temporal_profiles/v432_FM_.0.1x0.1.nc' +PROCESS_MONTHLY = False +MONTHLY_INPUT_NAME = 'monthly/v432_VOC_spec__2010__.0.1x0.1.nc' +MONTHLY_PATTERN_FILE = 'temporal_profiles/v432_FM_.0.1x0.1.nc' # ============================================================== """ @@ -56,17 +56,17 @@ Carles Tena Medina (carles.tena@bsc.es) from Barcelona Supercomputing Center (BS def create_bounds(coordinates, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coordinates: Coordinates in degrees (latitude or longitude) - :type coordinates: numpy.ndarray + :type coordinates: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ interval = coordinates[1] - coordinates[0] @@ -84,7 +84,7 @@ def create_bounds(coordinates, number_vertices=2): def get_grid_area(filename): """ - Calculates the area for each cell of the grid using CDO + Calculate the area for each cell of the grid using CDO :param filename: Path to the file to calculate the cell area :type filename: str @@ -222,11 +222,11 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr def do_yearly_transformation(year): # TODO Documentation print year - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): file_path = os.path.join( - input_path, - yearly_input_name.replace('', pollutant).replace('', str(year)).replace('', + INPUT_PATH, + YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', ipcc)) if os.path.exists(file_path): @@ -253,7 +253,7 @@ def do_yearly_transformation(year): 'units': 'kg.m-2.s-1', 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'yearly_mean', pollutant_aux.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', pollutant_aux.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) # print os.path.join(out_path_aux, '{0}_{1}.nc'.format(pollutant_aux.lower(), year)) @@ -269,11 +269,11 @@ def do_yearly_transformation(year): def do_monthly_transformation(year): # TODO Documentation print year - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): file_path = os.path.join( - input_path, - yearly_input_name.replace('', pollutant).replace('', str(year)).replace('', + INPUT_PATH, + YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', ipcc)) if os.path.exists(file_path): @@ -302,11 +302,11 @@ def do_monthly_transformation(year): 'coordinates': 'lat lon', 'grid_mapping': 'crs'} - out_path_aux = os.path.join(output_path, 'monthly_mean', pollutant_aux.lower() + '_' + sector.lower()) + out_path_aux = os.path.join(OUTPUT_PATH, 'monthly_mean', pollutant_aux.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) - nc_month_factors = Dataset(os.path.join(input_path, monthly_pattern_file.replace('', sector))) + nc_month_factors = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE.replace('', sector))) month_factors = nc_month_factors.variables[sector][:] for month in xrange(1, 12 + 1, 1): data_aux = data * month_factors[month - 1, :, :] @@ -323,12 +323,12 @@ def do_monthly_transformation(year): def do_2010_monthly_transformation(): # TODO Documentation - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for ipcc in ipcc_to_sector_dict().keys(): for month in xrange(1, 12 + 1, 1): file_path = os.path.join( - input_path, - monthly_input_name.replace('', pollutant).replace('', + INPUT_PATH, + MONTHLY_INPUT_NAME.replace('', pollutant).replace('', str(month)).replace('', ipcc)) if os.path.exists(file_path): @@ -358,7 +358,7 @@ def do_2010_monthly_transformation(): 'grid_mapping': 'crs'} out_path_aux = os.path.join( - output_path, 'monthly_mean', pollutant_aux.lower() + '_' + sector.lower()) + OUTPUT_PATH, 'monthly_mean', pollutant_aux.lower() + '_' + sector.lower()) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) write_netcdf(os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format( @@ -373,12 +373,12 @@ def do_2010_monthly_transformation(): if __name__ == '__main__': - if process_yearly: - for y in list_years: + if PROCESS_YEARLY: + for y in LIST_YEARS: do_yearly_transformation(y) - if process_monthly: - for y in list_years: + if PROCESS_MONTHLY: + for y in LIST_YEARS: if y == 2010: do_2010_monthly_transformation() else: diff --git a/preproc/emep_preproc.py b/preproc/emep_preproc.py old mode 100644 new mode 100755 index e4f526a5c733e99393dd7519e65ff1cef973d4bd..6df1d379fe453828f07f18cab1bc72bb1dadaa9b --- a/preproc/emep_preproc.py +++ b/preproc/emep_preproc.py @@ -24,12 +24,12 @@ from datetime import datetime # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/ceip/emepv18/original_files' -output_path = '/esarchive/recon/ceip/emepv18/yearly_mean' -input_name = '__2018_GRID_.txt' +INPUT_PATH = '/esarchive/recon/ceip/emepv18/original_files' +OUTPUT_PATH = '/esarchive/recon/ceip/emepv18/yearly_mean' +INPUT_NAME = '__2018_GRID_.txt' # list_years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016] -list_years = [2015] -list_pollutants = ['NOx', 'NMVOC', 'SOx', 'NH3', 'PM2_5', 'PM10', 'CO'] +LIST_YEARS = [2015] +LIST_POLLUTANTS = ['NOx', 'NMVOC', 'SOx', 'NH3', 'PM2_5', 'PM10', 'CO'] # ============================================================== @@ -83,11 +83,11 @@ def do_transformation(year): unit_factor = 1000./(365.*24.*3600.) # From Mg/year to Kg/s - for pollutant in list_pollutants: + for pollutant in LIST_POLLUTANTS: for sector in get_sectors(): in_file = os.path.join( - input_path, - input_name.replace('', str(year)).replace('', sector).replace('', pollutant)) + INPUT_PATH, + INPUT_NAME.replace('', str(year)).replace('', sector).replace('', pollutant)) if os.path.exists(in_file): print in_file @@ -123,7 +123,7 @@ def do_transformation(year): element['data'] = element['data'].reshape((1,) + element['data'].shape) - complete_output_dir = os.path.join(output_path, '{0}_{1}'.format(element['name'], sector.lower())) + complete_output_dir = os.path.join(OUTPUT_PATH, '{0}_{1}'.format(element['name'], sector.lower())) if not os.path.exists(complete_output_dir): os.makedirs(complete_output_dir) complete_output_dir = os.path.join(complete_output_dir, '{0}_{1}.nc'.format(element['name'], year)) @@ -148,5 +148,5 @@ def do_transformation(year): if __name__ == '__main__': - for y in list_years: + for y in LIST_YEARS: do_transformation(y) diff --git a/preproc/gfas12_preproc.py b/preproc/gfas12_preproc.py index ef2de315973ae8f028c342b461bad784e8cd00fa..6dd4a62bf2a9393c402f97d0e2ff23d7cddc0137 100755 --- a/preproc/gfas12_preproc.py +++ b/preproc/gfas12_preproc.py @@ -26,30 +26,30 @@ import datetime from datetime import datetime, timedelta # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' -input_name = 'ga_.grb' -output_path = '/esarchive/recon/ecmwf/gfas' +INPUT_PATH = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' +INPUT_NAME = 'ga_.grb' +OUTPUT_PATH = '/esarchive/recon/ecmwf/gfas' -starting_date = datetime(year=2018, month=8, day=29) -ending_date = datetime(year=2018, month=8, day=29) +STARTING_DATE = datetime(year=2018, month=8, day=29) +ENDIND_DATE = datetime(year=2018, month=8, day=29) -parameters_file = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_Parameters.csv' +PARAMETERS_FILE = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_Parameters.csv' # ============================================================== def create_bounds(coords, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) - :type coords: numpy.ndarray + :type coords: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np @@ -69,7 +69,7 @@ def create_bounds(coords, number_vertices=2): def get_grid_area(filename): """ - Calculates the area for each cell of the grid using CDO + Calculate the area for each cell of the grid using CDO :param filename: Path to the file to calculate the cell area :type filename: str @@ -237,7 +237,7 @@ def do_transformation(input_file, date, output_dir, variables_list): def do_var_list(variables_file): """ - Creates the List of dictionaries + Create the List of dictionaries :param variables_file: CSV file with the information of each variable :type variables_file: str @@ -261,13 +261,13 @@ def do_var_list(variables_file): if __name__ == '__main__': - var_list = do_var_list(parameters_file) + var_list = do_var_list(PARAMETERS_FILE) - date_aux = starting_date - while date_aux <= ending_date: - f = os.path.join(input_path, input_name.replace('', date_aux.strftime('%Y%m%d'))) + date_aux = STARTING_DATE + while date_aux <= ENDIND_DATE: + f = os.path.join(INPUT_PATH, INPUT_NAME.replace('', date_aux.strftime('%Y%m%d'))) if os.path.isfile(f): - do_transformation(f, date_aux, output_path, var_list) + do_transformation(f, date_aux, OUTPUT_PATH, var_list) else: print 'ERROR: file {0} not found'.format(f) diff --git a/preproc/htapv2_preproc.py b/preproc/htapv2_preproc.py index d178c71401a60679950095cee24797ce7148dcf9..8766dde1cd05e094329f6fe1d6f8855c2c047c39 100755 --- a/preproc/htapv2_preproc.py +++ b/preproc/htapv2_preproc.py @@ -19,27 +19,28 @@ import os -import sys # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/jrc/htapv2/original_files' -output_path = '/esarchive/recon/jrc/htapv2' +INPUT_PATH = '/esarchive/recon/jrc/htapv2/original_files' +OUTPUT_PATH = '/esarchive/recon/jrc/htapv2' -input_name = 'edgar_HTAP__emi___.0.1x0.1.nc' -input_name_air = 'edgar_HTAP_emi___.0.1x0.1.nc' -input_name_ships = 'edgar_HTAP__emi_SHIPS_.0.1x0.1.nc' -#HTAP auxiliary NMVOC emission data for the industry sub-sectors (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) -input_name_nmvoc_industry = 'HTAPv2_NMVOC___.0.1x0.1.nc' +INPUT_NAME = 'edgar_HTAP__emi___.0.1x0.1.nc' +INPUT_NAME_AIR = 'edgar_HTAP_emi___.0.1x0.1.nc' +INPUT_NAME_SHIPS = 'edgar_HTAP__emi_SHIPS_.0.1x0.1.nc' +# HTAP auxiliary NMVOC emission data for the industry sub-sectors +# (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) +INPUT_NAME_NMVOC_INDUSTRY = 'HTAPv2_NMVOC___.0.1x0.1.nc' # list_years = [2008, 2010] -list_years = [2010] - -#RETRO ratios applied to HTAPv2 NMVOC emissions (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) -voc_ratio_path = '/esarchive/recon/jrc/htapv2/original_files/retro_nmvoc_ratio_2000_01x01' -voc_ratio_name = 'retro_nmvoc_ratio__2000_0.1deg.nc' -voc_ratio_air_name = 'VOC_split_AIR.csv' -voc_ratio_ships_name = 'VOC_split_SHIP.csv' +LIST_YEARS = [2010] + +# RETRO ratios applied to HTAPv2 NMVOC emissions +# (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) +VOC_RATIO_PATH = '/esarchive/recon/jrc/htapv2/original_files/retro_nmvoc_ratio_2000_01x01' +VOC_RATIO_NAME = 'retro_nmvoc_ratio__2000_0.1deg.nc' +VOC_RATIO_AIR_NAME = 'VOC_split_AIR.csv' +VOC_RATIO_SHIPS_NAME = 'VOC_split_SHIP.csv' # ============================================================== @@ -64,11 +65,10 @@ def do_transformation_annual(filename, out_path, pollutant, sector, year): :return: """ - import os from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds print filename - c_lats, c_lons = extract_vars(filename, ['lat', 'lon']) + [c_lats, c_lons] = extract_vars(filename, ['lat', 'lon']) if pollutant == 'pm25': [data] = extract_vars(filename, ['emi_pm2.5'], @@ -90,7 +90,8 @@ def do_transformation_annual(filename, out_path, pollutant, sector, year): '2017-04-04: Added global attributes;\n' + '2017-04-04: Re-naming pollutant;\n' + '2017-04-04: Added cell_area variable;\n', - 'references': 'EC, JRC / US EPA, HTAP_V2. http://edgar.jrc.ec.europa.eu/htap/EDGAR-HTAP_v1_final_jan2012.pdf\n ' + + 'references': 'EC, JRC / US EPA, HTAP_V2. ' + + 'http://edgar.jrc.ec.europa.eu/htap/EDGAR-HTAP_v1_final_jan2012.pdf\n ' + 'http://edgar.jrc.ec.europa.eu/htap_v2/', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)', @@ -129,13 +130,12 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): :return: """ - import os from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds for month in xrange(1, 13): print filename_list[month - 1] - c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) + [c_lats, c_lons] = extract_vars(filename_list[month - 1], ['lat', 'lon']) if pollutant == 'pm25': [data] = extract_vars(filename_list[month - 1], ['emi_pm2.5'], @@ -157,7 +157,9 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): '2017-04-04: Added global attributes;\n' + '2017-04-04: Re-naming pollutant;\n' + '2017-04-04: Added cell_area variable;\n', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global ' + + 'emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, ' + + 'Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)', @@ -176,45 +178,57 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): def do_ratio_list(sector=None): # TODO Documentation + """ + + :param sector: + :return: + """ if sector == 'SHIPS': - return {'all': os.path.join(voc_ratio_path, voc_ratio_ships_name)} + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_SHIPS_NAME)} elif sector == 'AIR_CDS': - return {'all': os.path.join(voc_ratio_path, voc_ratio_air_name)} + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} elif sector == 'AIR_CRS': - return {'all': os.path.join(voc_ratio_path, voc_ratio_air_name)} + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} elif sector == 'AIR_LTO': - return {'all': os.path.join(voc_ratio_path, voc_ratio_air_name)} - else: - return { - 'voc01': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '01')), - 'voc02': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '02')), - 'voc03': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '03')), - 'voc04': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '04')), - 'voc05': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '05')), - 'voc06': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '06')), - 'voc07': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '07')), - 'voc08': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '08')), - 'voc09': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '09')), - 'voc12': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '12')), - 'voc13': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '13')), - 'voc14': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '14')), - 'voc15': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '15')), - 'voc16': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '16')), - 'voc17': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '17')), - 'voc18': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '18')), - 'voc19': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '19')), - 'voc20': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '20')), - 'voc21': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '21')), - 'voc22': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '22')), - 'voc23': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '23')), - 'voc24': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '24')), - 'voc25': os.path.join(voc_ratio_path, voc_ratio_name.replace('', '25')), - } + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} + return { + 'voc01': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '01')), + 'voc02': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '02')), + 'voc03': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '03')), + 'voc04': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '04')), + 'voc05': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '05')), + 'voc06': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '06')), + 'voc07': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '07')), + 'voc08': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '08')), + 'voc09': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '09')), + 'voc12': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '12')), + 'voc13': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '13')), + 'voc14': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '14')), + 'voc15': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '15')), + 'voc16': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '16')), + 'voc17': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '17')), + 'voc18': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '18')), + 'voc19': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '19')), + 'voc20': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '20')), + 'voc21': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '21')), + 'voc22': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '22')), + 'voc23': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '23')), + 'voc24': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '24')), + 'voc25': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '25')), + } def do_nmvoc_month_transformation(filename_list, out_path, sector, year): # TODO Docuemtnation - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area + """ + + :param filename_list: + :param out_path: + :param sector: + :param year: + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf from hermesv3_gr.tools.coordinates_tools import create_bounds nmvoc_ratio_list = do_ratio_list() @@ -258,7 +272,7 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): [ratio] = extract_vars(ratio_file, [ratio_var]) data_aux = data.copy() - data_aux['data'] = data['data']*ratio['data'] + data_aux['data'] = data['data'] * ratio['data'] data_aux['data'] = data_aux['data'].reshape((1,) + data_aux['data'].shape) data_aux['name'] = voc data_aux['units'] = 'kg m-2 s-1' @@ -286,12 +300,21 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): print out_path_aux write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], boundary_latitudes=create_bounds(c_lats['data']), - boundary_longitudes=create_bounds(c_lons['data']),global_attributes=global_attributes,) + boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) return True def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year): - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area + # TODO Documentation + """ + + :param filename_list: + :param out_path: + :param sector: + :param year: + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf from hermesv3_gr.tools.coordinates_tools import create_bounds nmvoc_ratio_list = do_ratio_list() @@ -314,22 +337,23 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year } if voc in ['voc02', 'voc03', 'voc04', 'voc05', 'voc07', 'voc08', 'voc12', 'voc13']: [r_inc, r_exf] = extract_vars(ratio_file, ['inc', 'exf']) - data.update({'data': ind['data']*r_inc['data'] + exf['data']*r_exf['data']}) + data.update({'data': ind['data'] * r_inc['data'] + exf['data'] * r_exf['data']}) elif voc in ['voc01', 'voc23', 'voc25']: [r_inc, r_sol] = extract_vars(ratio_file, ['inc', 'sol']) - data.update({'data': ind['data']*r_inc['data'] + sol['data']*r_sol['data']}) + data.update({'data': ind['data'] * r_inc['data'] + sol['data'] * r_sol['data']}) elif voc in ['voc09', 'voc16', 'voc21', 'voc22', 'voc24']: [r_inc] = extract_vars(ratio_file, ['inc']) - data.update({'data': ind['data']*r_inc['data']}) + data.update({'data': ind['data'] * r_inc['data']}) # elif voc in []: # [r_exf, r_sol] = extract_vars(ratio_file, ['exf', 'sol']) # data.update({'data': exf['data']*r_exf['data'] + sol['data']*r_sol['data']}) elif voc in ['voc18', 'voc19', 'voc20']: [r_sol] = extract_vars(ratio_file, ['sol']) - data.update({'data': sol['data']*r_sol['data']}) + data.update({'data': sol['data'] * r_sol['data']}) else: [r_inc, r_exf, r_sol] = extract_vars(ratio_file, ['inc', 'exf', 'sol']) - data.update({'data': ind['data']*r_inc['data'] + exf['data']*r_exf['data'] + sol['data']*r_sol['data']}) + data.update({'data': ind['data'] * r_inc['data'] + exf['data'] * r_exf['data'] + + sol['data'] * r_sol['data']}) global_attributes = { 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, voc), @@ -338,7 +362,10 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year 'source': 'HTAPv2', 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + '2017-04-28: ...', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and ' + + 'global emission grid maps for 2008 and 2010 to study hemispheric transport of air ' + + 'pollution, Atmos. Chem. Phys., 15, 11411-11432, ' + + 'https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)', @@ -351,22 +378,27 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) print out_path_aux write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data], - boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), - global_attributes=global_attributes,) + boundary_latitudes=create_bounds(c_lats['data']), + boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) def do_nmvoc_year_transformation(filename, out_path, sector, year): - import os + # TODO Documentation + """ + + :param filename: + :param out_path: + :param sector: + :param year: + :return: + """ import pandas as pd - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf from hermesv3_gr.tools.coordinates_tools import create_bounds nmvoc_ratio_file = do_ratio_list(sector)['all'] nmvoc_ratio_list = pd.read_csv(nmvoc_ratio_file, sep=';') - - print 'hola->',filename - c_lats, c_lons = extract_vars(filename, ['lat', 'lon']) [data] = extract_vars(filename, ['emi_nmvoc']) @@ -375,10 +407,8 @@ def do_nmvoc_year_transformation(filename, out_path, sector, year): pollutant = voc_ratio['voc_group'] ratio = voc_ratio['factor'] - # print pollutant, ratio - data_aux = data.copy() - data_aux['data'] = data['data']*ratio + data_aux['data'] = data['data'] * ratio data_aux['data'] = data_aux['data'].reshape((1,) + data_aux['data'].shape) data_aux['name'] = pollutant data_aux['units'] = 'kg m-2 s-1' @@ -389,7 +419,9 @@ def do_nmvoc_year_transformation(filename, out_path, sector, year): 'source': 'HTAPv2', 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + '2017-04-28: ...', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global ' + + 'emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, ' + + 'Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + '(Barcelona Supercomputing Center)\n ' + @@ -409,21 +441,31 @@ def do_nmvoc_year_transformation(filename, out_path, sector, year): def get_pollutant_dict(): - pollutant_dict = { - # 'bc': 'BC', - # 'co': 'CO', - # 'nh3': 'NH3', - # 'nox_no2': 'NOx', - # 'oc': 'OC', - # 'pm10': 'PM10', - # 'pm25': 'PM2.5', - # 'so2': 'SO2', + # TODO Documentation + """ + + :return: + """ + p_dict = { + 'bc': 'BC', + 'co': 'CO', + 'nh3': 'NH3', + 'nox_no2': 'NOx', + 'oc': 'OC', + 'pm10': 'PM10', + 'pm25': 'PM2.5', + 'so2': 'SO2', 'nmvoc': 'NMVOC' } - return pollutant_dict + return p_dict def get_sector_dict(): + # TODO Documentation + """ + + :return: + """ common_dict = { 'month': ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT'], 'year': ['SHIPS', 'AIR_CDS', 'AIR_CRS', 'AIR_LTO'] @@ -444,88 +486,93 @@ def get_sector_dict(): def get_nmvoc_sector_dict(): + # TODO Documentation + """ + + :return: + """ nmvoc_sectors = {'month': ['ENERGY', 'INDUSTRY_3subsectors', 'RESIDENTIAL', 'TRANSPORT'], 'year': ['SHIPS', 'AIR_CDS', 'AIR_CRS', 'AIR_LTO']} return nmvoc_sectors def check_vocs(year): + # TODO Documentation + """ + + :param year: + :return: + """ from hermesv3_gr.tools.netcdf_tools import extract_vars - for month in xrange(1, 12 +1, 1): + for month in xrange(1, 12 + 1, 1): for snap in ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT']: - nmvoc_path = os.path.join(output_path, 'monthly_mean', 'nmvoc_{0}'.format(snap.lower()), 'nmvoc_{0}{1}.nc'.format(year, str(month).zfill(2))) - # print nmvoc_path + nmvoc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', 'nmvoc_{0}'.format(snap.lower()), + 'nmvoc_{0}{1}.nc'.format(year, str(month).zfill(2))) [new_voc] = extract_vars(nmvoc_path, ['nmvoc']) nmvoc_sum = new_voc['data'].sum() voc_sum = 0 - for voc in ['voc{0}'.format(str(x).zfill(2)) for x in xrange(1, 25 +1, 1)]: - voc_path = os.path.join(output_path, 'monthly_mean', '{0}_{1}'.format(voc, snap.lower()), '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) - # print voc_path, os.path.exists(voc_path) + for voc in ['voc{0}'.format(str(x).zfill(2)) for x in xrange(1, 25 + 1, 1)]: + voc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(voc, snap.lower()), + '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) if os.path.exists(voc_path): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format(snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum)/nmvoc_sum, month) - + print '{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum) / nmvoc_sum, month) if __name__ == '__main__': - - for y in list_years: - # check_vocs(y) - # sys.exit(1) + for y in LIST_YEARS: for pollutant_dict in get_pollutant_dict().iteritems(): for current_sector in get_sector_dict()[pollutant_dict[0]]['month']: - input_name_aux = input_name.replace('', current_sector) + input_name_aux = INPUT_NAME.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) input_name_aux = input_name_aux.replace('', pollutant_dict[1]) - file_list = [os.path.join(input_path, input_name_aux.replace('', str(aux_month))) + file_list = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) for aux_month in xrange(1, 13)] - do_transformation(file_list, os.path.join(output_path, 'monthly_mean'), pollutant_dict[0], current_sector, - y) + do_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), pollutant_dict[0], + current_sector, y) # annual inventories for current_sector in get_sector_dict()[pollutant_dict[0]]['year']: if current_sector[0:3] == 'AIR': - input_name_aux = input_name_air + input_name_aux = INPUT_NAME_AIR else: - input_name_aux = input_name_ships + input_name_aux = INPUT_NAME_SHIPS input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) input_name_aux = input_name_aux.replace('', pollutant_dict[1]) - input_name_aux = os.path.join(input_path, input_name_aux) + input_name_aux = os.path.join(INPUT_PATH, input_name_aux) - do_transformation_annual(input_name_aux, os.path.join(output_path, 'yearly_mean', ), pollutant_dict[0], + do_transformation_annual(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean', ), pollutant_dict[0], current_sector, y) for current_sector in get_nmvoc_sector_dict()['month']: if current_sector == 'INDUSTRY_3subsectors': - input_name_aux = input_name_nmvoc_industry + input_name_aux = INPUT_NAME_NMVOC_INDUSTRY else: - input_name_aux = input_name + input_name_aux = INPUT_NAME input_name_aux = input_name_aux.replace('', 'NMVOC') input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) - file_list = [os.path.join(input_path, input_name_aux.replace('', str(aux_month))) + file_list = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) for aux_month in xrange(1, 13)] if current_sector == 'INDUSTRY_3subsectors': - do_nmvoc_industry_month_transformation(file_list, os.path.join(output_path, 'monthly_mean'), current_sector, - y) + do_nmvoc_industry_month_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), + current_sector, y) else: - do_nmvoc_month_transformation(file_list, os.path.join(output_path, 'monthly_mean'), current_sector, - y) + do_nmvoc_month_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), current_sector, y) for current_sector in get_nmvoc_sector_dict()['year']: if current_sector[0:3] == 'AIR': - input_name_aux = input_name_air + input_name_aux = INPUT_NAME_AIR else: - input_name_aux = input_name_ships + input_name_aux = INPUT_NAME_SHIPS input_name_aux = input_name_aux.replace('', 'NMVOC') input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) - input_name_aux = os.path.join(input_path, input_name_aux) + input_name_aux = os.path.join(INPUT_PATH, input_name_aux) print input_name_aux - do_nmvoc_year_transformation(input_name_aux, os.path.join(output_path, 'yearly_mean'), current_sector, - y) - + do_nmvoc_year_transformation(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean'), current_sector, y) diff --git a/preproc/tno_mac_iii_preproc.py b/preproc/tno_mac_iii_preproc.py old mode 100644 new mode 100755 index b9841db123bce2cfa1a2af54776a5f0e7e798db0..73c62820ade5f9a656f9c617a9c74f5de0f275cc --- a/preproc/tno_mac_iii_preproc.py +++ b/preproc/tno_mac_iii_preproc.py @@ -22,20 +22,19 @@ import os # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/tno/tno_macc_iii/original_files/ascii' -output_path = '/esarchive/recon/tno/tno_macc_iii/yearly_mean' -input_name = 'TNO_MACC_III_emissions_v1_1_.txt' +INPUT_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/ascii' +OUTPUT_PATH = '/esarchive/recon/tno/tno_macc_iii/yearly_mean' +INPUT_NAME = 'TNO_MACC_III_emissions_v1_1_.txt' # list_years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011] -list_years = [2011] -voc_ratio_path = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' -vor_ratio_name = 'ratio_.nc' - +LIST_YEARS = [2011] +VOC_RATIO_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' +VOC_RATIO_NAME = 'ratio_.nc' # ============================================================== def get_pollutants(in_path): """ - Finds the pollutants on the ASCII emissions table. + Find the pollutants on the ASCII emissions table. :param in_path: Path to the ASCII file that contains the information of the TNO_MAC-III emissions. :type in_path: str @@ -50,33 +49,55 @@ def get_pollutants(in_path): def calculate_grid_definition(in_path): - # TODO Documentation + """ + Calculate the latitude and longitude coordinates of the cell. + + :param in_path: Path to the file that contains all the information. + :type in_path: str + + :return: Latitudes array, Longitudes array, Latitude interval, Longitude interval. + :rtype: numpy.array, numpy.array, float, float + """ import pandas as pd import numpy as np - df = pd.read_table(in_path, sep=';') - df = df[df.SourceType != 'P'] + dataframe = pd.read_table(in_path, sep=';') + dataframe = dataframe[dataframe.SourceType != 'P'] # Longitudes - lons = np.sort(np.unique(df.Lon)) + lons = np.sort(np.unique(dataframe.Lon)) lons_interval = lons[1:] - lons[:-1] print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - df.Lon.min(), df.Lon.max(), lons_interval.min(), len(lons)) + dataframe.Lon.min(), dataframe.Lon.max(), lons_interval.min(), len(lons)) # Latitudes - lats = np.sort(np.unique(df.Lat)) + lats = np.sort(np.unique(dataframe.Lat)) lats_interval = lats[1:] - lats[:-1] print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - df.Lat.min(), df.Lat.max(), lats_interval.min(), len(lats)) + dataframe.Lat.min(), dataframe.Lat.max(), lats_interval.min(), len(lats)) - lats = np.arange(-90 + lats_interval.min()/2, 90, lats_interval.min(), dtype=np.float64) - lons = np.arange(-180 + lons_interval.min()/2, 180, lons_interval.min(), dtype=np.float64) + lats = np.arange(-90 + lats_interval.min() / 2, 90, lats_interval.min(), dtype=np.float64) + lons = np.arange(-180 + lons_interval.min() / 2, 180, lons_interval.min(), dtype=np.float64) return lats, lons, lats_interval.min(), lons_interval.min() def create_pollutant_empty_list(in_path, len_c_lats, len_c_lons): - # TODO Documentation + """ + Crate an empty pollutant list. + + :param in_path: Path to the file that conains the information. + :type in_path: str + + :param len_c_lats: Number of elements on the latitude array + :type len_c_lats: int + + :param len_c_lons: Number of elements on the longitude array + :type len_c_lons: int + + :return: Pollutant list + :rtype: list + """ import numpy as np pollutant_list = [] @@ -98,16 +119,24 @@ def create_pollutant_empty_list(in_path, len_c_lats, len_c_lons): def do_transformation(year): - # TODO Docuemtnation + """ + Make al the process to transform the emissions of the current year. + + :param year: year to process. + :type year: int + + :return: True when everything finish well. + :rtype: Bool + """ from hermesv3_gr.tools.netcdf_tools import write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds from datetime import datetime import pandas as pd import numpy as np - in_file = os.path.join(input_path, input_name.replace('', str(year))) + in_file = os.path.join(INPUT_PATH, INPUT_NAME.replace('', str(year))) - unit_factor = 1000./(365.*24.*3600.) # To pass from Mg/year to Kg/s + unit_factor = 1000. / (365. * 24. * 3600.) # To pass from Mg/year to Kg/s # unit_factor = 1000000 # To pass from Mg/m2.year to Mg/Km2.year c_lats, c_lons, lat_interval, lon_interval = calculate_grid_definition(in_file) @@ -115,20 +144,20 @@ def do_transformation(year): b_lats = create_bounds(c_lats, number_vertices=2) b_lons = create_bounds(c_lons, number_vertices=2) - df = pd.read_table(in_file, sep=';') + dataframe = pd.read_table(in_file, sep=';') - df_np = df[df.SourceType != 'P'] - df_p = df[df.SourceType == 'P'] + df_np = dataframe[dataframe.SourceType != 'P'] + df_p = dataframe[dataframe.SourceType == 'P'] df_np.loc[:, 'row_lat'] = np.array((df_np.Lat - (-90 + lat_interval / 2)) / lat_interval, dtype=np.int32) df_np.loc[:, 'col_lon'] = np.array((df_np.Lon - (-180 + lon_interval / 2)) / lon_interval, dtype=np.int32) - df_p.loc[:, 'row_lat'] = abs(np.array([c_lats]*len(df_p.Lat)) - df_p.Lat.values[:, None]).argmin(axis=1) - df_p.loc[:, 'col_lon'] = abs(np.array([c_lons]*len(df_p.Lon)) - df_p.Lon.values[:, None]).argmin(axis=1) + df_p.loc[:, 'row_lat'] = abs(np.array([c_lats] * len(df_p.Lat)) - df_p.Lat.values[:, None]).argmin(axis=1) + df_p.loc[:, 'col_lon'] = abs(np.array([c_lons] * len(df_p.Lon)) - df_p.Lon.values[:, None]).argmin(axis=1) - df = pd.concat([df_np, df_p]) + dataframe = pd.concat([df_np, df_p]) - for name, group in df.groupby('SNAP'): + for name, group in dataframe.groupby('SNAP'): print 'snap', name pollutant_list = create_pollutant_empty_list(in_file, len(c_lats), len(c_lons)) @@ -145,7 +174,7 @@ def do_transformation(year): pollutant_list[i]['data'] = pollutant_list[i]['data'].reshape((1,) + pollutant_list[i]['data'].shape) # print pollutant_list[i]['data'].max() - aux_output_path = os.path.join(output_path, '{0}_snap{1}'.format(pollutant_list[i]['name'], name)) + aux_output_path = os.path.join(OUTPUT_PATH, '{0}_snap{1}'.format(pollutant_list[i]['name'], name)) if not os.path.exists(aux_output_path): os.makedirs(aux_output_path) aux_output_path = os.path.join(aux_output_path, '{0}_{1}.nc'.format(pollutant_list[i]['name'], year)) @@ -153,7 +182,7 @@ def do_transformation(year): boundary_latitudes=b_lats, boundary_longitudes=b_lons) cell_area = get_grid_area(aux_output_path) - pollutant_list[i]['data'] = pollutant_list[i]['data']*unit_factor/cell_area + pollutant_list[i]['data'] = pollutant_list[i]['data'] * unit_factor/cell_area write_netcdf(aux_output_path, c_lats, c_lons, [pollutant_list[i]], date=datetime(year, month=1, day=1), boundary_latitudes=b_lats, boundary_longitudes=b_lons, cell_area=cell_area, @@ -164,14 +193,27 @@ def do_transformation(year): 'for air quality modelling Atmospheric Chemistry and Physics 14 ' + '10963-10976 2014', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + - '(Barcelona Supercomputing Center)' - } - ) + '(Barcelona Supercomputing Center)'}) return True -def extract_vars(netcdf_path, variables_list, attributes_list=list()): - # TODO Documentation +def extract_vars(netcdf_path, variables_list, attributes_list=()): + """ + Get the data from the list of variabbles. + + :param netcdf_path: Path to the netCDF file + :type netcdf_path: str + + :param variables_list: List of the names of the variables to get. + :type variables_list: list + + :param attributes_list: List of the names of the variable attributes to get. + :type attributes_list: list + + :return: List of the variables from the netCDF as a dictionary with data as values and with the other keys their + attributes. + :rtype: list. + """ from netCDF4 import Dataset data_list = [] # print netcdf_path @@ -196,7 +238,18 @@ def extract_vars(netcdf_path, variables_list, attributes_list=list()): def get_voc_ratio(ratio_path, snap): - # TODO Documentation + """ + Get the ratio of the VOC for the current SNAP. + + :param ratio_path: Path to the file with the ratios. + :type ratio_path: str + + :param snap: SNAP to get the ratio. + :type snap: str + + :return: VOC Ratio + :rtype: dict + """ if snap == 'snap34': snap = 'snap3' try: @@ -207,27 +260,45 @@ def get_voc_ratio(ratio_path, snap): def get_voc_list(): - # TODO Documentation + """ + Get the VOC list. + + :return: VOC list + :rtype: list + """ return ['voc{0}'.format(str(x).zfill(2)) for x in [1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]] def get_sector_list(): - # TODO Documentation + """ + Get the sector list. + + :return: Sector list + :rtype: list + """ return ['snap{0}'.format(x) for x in [1, 2, 34, 5, 6, 71, 72, 73, 74, 8, 9]] def do_voc_transformation(year): - # TODO Docuemtnation + """ + Make al the process to transform the VOC emissions of the current year. + + :param year: year to process. + :type year: int + + :return: True when everything finish well. + :rtype: Bool + """ + from warnings import warn as warning from hermesv3_gr.tools.netcdf_tools import write_netcdf, extract_vars from hermesv3_gr.tools.coordinates_tools import create_bounds - from warnings import warn as warning for snap in get_sector_list(): - in_path = os.path.join(output_path, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) + in_path = os.path.join(OUTPUT_PATH, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) [nmvoc, c_lats, c_lons, cell_area] = extract_vars(in_path, ['nmvoc', 'lat', 'lon', 'cell_area']) for voc in get_voc_list(): - ratio_path = os.path.join(voc_ratio_path, vor_ratio_name.replace('', voc)) + ratio_path = os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', voc)) ratios_dict = get_voc_ratio(ratio_path, snap) if ratios_dict is not None: new_voc = { @@ -241,7 +312,7 @@ def do_voc_transformation(year): new_voc['data'] = nmvoc['data'] * mask - out_dir_aux = os.path.join(output_path, '{0}_{1}'.format(voc, snap)) + out_dir_aux = os.path.join(OUTPUT_PATH, '{0}_{1}'.format(voc, snap)) if not os.path.exists(out_dir_aux): os.makedirs(out_dir_aux) # print os.path.join(out_dir_aux, '{0}_{1}.nc'.format(voc, year)) @@ -255,8 +326,7 @@ def do_voc_transformation(year): 'for air quality modelling Atmospheric Chemistry and Physics 14 ' + '10963-10976 2014', 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + - '(Barcelona Supercomputing Center)' - }) + '(Barcelona Supercomputing Center)'}) else: warning("The pollutant {0} for the sector {1} does not exist.\n SNAP not found: {2}".format( voc, snap, ratio_path)) @@ -265,26 +335,34 @@ def do_voc_transformation(year): def check_vocs(year): - # TODO Documentation + """ + Check that the VOCs are calculated correctly. + + :param year: Year to evaluate + :type year: int + + :return: True when finish. + :rtype: bool + """ for snap in get_sector_list(): - nmvoc_path = os.path.join(output_path, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) + nmvoc_path = os.path.join(OUTPUT_PATH, 'nmvoc_{0}'.format(snap), 'nmvoc_{0}.nc'.format(year)) [new_voc] = extract_vars(nmvoc_path, ['nmvoc']) nmvoc_sum = new_voc['data'].sum() voc_sum = 0 for voc in get_voc_list(): - voc_path = os.path.join(output_path, '{0}_{1}'.format(voc, snap), '{0}_{1}.nc'.format(voc, year)) + voc_path = os.path.join(OUTPUT_PATH, '{0}_{1}'.format(voc, snap), '{0}_{1}.nc'.format(voc, year)) if os.path.exists(voc_path): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() print '{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum)/nmvoc_sum) + snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum) return True if __name__ == '__main__': - for y in list_years: + for y in LIST_YEARS: do_transformation(y) do_voc_transformation(y) # check_vocs(y) diff --git a/preproc/tno_mac_iii_preproc_voc_ratios.py b/preproc/tno_mac_iii_preproc_voc_ratios.py old mode 100644 new mode 100755 index f1800a3f0c0864b5fa76687541ec51056b02d6d8..9aea92690893c217c28bd069905ab604ff598147 --- a/preproc/tno_mac_iii_preproc_voc_ratios.py +++ b/preproc/tno_mac_iii_preproc_voc_ratios.py @@ -23,16 +23,22 @@ import os # ============== CONFIGURATION PARAMETERS ====================== -output_path = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' -world_info_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/temporal/tz_world_country_iso3166.csv' -tno_world_mask = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/TNO_MACC-III_WorldMask.nc' -csv_path = '/esarchive/recon/tno/tno_macc_iii/original_files/TNO_MACC_NMVOC profile_country_SNAP_12_05_2010.csv' - +OUTPUT_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/nmvoc' +WORLD_INFO_PATH = '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/temporal/tz_world_country_iso3166.csv' +TNO_WORLD_MASK = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/TNO_MACC-III_WorldMask.nc' +CSV_PATH = '/esarchive/recon/tno/tno_macc_iii/original_files/TNO_MACC_NMVOC profile_country_SNAP_12_05_2010.csv' # ============================================================== -def extract_vars(netcdf_path, variables_list, attributes_list=list()): +def extract_vars(netcdf_path, variables_list, attributes_list=()): # TODO Docuemtnation + """ + + :param netcdf_path: + :param variables_list: + :param attributes_list: + :return: + """ from netCDF4 import Dataset data_list = [] # print netcdf_path @@ -59,20 +65,47 @@ def extract_vars(netcdf_path, variables_list, attributes_list=list()): def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, levels=None, date=None, hours=None, boundary_latitudes=None, boundary_longitudes=None, cell_area=None, global_attributes=None, - RegularLatLon=False, - Rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, - LambertConformalConic=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): + regular_latlon=False, + rotated=False, rotated_lats=None, rotated_lons=None, north_pole_lat=None, north_pole_lon=None, + lcc=False, lcc_x=None, lcc_y=None, lat_1_2=None, lon_0=None, lat_0=None): # TODO Docuemtnation + """ + + :param netcdf_path: + :param center_latitudes: + :param center_longitudes: + :param data_list: + :param levels: + :param date: + :param hours: + :param boundary_latitudes: + :param boundary_longitudes: + :param cell_area: + :param global_attributes: + :param regular_latlon: + :param rotated: + :param rotated_lats: + :param rotated_lons: + :param north_pole_lat: + :param north_pole_lon: + :param lcc: + :param lcc_x: + :param lcc_y: + :param lat_1_2: + :param lon_0: + :param lat_0: + :return: + """ from cf_units import Unit, encode_time from netCDF4 import Dataset - if not (RegularLatLon or LambertConformalConic or Rotated): - RegularLatLon = True + if not (regular_latlon or lcc or rotated): + regular_latlon = True print netcdf_path netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== - if RegularLatLon: + if regular_latlon: var_dim = ('lat', 'lon',) # Latitude @@ -96,7 +129,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, else: print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) sys.exit(1) - elif Rotated: + elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude @@ -113,7 +146,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) - elif LambertConformalConic: + elif lcc: var_dim = ('y', 'x',) netcdf.createDimension('y', len(lcc_y)) @@ -190,7 +223,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # print lon_bnds[:].shape, boundary_longitudes.shape lon_bnds[:] = boundary_longitudes - if Rotated: + if rotated: # Rotated Latitude rlat = netcdf.createVariable('rlat', 'f', ('rlat',), zlib=True) rlat.long_name = "latitude in rotated pole grid" @@ -204,7 +237,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, rlon.units = Unit("degrees").symbol rlon.standard_name = "grid_longitude" rlon[:] = rotated_lons - if LambertConformalConic: + if lcc: x = netcdf.createVariable('x', 'd', ('x',), zlib=True) x.units = Unit("km").symbol x.long_name = "x coordinate of projection" @@ -243,11 +276,11 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, var.coordinates = "lat lon" if cell_area is not None: var.cell_measures = 'area: cell_area' - if RegularLatLon: + if regular_latlon: var.grid_mapping = 'crs' - elif Rotated: + elif rotated: var.grid_mapping = 'rotated_pole' - elif LambertConformalConic: + elif lcc: var.grid_mapping = 'Lambert_conformal' # if variable['data'] is not 0: # print var[:].shape, variable['data'].shape @@ -257,19 +290,19 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) # Grid mapping - if RegularLatLon: + if regular_latlon: # CRS mapping = netcdf.createVariable('crs', 'i') mapping.grid_mapping_name = "latitude_longitude" mapping.semi_major_axis = 6371000.0 mapping.inverse_flattening = 0 - elif Rotated: + elif rotated: # Rotated pole mapping = netcdf.createVariable('rotated_pole', 'c') mapping.grid_mapping_name = 'rotated_latitude_longitude' mapping.grid_north_pole_latitude = north_pole_lat mapping.grid_north_pole_longitude = north_pole_lon - elif LambertConformalConic: + elif lcc: # CRS mapping = netcdf.createVariable('Lambert_conformal', 'i') mapping.grid_mapping_name = "lambert_conformal_conic" @@ -295,20 +328,20 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, def get_grid_area(filename): """ - Calculates the area of each cell. + Calculate the area of each cell. :param filename: Full path to the NetCDF to calculate the cell areas. :type filename: str :return: Returns the area of each cell. - :rtype: numpy.ndarray + :rtype: numpy.array """ from cdo import Cdo from netCDF4 import Dataset cdo = Cdo() - s = cdo.gridarea(input=filename) - nc_aux = Dataset(s, mode='r') + src = cdo.gridarea(input=filename) + nc_aux = Dataset(src, mode='r') grid_area = nc_aux.variables['cell_area'][:] nc_aux.close() @@ -317,17 +350,17 @@ def get_grid_area(filename): def create_bounds(coords, number_vertices=2): """ - Calculates the vertices coordinates. + Calculate the vertices coordinates. :param coords: Coordinates in degrees (latitude or longitude) - :type coords: numpy.ndarray + :type coords: numpy.array :param number_vertices: Non mandatory parameter that informs the number of vertices that must have the boundaries. (by default 2) :type number_vertices: int :return: Array with as many elements as vertices for each value of coords. - :rtype: numpy.ndarray + :rtype: numpy.array """ import numpy as np @@ -347,14 +380,19 @@ def create_bounds(coords, number_vertices=2): def create_voc_ratio(voc): # TODO Docuemtnation + """ + + :param voc: + :return: + """ import numpy as np - country_values, lat, lon = extract_vars(tno_world_mask, ['timezone_id', 'lat', 'lon']) + [country_values, lat, lon] = extract_vars(TNO_WORLD_MASK, ['timezone_id', 'lat', 'lon']) country_values = country_values['data'].reshape((country_values['data'].shape[1], country_values['data'].shape[1])) - print output_path - if not os.path.exists(output_path): - os.makedirs(output_path) + print OUTPUT_PATH + if not os.path.exists(OUTPUT_PATH): + os.makedirs(OUTPUT_PATH) - complete_output_path = os.path.join(output_path, 'ratio_{0}.nc'.format(voc)) + complete_output_path = os.path.join(OUTPUT_PATH, 'ratio_{0}.nc'.format(voc)) if not os.path.exists(complete_output_path): print 'Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path) data_list = [] @@ -383,9 +421,15 @@ def create_voc_ratio(voc): def get_default_ratio(voc, snap): # TODO Documentation + """ + + :param voc: + :param snap: + :return: + """ import pandas as pd - df = pd.read_csv(csv_path, sep=';') + df = pd.read_csv(CSV_PATH, sep=';') df = df.loc[df['vcode'] == voc.replace('voc', 'v'), :] df = df.loc[df['snap'] == snap, :] @@ -395,10 +439,14 @@ def get_default_ratio(voc, snap): def get_iso_codes(): # TODO Documentation + """ + + :return: + """ import pandas as pd # df = pd.read_csv(self.world_info, sep=';', index_col=False, names=["country", "country_code"]) - df = pd.read_csv(world_info_path, sep=';') + df = pd.read_csv(WORLD_INFO_PATH, sep=';') del df['time_zone'], df['time_zone_code'] df = df.drop_duplicates().dropna() df = df.set_index('country_code_alpha') @@ -410,9 +458,13 @@ def get_iso_codes(): def get_voc_list(): # TODO Documentation + """ + + :return: + """ import pandas as pd - df = pd.read_csv(csv_path, sep=';') + df = pd.read_csv(CSV_PATH, sep=';') del df['ISO3'], df['snap'], df['output substance name'], df['fr'] df = df.drop_duplicates().dropna() voc_list = df.vcode.values @@ -423,9 +475,14 @@ def get_voc_list(): def get_sector_list(voc): # TODO Documentation + """ + + :param voc: + :return: + """ import pandas as pd voc = voc.replace('voc', 'v') - df = pd.read_csv(csv_path, sep=';') + df = pd.read_csv(CSV_PATH, sep=';') df = df[df.vcode == voc] del df['ISO3'], df['vcode'], df['output substance name'], df['fr'] df = df.drop_duplicates().dropna() @@ -434,6 +491,11 @@ def get_sector_list(voc): def get_sector_list_text(voc): # TODO Documentation + """ + + :param voc: + :return: + """ voc = voc.replace('voc', 'v') sector_list = get_sector_list(voc) new_list = [] @@ -444,9 +506,15 @@ def get_sector_list_text(voc): def get_country_code_and_factor(voc, snap): # TODO Documentation + """ + + :param voc: + :param snap: + :return: + """ import pandas as pd voc = voc.replace('voc', 'v') - df = pd.read_csv(csv_path, sep=';') + df = pd.read_csv(CSV_PATH, sep=';') df = df[df.vcode == voc] df = df[df.snap == snap] del df['snap'], df['vcode'], df['output substance name'] diff --git a/preproc/wiedinmyer_preproc.py b/preproc/wiedinmyer_preproc.py index 069ca06f68b2f241836e269fa727a2d05cf4dbae..5af455f941d1c40e82b44f1307b61a0acf265236 100755 --- a/preproc/wiedinmyer_preproc.py +++ b/preproc/wiedinmyer_preproc.py @@ -24,13 +24,13 @@ from netCDF4 import Dataset # ============== CONFIGURATION PARAMETERS ====================== -input_path = '/esarchive/recon/ucar/wiedinmyer/original_files/' -output_path = '/esarchive/recon/ucar/wiedinmyer/' -list_pollutants = ['co2', 'co', 'so2', 'nox_no', 'nh3', 'ch4', 'c2h2', 'c2h4', 'c3h6', 'ch3oh', 'ch2o', 'ch3cooh', +INPUT_PATH = '/esarchive/recon/ucar/wiedinmyer/original_files/' +OUTPUT_PATH = '/esarchive/recon/ucar/wiedinmyer/' +LIST_POLLUTANTS = ['co2', 'co', 'so2', 'nox_no', 'nh3', 'ch4', 'c2h2', 'c2h4', 'c3h6', 'ch3oh', 'ch2o', 'ch3cooh', 'hcn', 'c6h6', 'pcb', 'pah', 'pcdd', 'pbdd', 'nmoc', 'hcl', 'hg', 'pm25', 'pm10', 'oc', 'bc'] -input_name = 'ALL_Emiss_04282014.nc' -year = 2010 +INPUT_NAME = 'ALL_Emiss_04282014.nc' +YEAR = 2010 # ============================================================== @@ -89,7 +89,7 @@ def do_transformation(filename): factor = 1000000./(365.*24.*3600.) # To pass from Gg/m2.year to Kg/m2.s - for output_pollutant in list_pollutants: + for output_pollutant in LIST_POLLUTANTS: input_pollutant = out_pollutant_to_in_pollutant(output_pollutant) data = nc_in.variables[input_pollutant][:] @@ -103,11 +103,11 @@ def do_transformation(filename): 'grid_mapping': 'crs'} data = np.array(data) - out_path_aux = os.path.join(output_path, 'yearly_mean', output_pollutant) + out_path_aux = os.path.join(OUTPUT_PATH, 'yearly_mean', output_pollutant) if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) - write_netcdf(os.path.join(out_path_aux, '{0}_{1}.nc'.format(output_pollutant, year)), - data, data_attributes, lats, lons, grid_area, year, 01) + write_netcdf(os.path.join(out_path_aux, '{0}_{1}.nc'.format(output_pollutant, YEAR)), + data, data_attributes, lats, lons, grid_area, YEAR, 01) nc_in.close() @@ -229,6 +229,6 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr if __name__ == '__main__': starting_time = timeit.default_timer() - do_transformation(os.path.join(input_path, input_name)) + do_transformation(os.path.join(INPUT_PATH, INPUT_NAME)) print 'Time(s):', timeit.default_timer() - starting_time diff --git a/run_test.py b/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..6eb2926793f63a9c62eee79c06273d12f95fab82 --- /dev/null +++ b/run_test.py @@ -0,0 +1,24 @@ +# coding=utf-8 +"""Script to run the tests for EarthDiagnostics and generate the code coverage report""" + +import os +import sys +import pytest + + +work_path = os.path.abspath(os.path.join(os.path.dirname(__file__))) +os.chdir(work_path) +print(work_path) + + +version = sys.version_info[0] +report_dir = 'tests/report/python{}'.format(version) +errno = pytest.main([ + 'tests', + '--ignore=tests/report', + '--cov=hermesv3_gr', + '--cov-report=term', + '--cov-report=html:{}/coverage_html'.format(report_dir), + '--cov-report=xml:{}/coverage.xml'.format(report_dir), +]) +sys.exit(errno) diff --git a/setup.py b/setup.py index ede6bbda45d92d89b8ee99ce6e4f1b0130d9774b..f7c01263178da84b0c675074dff85cd085fe1448 100644 --- a/setup.py +++ b/setup.py @@ -18,18 +18,16 @@ # along with HERMESv3_GR. If not, see . -from os import path from setuptools import find_packages from setuptools import setup +from hermesv3_gr import __version__ -here = path.abspath(path.dirname(__file__)) # Get the version number from the relevant file -with open(path.join(here, 'VERSION')) as f: - version = f.read().strip() +version = __version__ -with open("README.md", "r") as fh: - long_description = fh.read() +with open("README.md", "r") as f: + long_description = f.read() setup( name='hermesv3_gr', @@ -49,17 +47,18 @@ setup( install_requires=[ 'numpy', 'netCDF4>=1.3.1', - 'cdo>=1.3.4', + 'cdo>=1.3.3', 'pandas', 'geopandas', 'pyproj', 'configargparse', 'cf_units>=1.1.3', - 'ESMPy>=7.1.0', + 'ESMPy>=7.1.0.dev0', 'holidays', 'pytz', 'timezonefinder', - 'mpi4py' + 'mpi4py', + 'pytest', ], packages=find_packages(), classifiers=[ @@ -68,15 +67,51 @@ setup( "Operating System :: OS Independent", "Topic :: Scientific/Engineering :: Atmospheric Science" ], - include_package_data=True, - package_data={'hermesv3_gr': [ - 'data/*', - 'README', + package_data={'': [ + 'README.md', 'CHANGELOG', - 'VERSION', 'LICENSE', + 'data/*', + 'conf/*', + 'preproc/*' ] }, + data_files=[('.', ['LICENSE', 'CHANGELOG', ]), + ('conf', ['conf/hermes.conf', + 'conf/EI_configuration.csv', ]), + ('data', ['data/global_attributes.csv', + 'data/profiles/speciation/MolecularWeights.csv', + 'data/profiles/speciation/Speciation_profile_cb05_aero5_CMAQ.csv', + 'data/profiles/speciation/Speciation_profile_cb05_aero5_MONARCH_aerosols.csv', + 'data/profiles/speciation/Speciation_profile_cb05_aero5_MONARCH_fullchem.csv', + 'data/profiles/speciation/Speciation_profile_radm2_madesorgam_WRF_CHEM.csv', + 'data/profiles/temporal/TemporalProfile_Daily.csv', + 'data/profiles/temporal/TemporalProfile_Hourly.csv', + 'data/profiles/temporal/TemporalProfile_Monthly.csv', + 'data/profiles/temporal/tz_world_country_iso3166.csv', + 'data/profiles/vertical/CMAQ_15layers_vertical_description.csv', + 'data/profiles/vertical/MONARCH_Global_48layers_vertical_description.csv', + 'data/profiles/vertical/MONARCH_regional_48layers_vertical_description.csv', + 'data/profiles/vertical/Vertical_profile.csv', ]), + ('preproc', ['preproc/ceds_preproc.py', + 'preproc/eclipsev5a_preproc.py', + 'preproc/edgarv432_ap_preproc.py', + 'preproc/edgarv432_voc_preproc.py', + 'preproc/emep_preproc.py', + 'preproc/gfas12_preproc.py', + 'preproc/htapv2_preproc.py', + 'preproc/tno_mac_iii_preproc.py', + 'preproc/tno_mac_iii_preproc_voc_ratios.py', + 'preproc/wiedinmyer_preproc.py', ]), + ], - scripts=['bin/hermes_gr'] + include_package_data=True, + + entry_points={ + 'console_scripts': [ + 'hermesv3_gr = hermesv3_gr.hermes:run', + 'hermesv3_gr_copy_config_files = hermesv3_gr.tools.sample_files:copy_config_files', + 'hermesv3_gr_copy_preproc_files = hermesv3_gr.tools.sample_files:copy_preproc_files', + ], + }, ) diff --git a/tests/unit/test_lint.py b/tests/unit/test_lint.py new file mode 100644 index 0000000000000000000000000000000000000000..5a5fd7679dbee05f170d2a09184e7e086f3473b7 --- /dev/null +++ b/tests/unit/test_lint.py @@ -0,0 +1,33 @@ +""" Lint tests """ +import os +import unittest + +import pycodestyle # formerly known as pep8 + + +class TestLint(unittest.TestCase): + + def test_pep8_conformance(self): + """Test that we conform to PEP-8.""" + + check_paths = [ + 'hermesv3_gr', + 'tests', + ] + exclude_paths = [ + + ] + + print("PEP8 check of directories: {}\n".format(', '.join(check_paths))) + + # Get paths wrt package root + package_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + for paths in (check_paths, exclude_paths): + for i, path in enumerate(paths): + paths[i] = os.path.join(package_root, path) + + style = pycodestyle.StyleGuide() + style.options.exclude.extend(exclude_paths) + style.options.max_line_length = 120 + + self.assertEqual(style.check_files(check_paths).total_errors, 0) diff --git a/tests/unit/test_temporal.py b/tests/unit/test_temporal.py index 3ffed3c89bfc3964737b1acc72c6017d36fcbcff..6174229171f50de40ad585709123516a2b0b7446 100644 --- a/tests/unit/test_temporal.py +++ b/tests/unit/test_temporal.py @@ -28,388 +28,390 @@ class TestTemporalDistribution(unittest.TestCase): def setUp(self): pass - def testing_calculate_ending_date_1hour(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 1, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_ending_date(), - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0)) - - def testing_calculate_ending_date_24hours(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 24, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_ending_date(), - datetime(year=2016, month=01, day=01, hour=23, minute=0, second=0)) - - def testing_calculate_ending_date_3hour_each2(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 3, 2, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_ending_date(), - datetime(year=2016, month=01, day=01, hour=4, minute=0, second=0)) - - def testing_def_calculate_timedelta_3hour_each2(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 3, 2, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_timedelta(datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0)), - timedelta(hours=2)) - - def testing_def_calculate_timedelta_month(self): - temporal = TemporalDistribution( - datetime(year=2017, month=02, day=01, hour=0, minute=0, second=0), 'monthly', 1, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_timedelta(datetime(year=2017, month=02, day=01, hour=0, minute=0, second=0)), - timedelta(hours=24*28)) - - def testing_def_calculate_timedelta_month_leapyear(self): - temporal = TemporalDistribution( - datetime(year=2016, month=02, day=01, hour=0, minute=0, second=0), 'monthly', 1, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - self.assertEqual( - temporal.calculate_timedelta(datetime(year=2016, month=02, day=01, hour=0, minute=0, second=0)), - timedelta(hours=24*29)) - - def testing_get_tz_from_id(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.get_tz_from_id(309), - "Europe/Andorra") - - def testing_get_id_from_tz(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.get_id_from_tz("Europe/Andorra"), - 309) - - def testing_parse_tz(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.parse_tz("America/Fort_Nelson"), - 'America/Vancouver') - - def testing_find_closest_timezone_BCN(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.find_closest_timezone(41.390205, 2.154007), - 'Europe/Madrid') - - def testing_find_closest_timezone_MEX(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.find_closest_timezone(19.451054, -99.125519), - "America/Mexico_City") - - def testing_find_closest_timezone_Kuwait(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.find_closest_timezone(29.378586, 47.990341), - "Asia/Kuwait") - - def testing_find_closest_timezone_Shanghai(self): - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') - - self.assertEqual( - temporal.find_closest_timezone(31.267401, 121.522179), - "Asia/Shanghai") - - def testing_create_netcdf_timezones(self): - import numpy as np - from hermesv3_gr.modules.grids.grid import Grid - from hermesv3_gr.tools.netcdf_tools import extract_vars - - aux_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing' - if not os.path.exists(aux_path): - os.makedirs(aux_path) - - grid = Grid('global', aux_path) - grid.center_latitudes = np.array([[41.390205, 19.451054], [29.378586, 31.267401]]) - grid.center_longitudes = np.array([[2.154007, -99.125519], [47.990341, 121.522179]]) - - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - aux_path) - - self.assertTrue(temporal.create_netcdf_timezones(grid)) - - [timezones] = extract_vars(temporal.netcdf_timezones, ['timezone_id']) - timezones = list(timezones['data'][0, :].astype(int).flatten()) - - self.assertEqual(timezones, - [335, 147, 247, 268]) - - def testing_calculate_timezones(self): - self.testing_create_netcdf_timezones() - - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - self.assertEqual(temporal.calculate_timezones().tolist(), - [['Europe/Madrid', "America/Mexico_City"], ["Asia/Kuwait", "Asia/Shanghai"]]) - - def testing_calculate_2d_temporal_factors(self): - self.testing_create_netcdf_timezones() - - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - timezones = temporal.calculate_timezones() - - temporal.monthly_profile = {1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1., - 7: 1., - 8: 1., - 9: 1., - 10: 1., - 11: 1., - 12: 1.} - temporal.daily_profile_id = {0: 1., - 1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1.} - temporal.hourly_profile = {0: 1., - 1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1., - 7: 1., - 8: 1., - 9: 1., - 10: 1., - 11: 1., - 12: 1., - 13: 20., - 14: 1., - 15: 1., - 16: 1., - 17: 1., - 18: 1., - 19: 1., - 20: 1., - 21: 1., - 22: 1., - 23: 1.} - - self.assertEqual(temporal.calculate_2d_temporal_factors(datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), timezones).tolist(), - [[20., 1.], [1., 1.]]) - - def testing_do_temporal(self): - import numpy as np - from hermesv3_gr.modules.grids.grid import Grid - self.testing_create_netcdf_timezones() - - aux_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing' - - temporal = TemporalDistribution( - datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), 'hourly', 1, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - aux_path) - temporal.monthly_profile = {1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1., - 7: 1., - 8: 1., - 9: 1., - 10: 1., - 11: 1., - 12: 1.} - temporal.daily_profile_id = {0: 1., - 1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1.} - temporal.hourly_profile = {0: 1., - 1: 1., - 2: 1., - 3: 1., - 4: 1., - 5: 1., - 6: 1., - 7: 1., - 8: 1., - 9: 1., - 10: 1., - 11: 1., - 12: 1., - 13: 20., - 14: 1., - 15: 1., - 16: 1., - 17: 1., - 18: 1., - 19: 1., - 20: 1., - 21: 1., - 22: 1., - 23: 1.} - - grid = Grid('global', aux_path) - grid.center_latitudes = np.array([[41.390205, 19.451054], [29.378586, 31.267401]]) - grid.center_longitudes = np.array([[2.154007, -99.125519], [47.990341, 121.522179]]) - data_in = [{'data': np.array([[10., 10.], [10., 10.]])}] - # data_out = [{'data': np.array([[200., 10.], [10., 10.]])}] - data_out = temporal.do_temporal(data_in, grid) - - self.assertEqual(data_out[0]['data'].tolist(), [[[200., 10.], [10., 10.]]]) - - def testing_calculate_weekdays_no_leap_year(self): - from datetime import datetime - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - self.assertEqual(temporal.calculate_weekdays(datetime(year=2017, month=02, day=1)), - {0: 4, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}) - - def testing_calculate_weekdays_leap_year(self): - from datetime import datetime - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - self.assertEqual(temporal.calculate_weekdays(datetime(year=2016, month=02, day=1)), - {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}) - - def testing_calculate_weekdays_factors_full_month(self): - from datetime import datetime - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - - self.assertEqual(round(temporal.calculate_weekday_factor_full_month( - {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}), 5), - round(0.2/29, 5)) - - def testing_calculate_rebalance_factor(self): - from datetime import datetime - temporal = TemporalDistribution( - datetime(year=2016, month=01, day=01), 'monthly', 48, 1, - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', - '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', - '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') - - self.assertEqual(round(temporal.calculate_rebalance_factor( - {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, datetime(year=2016, month=02, day=1)), 5), - round(0.2/29, 5)) + # def testing_calculate_ending_date_1hour(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 1, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_ending_date(), + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0)) + # + # def testing_calculate_ending_date_24hours(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 24, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_ending_date(), + # datetime(year=2016, month=01, day=01, hour=23, minute=0, second=0)) + # + # def testing_calculate_ending_date_3hour_each2(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 3, 2, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_ending_date(), + # datetime(year=2016, month=01, day=01, hour=4, minute=0, second=0)) + # + # def testing_def_calculate_timedelta_3hour_each2(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0), 'hourly', 3, 2, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_timedelta(datetime(year=2016, month=01, day=01, hour=0, minute=0, second=0)), + # timedelta(hours=2)) + # + # def testing_def_calculate_timedelta_month(self): + # temporal = TemporalDistribution( + # datetime(year=2017, month=02, day=01, hour=0, minute=0, second=0), 'monthly', 1, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_timedelta(datetime(year=2017, month=02, day=01, hour=0, minute=0, second=0)), + # timedelta(hours=24*28)) + # + # def testing_def_calculate_timedelta_month_leapyear(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=02, day=01, hour=0, minute=0, second=0), 'monthly', 1, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # self.assertEqual( + # temporal.calculate_timedelta(datetime(year=2016, month=02, day=01, hour=0, minute=0, second=0)), + # timedelta(hours=24*29)) + # + # def testing_get_tz_from_id(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.get_tz_from_id(309), + # "Europe/Andorra") + # + # def testing_get_id_from_tz(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.get_id_from_tz("Europe/Andorra"), + # 309) + # + # def testing_parse_tz(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.parse_tz("America/Fort_Nelson"), + # 'America/Vancouver') + # + # def testing_find_closest_timezone_BCN(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.find_closest_timezone(41.390205, 2.154007), + # 'Europe/Madrid') + # + # def testing_find_closest_timezone_MEX(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.find_closest_timezone(19.451054, -99.125519), + # "America/Mexico_City") + # + # def testing_find_closest_timezone_Kuwait(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.find_closest_timezone(29.378586, 47.990341), + # "Asia/Kuwait") + # + # def testing_find_closest_timezone_Shanghai(self): + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/global_1.0_1.40625') + # + # self.assertEqual( + # temporal.find_closest_timezone(31.267401, 121.522179), + # "Asia/Shanghai") + # + # def testing_create_netcdf_timezones(self): + # import numpy as np + # from hermesv3_gr.modules.grids.grid import Grid + # from hermesv3_gr.tools.netcdf_tools import extract_vars + # + # aux_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing' + # if not os.path.exists(aux_path): + # os.makedirs(aux_path) + # + # grid = Grid('global', aux_path) + # grid.center_latitudes = np.array([[41.390205, 19.451054], [29.378586, 31.267401]]) + # grid.center_longitudes = np.array([[2.154007, -99.125519], [47.990341, 121.522179]]) + # + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # aux_path) + # + # self.assertTrue(temporal.create_netcdf_timezones(grid)) + # + # [timezones] = extract_vars(temporal.netcdf_timezones, ['timezone_id']) + # timezones = list(timezones['data'][0, :].astype(int).flatten()) + # + # self.assertEqual(timezones, + # [335, 147, 247, 268]) + # + # def testing_calculate_timezones(self): + # self.testing_create_netcdf_timezones() + # + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # self.assertEqual(temporal.calculate_timezones().tolist(), + # [['Europe/Madrid', "America/Mexico_City"], ["Asia/Kuwait", "Asia/Shanghai"]]) + # + # def testing_calculate_2d_temporal_factors(self): + # self.testing_create_netcdf_timezones() + # + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # timezones = temporal.calculate_timezones() + # + # temporal.monthly_profile = {1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1., + # 7: 1., + # 8: 1., + # 9: 1., + # 10: 1., + # 11: 1., + # 12: 1.} + # temporal.daily_profile_id = {0: 1., + # 1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1.} + # temporal.hourly_profile = {0: 1., + # 1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1., + # 7: 1., + # 8: 1., + # 9: 1., + # 10: 1., + # 11: 1., + # 12: 1., + # 13: 20., + # 14: 1., + # 15: 1., + # 16: 1., + # 17: 1., + # 18: 1., + # 19: 1., + # 20: 1., + # 21: 1., + # 22: 1., + # 23: 1.} + # + # self.assertEqual( + # temporal.calculate_2d_temporal_factors( + # datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), timezones).tolist(), + # [[20., 1.], [1., 1.]]) + # + # def testing_do_temporal(self): + # import numpy as np + # from hermesv3_gr.modules.grids.grid import Grid + # self.testing_create_netcdf_timezones() + # + # aux_path = '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing' + # + # temporal = TemporalDistribution( + # datetime(year=2017, month=6, day=23, hour=11, minute=0, second=0), 'hourly', 1, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # aux_path) + # temporal.monthly_profile = {1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1., + # 7: 1., + # 8: 1., + # 9: 1., + # 10: 1., + # 11: 1., + # 12: 1.} + # temporal.daily_profile_id = {0: 1., + # 1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1.} + # temporal.hourly_profile = {0: 1., + # 1: 1., + # 2: 1., + # 3: 1., + # 4: 1., + # 5: 1., + # 6: 1., + # 7: 1., + # 8: 1., + # 9: 1., + # 10: 1., + # 11: 1., + # 12: 1., + # 13: 20., + # 14: 1., + # 15: 1., + # 16: 1., + # 17: 1., + # 18: 1., + # 19: 1., + # 20: 1., + # 21: 1., + # 22: 1., + # 23: 1.} + # + # grid = Grid('global', aux_path) + # grid.center_latitudes = np.array([[41.390205, 19.451054], [29.378586, 31.267401]]) + # grid.center_longitudes = np.array([[2.154007, -99.125519], [47.990341, 121.522179]]) + # data_in = [{'data': np.array([[10., 10.], [10., 10.]])}] + # # data_out = [{'data': np.array([[200., 10.], [10., 10.]])}] + # data_out = temporal.do_temporal(data_in, grid) + # + # self.assertEqual(data_out[0]['data'].tolist(), [[[200., 10.], [10., 10.]]]) + # + # def testing_calculate_weekdays_no_leap_year(self): + # from datetime import datetime + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # self.assertEqual(temporal.calculate_weekdays(datetime(year=2017, month=02, day=1)), + # {0: 4, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}) + # + # def testing_calculate_weekdays_leap_year(self): + # from datetime import datetime + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # self.assertEqual(temporal.calculate_weekdays(datetime(year=2016, month=02, day=1)), + # {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}) + # + # def testing_calculate_weekdays_factors_full_month(self): + # from datetime import datetime + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # + # self.assertEqual(round(temporal.calculate_weekday_factor_full_month( + # {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}), 5), + # round(0.2/29, 5)) + # + # def testing_calculate_rebalance_factor(self): + # from datetime import datetime + # temporal = TemporalDistribution( + # datetime(year=2016, month=01, day=01), 'monthly', 48, 1, + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Monthly.csv', 'M001', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Daily.csv', 'D000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/TemporalProfile_Hourly.csv', 'H000', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', + # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') + # + # self.assertEqual(round(temporal.calculate_rebalance_factor( + # {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, datetime(year=2016, month=02, day=1)), 5), + # round(0.2/29, 5)) # def testing_get_temporal_daily_profile(self): # from datetime import datetime @@ -424,4 +426,3 @@ class TestTemporalDistribution(unittest.TestCase): # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') # # print temporal.get_temporal_daily_profile(date) -