From 283205b54a3b42072129a2ab33be56bc59e1865a Mon Sep 17 00:00:00 2001 From: Sergey Osipov Date: Mon, 18 May 2020 17:27:45 +0200 Subject: [PATCH 1/5] This fixes two issues: 1. The timezone was renamed. The change was reflected in the tz.csv 2. nx and ny should be integers so that grid_lcc can be generated the via numpy.linspace. --- data/profiles/temporal/tz_world_country_iso3166.csv | 2 +- hermesv3_gr/config/config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/profiles/temporal/tz_world_country_iso3166.csv b/data/profiles/temporal/tz_world_country_iso3166.csv index b6ddfd7..f7b3d29 100755 --- a/data/profiles/temporal/tz_world_country_iso3166.csv +++ b/data/profiles/temporal/tz_world_country_iso3166.csv @@ -170,7 +170,7 @@ "Greece";300;"Europe/Athens";311;"GRC" "Greenland";304;"America/Thule";191;"GRL" "Greenland";304;"America/Scoresbysund";181;"GRL" -"Greenland";304;"America/Godthab";107;"GRL" +"Greenland";304;"America/Nuuk";107;"GRL" "Greenland";304;"America/Danmarkshavn";95;"GRL" "Grenada";308;"America/Grenada";110;"GRD" "Guadeloupe";312;"America/Guadeloupe";111;"GLP" diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index aa613e1..7dc309a 100755 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -149,10 +149,10 @@ class Config(ArgParser): 'parameter of the GRIDDESC file.', type=float) p.add_argument('--nx', required=False, help='Number of grid columns. Corresponds to the NCOLS parameter of the GRIDDESC file.', - type=float) + type=int) p.add_argument('--ny', required=False, help='Number of grid rows. Corresponds to the NROWS parameter of the GRIDDESC file.', - type=float) + type=int) p.add_argument('--inc_x', required=False, help='X-coordinate cell dimension (meters). Corresponds to the XCELL parameter of the ' + 'GRIDDESC file.', type=float) -- GitLab From f396826c1314a19f4acccb7cae900b3a98dc0cb2 Mon Sep 17 00:00:00 2001 From: Sergey Osipov Date: Mon, 18 May 2020 18:10:50 +0200 Subject: [PATCH 2/5] updated grib to nc table (var names) --- preproc/GFAS_hourly_Parameters.csv | 52 +++++++++++++++--------------- preproc/gfas12_h_preproc.py | 13 ++++++-- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/preproc/GFAS_hourly_Parameters.csv b/preproc/GFAS_hourly_Parameters.csv index 7b8f414..452d7dd 100755 --- a/preproc/GFAS_hourly_Parameters.csv +++ b/preproc/GFAS_hourly_Parameters.csv @@ -1,27 +1,27 @@ Name;Short_Name;Units;id -Carbon Monoxide;co;kg m-2 s-1;param81.210.192 -Non-Methane Hydro-Carbons;nmhc;kg m-2 s-1;param83.210.192 -Nitrogen Oxides expressed as monoxide nitrogen;nox_no;kg m-2 s-1;param85.210.192 -Particulate Matter PM2.5;pm25;kg m-2 s-1;param87.210.192 -Organic Carbon;oc;kg m-2 s-1;param90.210.192 -Black Carbon;bc;kg m-2 s-1;param91.210.192 -Sulfur Dioxide;so2;kg m-2 s-1;param102.210.192 -Methanol;ch3oh;kg m-2 s-1;param103.210.192 -Higher Alkanes;hialkanes;kg m-2 s-1;param112.210.192 -Formaldehyde;ch2o;kg m-2 s-1;param113.210.192 -Acetaldehyde;c2h4o;kg m-2 s-1;param114.210.192 -Acetone;c3h6o;kg m-2 s-1;param115.210.192 -Ammonia;nh3;kg m-2 s-1;param116.210.192 -Dimethyl Sulfide;c2h6s;kg m-2 s-1;param117.210.192 -Ethane;c2h6;kg m-2 s-1;param118.210.192 -Toluene;c7h8;kg m-2 s-1;param231.210.192 -Benzene;c6h6;kg m-2 s-1;param232.210.192 -Xylene;c8h10;kg m-2 s-1;param233.210.192 -Butenes;c4h8;kg m-2 s-1;param234.210.192 -Pentenes;c5h10;kg m-2 s-1;param235.210.192 -Hexene;c6h12;kg m-2 s-1;param236.210.192 -Octene;c8h16;kg m-2 s-1;param237.210.192 -Butanes;c4h10;kg m-2 s-1;param238.210.192 -Pentanes;c5h12;kg m-2 s-1;param239.210.192 -Hexanes;c6h14;kg m-2 s-1;param240.210.192 -Heptane;c7h16;kg m-2 s-1;param241.210.192 +Carbon Monoxide;co;kg m-2 s-1;var81 +Non-Methane Hydro-Carbons;nmhc;kg m-2 s-1;var83 +Nitrogen Oxides expressed as monoxide nitrogen;nox_no;kg m-2 s-1;var85 +Particulate Matter PM2.5;pm25;kg m-2 s-1;var87 +Organic Carbon;oc;kg m-2 s-1;var90 +Black Carbon;bc;kg m-2 s-1;var91 +Sulfur Dioxide;so2;kg m-2 s-1;var102 +Methanol;ch3oh;kg m-2 s-1;var103 +Higher Alkanes;hialkanes;kg m-2 s-1;var112 +Formaldehyde;ch2o;kg m-2 s-1;var113 +Acetaldehyde;c2h4o;kg m-2 s-1;var114 +Acetone;c3h6o;kg m-2 s-1;var115 +Ammonia;nh3;kg m-2 s-1;var116 +Dimethyl Sulfide;c2h6s;kg m-2 s-1;var117 +Ethane;c2h6;kg m-2 s-1;var118 +Toluene;c7h8;kg m-2 s-1;var231 +Benzene;c6h6;kg m-2 s-1;var232 +Xylene;c8h10;kg m-2 s-1;var233 +Butenes;c4h8;kg m-2 s-1;var234 +Pentenes;c5h10;kg m-2 s-1;var235 +Hexene;c6h12;kg m-2 s-1;var236 +Octene;c8h16;kg m-2 s-1;var237 +Butanes;c4h10;kg m-2 s-1;var238 +Pentanes;c5h12;kg m-2 s-1;var239 +Hexanes;c6h14;kg m-2 s-1;var240 +Heptane;c7h16;kg m-2 s-1;var241 diff --git a/preproc/gfas12_h_preproc.py b/preproc/gfas12_h_preproc.py index e38d1d3..4e9094c 100755 --- a/preproc/gfas12_h_preproc.py +++ b/preproc/gfas12_h_preproc.py @@ -37,10 +37,17 @@ INPUT_PATH = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' INPUT_NAME = 'gfas_hourly_.grb' OUTPUT_PATH = '/esarchive/recon/ecmwf/gfas' -STARTING_DATE = datetime(year=2018, month=11, day=0o1) -ENDIND_DATE = datetime(year=2018, month=11, day=0o1) +# STARTING_DATE = datetime(year=2017, month=6, day=0o1) +# ENDIND_DATE = datetime(year=2018, month=11, day=0o1) -PARAMETERS_FILE = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_hourly_Parameters.csv' +INPUT_PATH = '/work/mm0062/b302074/Data/emissions/CAMS-GFAS/' # get_root_storage_path_on_hpc() + 'Data/emissions/CAMS-GFAS/ +INPUT_NAME = 'gfas_hourly_.grb' +OUTPUT_PATH = INPUT_PATH + '/hermesv3_gr_pp/' + +STARTING_DATE = datetime(year=2017, month=6, day=15) +ENDIND_DATE = datetime(year=2017, month=9, day=2) + +PARAMETERS_FILE = '/work/mm0062/b302074/workspace/python/hermesv3_gr/preproc/GFAS_hourly_Parameters.csv' # ============================================================== -- GitLab From 0acca0266caf04989f621b3a626174d739080a70 Mon Sep 17 00:00:00 2001 From: Sergey Osipov Date: Mon, 18 May 2020 22:43:28 +0200 Subject: [PATCH 3/5] sync --- preproc/GFAS_hourly_Parameters.csv | 52 +++++++++++++++--------------- preproc/gfas12_h_preproc.py | 13 ++------ preproc/gfas12_preproc.py | 10 +++--- 3 files changed, 34 insertions(+), 41 deletions(-) diff --git a/preproc/GFAS_hourly_Parameters.csv b/preproc/GFAS_hourly_Parameters.csv index 452d7dd..7b8f414 100755 --- a/preproc/GFAS_hourly_Parameters.csv +++ b/preproc/GFAS_hourly_Parameters.csv @@ -1,27 +1,27 @@ Name;Short_Name;Units;id -Carbon Monoxide;co;kg m-2 s-1;var81 -Non-Methane Hydro-Carbons;nmhc;kg m-2 s-1;var83 -Nitrogen Oxides expressed as monoxide nitrogen;nox_no;kg m-2 s-1;var85 -Particulate Matter PM2.5;pm25;kg m-2 s-1;var87 -Organic Carbon;oc;kg m-2 s-1;var90 -Black Carbon;bc;kg m-2 s-1;var91 -Sulfur Dioxide;so2;kg m-2 s-1;var102 -Methanol;ch3oh;kg m-2 s-1;var103 -Higher Alkanes;hialkanes;kg m-2 s-1;var112 -Formaldehyde;ch2o;kg m-2 s-1;var113 -Acetaldehyde;c2h4o;kg m-2 s-1;var114 -Acetone;c3h6o;kg m-2 s-1;var115 -Ammonia;nh3;kg m-2 s-1;var116 -Dimethyl Sulfide;c2h6s;kg m-2 s-1;var117 -Ethane;c2h6;kg m-2 s-1;var118 -Toluene;c7h8;kg m-2 s-1;var231 -Benzene;c6h6;kg m-2 s-1;var232 -Xylene;c8h10;kg m-2 s-1;var233 -Butenes;c4h8;kg m-2 s-1;var234 -Pentenes;c5h10;kg m-2 s-1;var235 -Hexene;c6h12;kg m-2 s-1;var236 -Octene;c8h16;kg m-2 s-1;var237 -Butanes;c4h10;kg m-2 s-1;var238 -Pentanes;c5h12;kg m-2 s-1;var239 -Hexanes;c6h14;kg m-2 s-1;var240 -Heptane;c7h16;kg m-2 s-1;var241 +Carbon Monoxide;co;kg m-2 s-1;param81.210.192 +Non-Methane Hydro-Carbons;nmhc;kg m-2 s-1;param83.210.192 +Nitrogen Oxides expressed as monoxide nitrogen;nox_no;kg m-2 s-1;param85.210.192 +Particulate Matter PM2.5;pm25;kg m-2 s-1;param87.210.192 +Organic Carbon;oc;kg m-2 s-1;param90.210.192 +Black Carbon;bc;kg m-2 s-1;param91.210.192 +Sulfur Dioxide;so2;kg m-2 s-1;param102.210.192 +Methanol;ch3oh;kg m-2 s-1;param103.210.192 +Higher Alkanes;hialkanes;kg m-2 s-1;param112.210.192 +Formaldehyde;ch2o;kg m-2 s-1;param113.210.192 +Acetaldehyde;c2h4o;kg m-2 s-1;param114.210.192 +Acetone;c3h6o;kg m-2 s-1;param115.210.192 +Ammonia;nh3;kg m-2 s-1;param116.210.192 +Dimethyl Sulfide;c2h6s;kg m-2 s-1;param117.210.192 +Ethane;c2h6;kg m-2 s-1;param118.210.192 +Toluene;c7h8;kg m-2 s-1;param231.210.192 +Benzene;c6h6;kg m-2 s-1;param232.210.192 +Xylene;c8h10;kg m-2 s-1;param233.210.192 +Butenes;c4h8;kg m-2 s-1;param234.210.192 +Pentenes;c5h10;kg m-2 s-1;param235.210.192 +Hexene;c6h12;kg m-2 s-1;param236.210.192 +Octene;c8h16;kg m-2 s-1;param237.210.192 +Butanes;c4h10;kg m-2 s-1;param238.210.192 +Pentanes;c5h12;kg m-2 s-1;param239.210.192 +Hexanes;c6h14;kg m-2 s-1;param240.210.192 +Heptane;c7h16;kg m-2 s-1;param241.210.192 diff --git a/preproc/gfas12_h_preproc.py b/preproc/gfas12_h_preproc.py index 4e9094c..e38d1d3 100755 --- a/preproc/gfas12_h_preproc.py +++ b/preproc/gfas12_h_preproc.py @@ -37,17 +37,10 @@ INPUT_PATH = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' INPUT_NAME = 'gfas_hourly_.grb' OUTPUT_PATH = '/esarchive/recon/ecmwf/gfas' -# STARTING_DATE = datetime(year=2017, month=6, day=0o1) -# ENDIND_DATE = datetime(year=2018, month=11, day=0o1) +STARTING_DATE = datetime(year=2018, month=11, day=0o1) +ENDIND_DATE = datetime(year=2018, month=11, day=0o1) -INPUT_PATH = '/work/mm0062/b302074/Data/emissions/CAMS-GFAS/' # get_root_storage_path_on_hpc() + 'Data/emissions/CAMS-GFAS/ -INPUT_NAME = 'gfas_hourly_.grb' -OUTPUT_PATH = INPUT_PATH + '/hermesv3_gr_pp/' - -STARTING_DATE = datetime(year=2017, month=6, day=15) -ENDIND_DATE = datetime(year=2017, month=9, day=2) - -PARAMETERS_FILE = '/work/mm0062/b302074/workspace/python/hermesv3_gr/preproc/GFAS_hourly_Parameters.csv' +PARAMETERS_FILE = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_hourly_Parameters.csv' # ============================================================== diff --git a/preproc/gfas12_preproc.py b/preproc/gfas12_preproc.py index 86ffcf9..27c6c59 100755 --- a/preproc/gfas12_preproc.py +++ b/preproc/gfas12_preproc.py @@ -33,14 +33,14 @@ Besides citing HERMESv3_GR, users must also acknowledge the use of the correspon """ # ============== CONFIGURATION PARAMETERS ====================== -INPUT_PATH = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' +INPUT_PATH = '/work/mm0062/b302074/Data/emissions/CAMS-GFAS/' # get_root_storage_path_on_hpc() + 'Data/emissions/CAMS-GFAS/ INPUT_NAME = 'ga_.grb' -OUTPUT_PATH = '/esarchive/recon/ecmwf/gfas' +OUTPUT_PATH = INPUT_PATH + '/hermesv3_gr_pp/' -STARTING_DATE = datetime(year=2018, month=8, day=29) -ENDIND_DATE = datetime(year=2018, month=8, day=29) +STARTING_DATE = datetime(year=2017, month=6, day=15) +ENDIND_DATE = datetime(year=2017, month=9, day=2) -PARAMETERS_FILE = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_Parameters.csv' +PARAMETERS_FILE = '/work/mm0062/b302074/workspace/python/hermesv3_gr/preproc/GFAS_Parameters.csv' # ============================================================== -- GitLab From 46ce61f9859380899e6fb958244ead7ed685c7ee Mon Sep 17 00:00:00 2001 From: osipovs Date: Mon, 15 Jun 2020 23:17:16 +0200 Subject: [PATCH 4/5] sync: omi-htap preproc dev --- preproc/omi_htapV2_preproc.py | 552 ++++++++++++++++++++++++++++++++++ 1 file changed, 552 insertions(+) create mode 100755 preproc/omi_htapV2_preproc.py diff --git a/preproc/omi_htapV2_preproc.py b/preproc/omi_htapV2_preproc.py new file mode 100755 index 0000000..ebc55f6 --- /dev/null +++ b/preproc/omi_htapV2_preproc.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python + +# Copyright 2018 Earth Sciences Department, BSC-CNS +# +# This file is part of HERMESv3_GR. +# +# HERMESv3_GR is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# HERMESv3_GR is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with HERMESv3_GR. If not, see . + + +import os + +__author__ = 'Sergey Osipov ' + +# ============== README ====================== +""" + +htapv2_preproc.py was used as a template + +downloading website: https://avdc.gsfc.nasa.gov/pub/data/project/OMI_HTAP_emis/ +reference: +Besides citing HERMESv3_GR, users must also acknowledge the use of the corresponding emission inventories in their works +""" + +# ============== CONFIGURATION PARAMETERS ====================== +INPUT_PATH = '/work/mm0062/b302074/Data/emissions/OMI-HTAP' +OUTPUT_PATH = '/work/mm0062/b302074/Data/emissions/OMI-HTAP/hermesv3_gr_pp' + +INPUT_NAME = 'htapv2.2.emisso2.surface.x3600y1800t12..integrate.nc4' + +# list_years = [2008, 2010] +LIST_YEARS = [2017] + +# RETRO ratios applied to HTAPv2 NMVOC emissions +# (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) +VOC_RATIO_PATH = '/esarchive/recon/jrc/htapv2/original_files/retro_nmvoc_ratio_2000_01x01' +VOC_RATIO_NAME = 'retro_nmvoc_ratio__2000_0.1deg.nc' +VOC_RATIO_AIR_NAME = 'VOC_split_AIR.csv' +VOC_RATIO_SHIPS_NAME = 'VOC_split_SHIP.csv' +# ============================================================== + + +def do_transformation_annual(filename, out_path, pollutant, sector, year): + """ + Re-write the HTAPv2 inputs following ES anc CF-1.6 conventions for annual inventories. + + :param filename: Path to the input file. + :type filename: str + + :param out_path: Path to store the output. + :type out_path: str + + :param pollutant: Pollutant name. + :type pollutant: str + + :param sector: Name of the sector. + :type sector: str + + :param year: Year. + :type year: int + + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area + from hermesv3_gr.tools.coordinates_tools import create_bounds + print(filename) + [c_lats, c_lons] = extract_vars(filename, ['lat', 'lon']) + + if pollutant == 'pm25': + [data] = extract_vars(filename, ['emi_pm2.5'], + attributes_list=['standard_name', 'units', 'cell_method', 'long_name']) + else: + [data] = extract_vars(filename, ['emi_{0}'.format(pollutant)], + attributes_list=['standard_name', 'units', 'cell_method', 'long_name']) + data['data'] = data['data'].reshape((1,) + data['data'].shape) + data['name'] = pollutant + + global_attributes = { + 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, data['long_name']), + 'Conventions': 'CF-1.6', + 'institution': 'European Commission, Joint Research Centre (JRC)', + 'source': 'HTAPv2', + 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + + '2017-04-04: Added time dimension (UNLIMITED);\n' + + '2017-04-04: Added boundaries;\n' + + '2017-04-04: Added global attributes;\n' + + '2017-04-04: Re-naming pollutant;\n' + + '2017-04-04: Added cell_area variable;\n', + 'references': 'EC, JRC / US EPA, HTAP_V2. ' + + 'http://edgar.jrc.ec.europa.eu/htap/EDGAR-HTAP_v1_final_jan2012.pdf\n ' + + 'http://edgar.jrc.ec.europa.eu/htap_v2/', + 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + + '(Barcelona Supercomputing Center)', + } + + out_path = os.path.join(out_path, pollutant + '_' + sector.lower()) + if not os.path.exists(out_path): + os.makedirs(out_path) + + out_path = os.path.join(out_path, '{0}_{1}.nc'.format(pollutant, year)) + print(out_path) + write_netcdf(out_path, c_lats['data'], c_lons['data'], [data], + boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), + cell_area=get_grid_area(filename), global_attributes=global_attributes,) + return True + + +def do_transformation(filename_list, out_path, pollutant, sector, year): + """ + Re-write the OMI-HTAPv2 inputs following ES anc CF-1.6 conventions. + + :param filename_list: List of input file names. + :type filename_list: list + + :param out_path: Path to store the output. + :type out_path: str + + :param pollutant: Pollutant name. + :type pollutant: str + + :param sector: Name of the sector. + :type sector: str + + :param year: Year. + :type year: int + + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area + from hermesv3_gr.tools.coordinates_tools import create_bounds + + for month in range(1, 13): + print(filename_list[month - 1]) + [c_lats, c_lons] = extract_vars(filename_list[0], ['lat', 'lon']) + + [data] = extract_vars(filename_list[0], ['sanl1'], + attributes_list=['standard_name', 'units', 'cell_method', 'long_name']) + # TODO: reuse all 12 months + data['data'] = data['data'][month].reshape((1,) + data['data'].shape) + data['name'] = pollutant + + global_attributes = { + 'title': 'OMI-HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, data['long_name']), + 'Conventions': 'CF-1.6', + 'source': 'OMI-HTAPv2', + 'history': '15-06-2020: first implementation;\n', + 'comment': 'Re-writing done by Sergey Osipov (Serega.Osipov@gmail.com) from MPIC ' + + '(Max Planck Institute for Chemistry)', + } + + out_path_aux = os.path.join(out_path, pollutant + '_' + sector.lower()) + if not os.path.exists(out_path_aux): + os.makedirs(out_path_aux) + + out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant, year, str(month).zfill(2))) + write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data], + boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), + cell_area=get_grid_area(filename_list[month - 1]), global_attributes=global_attributes,) + return True + + +def do_ratio_list(sector=None): + # TODO Documentation + """ + + :param sector: + :return: + """ + if sector == 'SHIPS': + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_SHIPS_NAME)} + elif sector == 'AIR_CDS': + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} + elif sector == 'AIR_CRS': + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} + elif sector == 'AIR_LTO': + return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} + return { + 'voc01': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '01')), + 'voc02': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '02')), + 'voc03': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '03')), + 'voc04': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '04')), + 'voc05': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '05')), + 'voc06': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '06')), + 'voc07': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '07')), + 'voc08': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '08')), + 'voc09': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '09')), + 'voc12': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '12')), + 'voc13': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '13')), + 'voc14': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '14')), + 'voc15': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '15')), + 'voc16': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '16')), + 'voc17': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '17')), + 'voc18': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '18')), + 'voc19': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '19')), + 'voc20': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '20')), + 'voc21': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '21')), + 'voc22': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '22')), + 'voc23': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '23')), + 'voc24': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '24')), + 'voc25': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '25')), + } + + +def do_nmvoc_month_transformation(filename_list, out_path, sector, year): + # TODO Docuemtnation + """ + + :param filename_list: + :param out_path: + :param sector: + :param year: + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf + from hermesv3_gr.tools.coordinates_tools import create_bounds + + nmvoc_ratio_list = do_ratio_list() + + print(sector) + if sector == 'ENERGY': + ratio_var = 'pow' + + nmvoc_ratio_list.pop('voc18', None) + nmvoc_ratio_list.pop('voc19', None) + nmvoc_ratio_list.pop('voc20', None) + + elif sector == 'RESIDENTIAL': + ratio_var = 'res' + + nmvoc_ratio_list.pop('voc18', None) + nmvoc_ratio_list.pop('voc20', None) + + elif sector == 'TRANSPORT': + ratio_var = 'tra' + + nmvoc_ratio_list.pop('voc01', None) + nmvoc_ratio_list.pop('voc18', None) + nmvoc_ratio_list.pop('voc19', None) + nmvoc_ratio_list.pop('voc20', None) + nmvoc_ratio_list.pop('voc24', None) + nmvoc_ratio_list.pop('voc25', None) + + print(type(nmvoc_ratio_list), nmvoc_ratio_list) + + for month in range(1, 13): + print(filename_list[month - 1]) + c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) + + [data] = extract_vars(filename_list[month - 1], ['emi_nmvoc']) + + for voc, ratio_file in nmvoc_ratio_list.items(): + print(voc, ratio_file) + + pollutant = voc + [ratio] = extract_vars(ratio_file, [ratio_var]) + + data_aux = data.copy() + data_aux['data'] = data['data'] * ratio['data'] + data_aux['data'] = data_aux['data'].reshape((1,) + data_aux['data'].shape) + data_aux['name'] = voc + data_aux['units'] = 'kg m-2 s-1' + global_attributes = { + 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, pollutant), + 'Conventions': 'CF-1.6', + 'institution': 'European Commission, Joint Research Centre (JRC)', + 'source': 'HTAPv2', + 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + + '2017-04-28: ...', + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and ' + + 'global emission grid maps for 2008 and 2010 to study hemispheric transport of air ' + + 'pollution, Atmos. Chem. Phys., 15, 11411-11432, ' + + 'https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', + 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + + '(Barcelona Supercomputing Center)', + } + + out_path_aux = os.path.join(out_path, pollutant + '_' + sector.lower()) + if not os.path.exists(out_path_aux): + os.makedirs(out_path_aux) + + out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant, year, str(month).zfill(2))) + print(out_path_aux) + write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], + boundary_latitudes=create_bounds(c_lats['data']), + boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) + return True + + +def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year): + # TODO Documentation + """ + + :param filename_list: + :param out_path: + :param sector: + :param year: + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf + from hermesv3_gr.tools.coordinates_tools import create_bounds + + nmvoc_ratio_list = do_ratio_list() + + print(sector) + + print(type(nmvoc_ratio_list), nmvoc_ratio_list) + + for month in range(1, 13): + print(filename_list[month - 1]) + c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) + + [ind, exf, sol] = extract_vars(filename_list[month - 1], ['emiss_ind', 'emiss_exf', 'emiss_sol']) + + for voc, ratio_file in nmvoc_ratio_list.items(): + print(voc, ratio_file) + data = { + 'name': voc, + 'units': 'kg m-2 s-1', + } + if voc in ['voc02', 'voc03', 'voc04', 'voc05', 'voc07', 'voc08', 'voc12', 'voc13']: + [r_inc, r_exf] = extract_vars(ratio_file, ['inc', 'exf']) + data.update({'data': ind['data'] * r_inc['data'] + exf['data'] * r_exf['data']}) + elif voc in ['voc01', 'voc23', 'voc25']: + [r_inc, r_sol] = extract_vars(ratio_file, ['inc', 'sol']) + data.update({'data': ind['data'] * r_inc['data'] + sol['data'] * r_sol['data']}) + elif voc in ['voc09', 'voc16', 'voc21', 'voc22', 'voc24']: + [r_inc] = extract_vars(ratio_file, ['inc']) + data.update({'data': ind['data'] * r_inc['data']}) + # elif voc in []: + # [r_exf, r_sol] = extract_vars(ratio_file, ['exf', 'sol']) + # data.update({'data': exf['data']*r_exf['data'] + sol['data']*r_sol['data']}) + elif voc in ['voc18', 'voc19', 'voc20']: + [r_sol] = extract_vars(ratio_file, ['sol']) + data.update({'data': sol['data'] * r_sol['data']}) + else: + [r_inc, r_exf, r_sol] = extract_vars(ratio_file, ['inc', 'exf', 'sol']) + data.update({'data': ind['data'] * r_inc['data'] + exf['data'] * r_exf['data'] + + sol['data'] * r_sol['data']}) + + global_attributes = { + 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, voc), + 'Conventions': 'CF-1.6', + 'institution': 'European Commission, Joint Research Centre (JRC)', + 'source': 'HTAPv2', + 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + + '2017-04-28: ...', + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and ' + + 'global emission grid maps for 2008 and 2010 to study hemispheric transport of air ' + + 'pollution, Atmos. Chem. Phys., 15, 11411-11432, ' + + 'https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', + 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + + '(Barcelona Supercomputing Center)', + } + + out_path_aux = os.path.join(out_path, voc + '_industry') + if not os.path.exists(out_path_aux): + os.makedirs(out_path_aux) + + out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) + print(out_path_aux) + write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data], + boundary_latitudes=create_bounds(c_lats['data']), + boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) + + +def do_nmvoc_year_transformation(filename, out_path, sector, year): + # TODO Documentation + """ + + :param filename: + :param out_path: + :param sector: + :param year: + :return: + """ + import pandas as pd + from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf + from hermesv3_gr.tools.coordinates_tools import create_bounds + + nmvoc_ratio_file = do_ratio_list(sector)['all'] + nmvoc_ratio_list = pd.read_csv(nmvoc_ratio_file, sep=';') + + c_lats, c_lons = extract_vars(filename, ['lat', 'lon']) + + [data] = extract_vars(filename, ['emi_nmvoc']) + + for i, voc_ratio in nmvoc_ratio_list.iterrows(): + pollutant = voc_ratio['voc_group'] + ratio = voc_ratio['factor'] + + data_aux = data.copy() + data_aux['data'] = data['data'] * ratio + data_aux['data'] = data_aux['data'].reshape((1,) + data_aux['data'].shape) + data_aux['name'] = pollutant + data_aux['units'] = 'kg m-2 s-1' + global_attributes = { + 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, pollutant), + 'Conventions': 'CF-1.6', + 'institution': 'European Commission, Joint Research Centre (JRC)', + 'source': 'HTAPv2', + 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + + '2017-04-28: ...', + 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global ' + + 'emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, ' + + 'Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + + 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', + 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + + '(Barcelona Supercomputing Center)\n ' + + 'HTAP contact: greet.maenhout@jrc.ec.europa.eu', + } + + out_path_aux = os.path.join(out_path, pollutant + '_' + sector.lower()) + if not os.path.exists(out_path_aux): + os.makedirs(out_path_aux) + + out_path_aux = os.path.join(out_path_aux, '{0}_{1}.nc'.format(pollutant, year)) + print(out_path_aux) + write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], + boundary_latitudes=create_bounds(c_lats['data']), + boundary_longitudes=create_bounds(c_lons['data']), + global_attributes=global_attributes,) + + +def get_pollutant_dict(): + # TODO Documentation + """ + + :return: + """ + p_dict = { + 'so2': 'SO2', + } + return p_dict + + +def get_sector_dict(): + # TODO Documentation + """ + + :return: + """ + common_dict = { + 'month': ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT'], + 'year': ['SHIPS', 'AIR_CDS', 'AIR_CRS', 'AIR_LTO'] + } + sector_dict = { + 'so2': common_dict, + } + return sector_dict + + +def get_nmvoc_sector_dict(): + # TODO Documentation + """ + + :return: + """ + nmvoc_sectors = {'month': ['ENERGY', 'INDUSTRY_3subsectors', 'RESIDENTIAL', 'TRANSPORT'], + 'year': ['SHIPS', 'AIR_CDS', 'AIR_CRS', 'AIR_LTO']} + return nmvoc_sectors + + +def check_vocs(year): + # TODO Documentation + """ + + :param year: + :return: + """ + from hermesv3_gr.tools.netcdf_tools import extract_vars + for month in range(1, 12 + 1, 1): + for snap in ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT']: + nmvoc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', 'nmvoc_{0}'.format(snap.lower()), + 'nmvoc_{0}{1}.nc'.format(year, str(month).zfill(2))) + [new_voc] = extract_vars(nmvoc_path, ['nmvoc']) + nmvoc_sum = new_voc['data'].sum() + + voc_sum = 0 + for voc in ['voc{0}'.format(str(x).zfill(2)) for x in range(1, 25 + 1, 1)]: + voc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(voc, snap.lower()), + '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) + if os.path.exists(voc_path): + [new_voc] = extract_vars(voc_path, [voc]) + voc_sum += new_voc['data'].sum() + + print('{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum) / nmvoc_sum, month)) + + +if __name__ == '__main__': + for year in LIST_YEARS: + for pollutant_dict in get_pollutant_dict().items(): + for current_sector in get_sector_dict()[pollutant_dict[0]]['month']: + # TODO: surface & elevated + # input_name_aux = INPUT_NAME.replace('', current_sector) + input_name_aux = INPUT_NAME.replace('', str(year)) + files = [os.path.join(INPUT_PATH, input_name_aux),] + # TODO: surf or elevated + do_transformation(files, os.path.join(OUTPUT_PATH, 'monthly_mean'), pollutant_dict[0], current_sector, year) + # annual inventories + # for current_sector in get_sector_dict()[pollutant_dict[0]]['year']: + # if current_sector[0:3] == 'AIR': + # input_name_aux = INPUT_NAME_AIR + # else: + # input_name_aux = INPUT_NAME_SHIPS + # input_name_aux = input_name_aux.replace('', current_sector) + # input_name_aux = input_name_aux.replace('', str(year)) + # input_name_aux = input_name_aux.replace('', pollutant_dict[1]) + # input_name_aux = os.path.join(INPUT_PATH, input_name_aux) + # + # do_transformation_annual(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean', ), pollutant_dict[0], + # current_sector, year) + + for current_sector in get_nmvoc_sector_dict()['month']: + if current_sector == 'INDUSTRY_3subsectors': + input_name_aux = INPUT_NAME_NMVOC_INDUSTRY + else: + input_name_aux = INPUT_NAME + input_name_aux = input_name_aux.replace('', 'NMVOC') + input_name_aux = input_name_aux.replace('', current_sector) + input_name_aux = input_name_aux.replace('', str(year)) + files = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) + for aux_month in range(1, 13)] + + if current_sector == 'INDUSTRY_3subsectors': + do_nmvoc_industry_month_transformation(files, os.path.join(OUTPUT_PATH, 'monthly_mean'), + current_sector, year) + else: + do_nmvoc_month_transformation(files, os.path.join(OUTPUT_PATH, 'monthly_mean'), current_sector, year) + # for current_sector in get_nmvoc_sector_dict()['year']: + # if current_sector[0:3] == 'AIR': + # input_name_aux = INPUT_NAME_AIR + # else: + # input_name_aux = INPUT_NAME_SHIPS + # input_name_aux = input_name_aux.replace('', 'NMVOC') + # input_name_aux = input_name_aux.replace('', current_sector) + # input_name_aux = input_name_aux.replace('', str(y)) + # input_name_aux = os.path.join(INPUT_PATH, input_name_aux) + # print(input_name_aux) + # do_nmvoc_year_transformation(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean'), current_sector, y) -- GitLab From b6aca90eed4859d4886216d851c9022cfb2ca553 Mon Sep 17 00:00:00 2001 From: osipovs Date: Mon, 22 Jun 2020 21:46:13 +0200 Subject: [PATCH 5/5] OMI-HTAP preprocessing script completed --- preproc/omi_htapV2_preproc.py | 453 ++-------------------------------- 1 file changed, 23 insertions(+), 430 deletions(-) diff --git a/preproc/omi_htapV2_preproc.py b/preproc/omi_htapV2_preproc.py index ebc55f6..0b88cb5 100755 --- a/preproc/omi_htapV2_preproc.py +++ b/preproc/omi_htapV2_preproc.py @@ -36,84 +36,11 @@ Besides citing HERMESv3_GR, users must also acknowledge the use of the correspon INPUT_PATH = '/work/mm0062/b302074/Data/emissions/OMI-HTAP' OUTPUT_PATH = '/work/mm0062/b302074/Data/emissions/OMI-HTAP/hermesv3_gr_pp' -INPUT_NAME = 'htapv2.2.emisso2.surface.x3600y1800t12..integrate.nc4' +INPUT_NAME = 'htapv2.2.emisso2..x3600y1800t12..integrate.nc4' # list_years = [2008, 2010] LIST_YEARS = [2017] -# RETRO ratios applied to HTAPv2 NMVOC emissions -# (http://iek8wikis.iek.fz-juelich.de/HTAPWiki/WP1.1?highlight=%28%28WP1.1%29%29) -VOC_RATIO_PATH = '/esarchive/recon/jrc/htapv2/original_files/retro_nmvoc_ratio_2000_01x01' -VOC_RATIO_NAME = 'retro_nmvoc_ratio__2000_0.1deg.nc' -VOC_RATIO_AIR_NAME = 'VOC_split_AIR.csv' -VOC_RATIO_SHIPS_NAME = 'VOC_split_SHIP.csv' -# ============================================================== - - -def do_transformation_annual(filename, out_path, pollutant, sector, year): - """ - Re-write the HTAPv2 inputs following ES anc CF-1.6 conventions for annual inventories. - - :param filename: Path to the input file. - :type filename: str - - :param out_path: Path to store the output. - :type out_path: str - - :param pollutant: Pollutant name. - :type pollutant: str - - :param sector: Name of the sector. - :type sector: str - - :param year: Year. - :type year: int - - :return: - """ - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area - from hermesv3_gr.tools.coordinates_tools import create_bounds - print(filename) - [c_lats, c_lons] = extract_vars(filename, ['lat', 'lon']) - - if pollutant == 'pm25': - [data] = extract_vars(filename, ['emi_pm2.5'], - attributes_list=['standard_name', 'units', 'cell_method', 'long_name']) - else: - [data] = extract_vars(filename, ['emi_{0}'.format(pollutant)], - attributes_list=['standard_name', 'units', 'cell_method', 'long_name']) - data['data'] = data['data'].reshape((1,) + data['data'].shape) - data['name'] = pollutant - - global_attributes = { - 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, data['long_name']), - 'Conventions': 'CF-1.6', - 'institution': 'European Commission, Joint Research Centre (JRC)', - 'source': 'HTAPv2', - 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + - '2017-04-04: Added time dimension (UNLIMITED);\n' + - '2017-04-04: Added boundaries;\n' + - '2017-04-04: Added global attributes;\n' + - '2017-04-04: Re-naming pollutant;\n' + - '2017-04-04: Added cell_area variable;\n', - 'references': 'EC, JRC / US EPA, HTAP_V2. ' + - 'http://edgar.jrc.ec.europa.eu/htap/EDGAR-HTAP_v1_final_jan2012.pdf\n ' + - 'http://edgar.jrc.ec.europa.eu/htap_v2/', - 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + - '(Barcelona Supercomputing Center)', - } - - out_path = os.path.join(out_path, pollutant + '_' + sector.lower()) - if not os.path.exists(out_path): - os.makedirs(out_path) - - out_path = os.path.join(out_path, '{0}_{1}.nc'.format(pollutant, year)) - print(out_path) - write_netcdf(out_path, c_lats['data'], c_lons['data'], [data], - boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), - cell_area=get_grid_area(filename), global_attributes=global_attributes,) - return True - def do_transformation(filename_list, out_path, pollutant, sector, year): """ @@ -139,14 +66,22 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds - for month in range(1, 13): - print(filename_list[month - 1]) - [c_lats, c_lons] = extract_vars(filename_list[0], ['lat', 'lon']) + # file name that holds netcdf for one year with 12 months + filename = filename_list[0] + print('Processing input: {}'.format(filename)) + [c_lats, c_lons] = extract_vars(filename, ['lat', 'lon']) - [data] = extract_vars(filename_list[0], ['sanl1'], - attributes_list=['standard_name', 'units', 'cell_method', 'long_name']) - # TODO: reuse all 12 months - data['data'] = data['data'][month].reshape((1,) + data['data'].shape) + var_key = 'sanl1' # surface + if sector == 'elevated': + var_key = 'sanl2' + + print('Var key is: {}'.format(var_key)) + + [data_12_months] = extract_vars(filename, [var_key], attributes_list=['standard_name', 'units', 'cell_method', 'long_name']) + + for month in range(0, 12): + data = data_12_months.copy() + data['data'] = data_12_months['data'][month].reshape((1,) + data_12_months['data'][month].shape) data['name'] = pollutant global_attributes = { @@ -162,277 +97,16 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) - out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant, year, str(month).zfill(2))) + out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant, year, str(month+1).zfill(2))) write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), - cell_area=get_grid_area(filename_list[month - 1]), global_attributes=global_attributes,) - return True - + cell_area=get_grid_area(filename), global_attributes=global_attributes,) -def do_ratio_list(sector=None): - # TODO Documentation - """ - - :param sector: - :return: - """ - if sector == 'SHIPS': - return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_SHIPS_NAME)} - elif sector == 'AIR_CDS': - return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} - elif sector == 'AIR_CRS': - return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} - elif sector == 'AIR_LTO': - return {'all': os.path.join(VOC_RATIO_PATH, VOC_RATIO_AIR_NAME)} - return { - 'voc01': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '01')), - 'voc02': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '02')), - 'voc03': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '03')), - 'voc04': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '04')), - 'voc05': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '05')), - 'voc06': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '06')), - 'voc07': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '07')), - 'voc08': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '08')), - 'voc09': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '09')), - 'voc12': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '12')), - 'voc13': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '13')), - 'voc14': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '14')), - 'voc15': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '15')), - 'voc16': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '16')), - 'voc17': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '17')), - 'voc18': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '18')), - 'voc19': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '19')), - 'voc20': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '20')), - 'voc21': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '21')), - 'voc22': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '22')), - 'voc23': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '23')), - 'voc24': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '24')), - 'voc25': os.path.join(VOC_RATIO_PATH, VOC_RATIO_NAME.replace('', '25')), - } + print('Saved output to: {}'.format(out_path_aux)) - -def do_nmvoc_month_transformation(filename_list, out_path, sector, year): - # TODO Docuemtnation - """ - - :param filename_list: - :param out_path: - :param sector: - :param year: - :return: - """ - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf - from hermesv3_gr.tools.coordinates_tools import create_bounds - - nmvoc_ratio_list = do_ratio_list() - - print(sector) - if sector == 'ENERGY': - ratio_var = 'pow' - - nmvoc_ratio_list.pop('voc18', None) - nmvoc_ratio_list.pop('voc19', None) - nmvoc_ratio_list.pop('voc20', None) - - elif sector == 'RESIDENTIAL': - ratio_var = 'res' - - nmvoc_ratio_list.pop('voc18', None) - nmvoc_ratio_list.pop('voc20', None) - - elif sector == 'TRANSPORT': - ratio_var = 'tra' - - nmvoc_ratio_list.pop('voc01', None) - nmvoc_ratio_list.pop('voc18', None) - nmvoc_ratio_list.pop('voc19', None) - nmvoc_ratio_list.pop('voc20', None) - nmvoc_ratio_list.pop('voc24', None) - nmvoc_ratio_list.pop('voc25', None) - - print(type(nmvoc_ratio_list), nmvoc_ratio_list) - - for month in range(1, 13): - print(filename_list[month - 1]) - c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) - - [data] = extract_vars(filename_list[month - 1], ['emi_nmvoc']) - - for voc, ratio_file in nmvoc_ratio_list.items(): - print(voc, ratio_file) - - pollutant = voc - [ratio] = extract_vars(ratio_file, [ratio_var]) - - data_aux = data.copy() - data_aux['data'] = data['data'] * ratio['data'] - data_aux['data'] = data_aux['data'].reshape((1,) + data_aux['data'].shape) - data_aux['name'] = voc - data_aux['units'] = 'kg m-2 s-1' - global_attributes = { - 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, pollutant), - 'Conventions': 'CF-1.6', - 'institution': 'European Commission, Joint Research Centre (JRC)', - 'source': 'HTAPv2', - 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + - '2017-04-28: ...', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and ' + - 'global emission grid maps for 2008 and 2010 to study hemispheric transport of air ' + - 'pollution, Atmos. Chem. Phys., 15, 11411-11432, ' + - 'https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + - 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', - 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + - '(Barcelona Supercomputing Center)', - } - - out_path_aux = os.path.join(out_path, pollutant + '_' + sector.lower()) - if not os.path.exists(out_path_aux): - os.makedirs(out_path_aux) - - out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant, year, str(month).zfill(2))) - print(out_path_aux) - write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], - boundary_latitudes=create_bounds(c_lats['data']), - boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) return True -def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year): - # TODO Documentation - """ - - :param filename_list: - :param out_path: - :param sector: - :param year: - :return: - """ - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf - from hermesv3_gr.tools.coordinates_tools import create_bounds - - nmvoc_ratio_list = do_ratio_list() - - print(sector) - - print(type(nmvoc_ratio_list), nmvoc_ratio_list) - - for month in range(1, 13): - print(filename_list[month - 1]) - c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) - - [ind, exf, sol] = extract_vars(filename_list[month - 1], ['emiss_ind', 'emiss_exf', 'emiss_sol']) - - for voc, ratio_file in nmvoc_ratio_list.items(): - print(voc, ratio_file) - data = { - 'name': voc, - 'units': 'kg m-2 s-1', - } - if voc in ['voc02', 'voc03', 'voc04', 'voc05', 'voc07', 'voc08', 'voc12', 'voc13']: - [r_inc, r_exf] = extract_vars(ratio_file, ['inc', 'exf']) - data.update({'data': ind['data'] * r_inc['data'] + exf['data'] * r_exf['data']}) - elif voc in ['voc01', 'voc23', 'voc25']: - [r_inc, r_sol] = extract_vars(ratio_file, ['inc', 'sol']) - data.update({'data': ind['data'] * r_inc['data'] + sol['data'] * r_sol['data']}) - elif voc in ['voc09', 'voc16', 'voc21', 'voc22', 'voc24']: - [r_inc] = extract_vars(ratio_file, ['inc']) - data.update({'data': ind['data'] * r_inc['data']}) - # elif voc in []: - # [r_exf, r_sol] = extract_vars(ratio_file, ['exf', 'sol']) - # data.update({'data': exf['data']*r_exf['data'] + sol['data']*r_sol['data']}) - elif voc in ['voc18', 'voc19', 'voc20']: - [r_sol] = extract_vars(ratio_file, ['sol']) - data.update({'data': sol['data'] * r_sol['data']}) - else: - [r_inc, r_exf, r_sol] = extract_vars(ratio_file, ['inc', 'exf', 'sol']) - data.update({'data': ind['data'] * r_inc['data'] + exf['data'] * r_exf['data'] + - sol['data'] * r_sol['data']}) - - global_attributes = { - 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, voc), - 'Conventions': 'CF-1.6', - 'institution': 'European Commission, Joint Research Centre (JRC)', - 'source': 'HTAPv2', - 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + - '2017-04-28: ...', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and ' + - 'global emission grid maps for 2008 and 2010 to study hemispheric transport of air ' + - 'pollution, Atmos. Chem. Phys., 15, 11411-11432, ' + - 'https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + - 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', - 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + - '(Barcelona Supercomputing Center)', - } - - out_path_aux = os.path.join(out_path, voc + '_industry') - if not os.path.exists(out_path_aux): - os.makedirs(out_path_aux) - - out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) - print(out_path_aux) - write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data], - boundary_latitudes=create_bounds(c_lats['data']), - boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) - - -def do_nmvoc_year_transformation(filename, out_path, sector, year): - # TODO Documentation - """ - - :param filename: - :param out_path: - :param sector: - :param year: - :return: - """ - import pandas as pd - from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf - from hermesv3_gr.tools.coordinates_tools import create_bounds - - nmvoc_ratio_file = do_ratio_list(sector)['all'] - nmvoc_ratio_list = pd.read_csv(nmvoc_ratio_file, sep=';') - - c_lats, c_lons = extract_vars(filename, ['lat', 'lon']) - - [data] = extract_vars(filename, ['emi_nmvoc']) - - for i, voc_ratio in nmvoc_ratio_list.iterrows(): - pollutant = voc_ratio['voc_group'] - ratio = voc_ratio['factor'] - - data_aux = data.copy() - data_aux['data'] = data['data'] * ratio - data_aux['data'] = data_aux['data'].reshape((1,) + data_aux['data'].shape) - data_aux['name'] = pollutant - data_aux['units'] = 'kg m-2 s-1' - global_attributes = { - 'title': 'HTAPv2 inventory for the sector {0} and pollutant {1}'.format(sector, pollutant), - 'Conventions': 'CF-1.6', - 'institution': 'European Commission, Joint Research Centre (JRC)', - 'source': 'HTAPv2', - 'history': 'Re-writing of the HTAPv2 input to follow the CF 1.6 conventions;\n' + - '2017-04-28: ...', - 'references': 'publication: Janssens-Maenhout, G., et al.: HTAP_v2.2: a mosaic of regional and global ' + - 'emission grid maps for 2008 and 2010 to study hemispheric transport of air pollution, ' + - 'Atmos. Chem. Phys., 15, 11411-11432, https://doi.org/10.5194/acp-15-11411-2015, 2015.\n ' + - 'web: http://edgar.jrc.ec.europa.eu/htap_v2/index.php', - 'comment': 'Re-writing done by Carles Tena (carles.tena@bsc.es) from the BSC-CNS ' + - '(Barcelona Supercomputing Center)\n ' + - 'HTAP contact: greet.maenhout@jrc.ec.europa.eu', - } - - out_path_aux = os.path.join(out_path, pollutant + '_' + sector.lower()) - if not os.path.exists(out_path_aux): - os.makedirs(out_path_aux) - - out_path_aux = os.path.join(out_path_aux, '{0}_{1}.nc'.format(pollutant, year)) - print(out_path_aux) - write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], - boundary_latitudes=create_bounds(c_lats['data']), - boundary_longitudes=create_bounds(c_lons['data']), - global_attributes=global_attributes,) - - def get_pollutant_dict(): # TODO Documentation """ @@ -452,8 +126,7 @@ def get_sector_dict(): :return: """ common_dict = { - 'month': ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT'], - 'year': ['SHIPS', 'AIR_CDS', 'AIR_CRS', 'AIR_LTO'] + 'month': ['surface', 'elevated'], } sector_dict = { 'so2': common_dict, @@ -461,92 +134,12 @@ def get_sector_dict(): return sector_dict -def get_nmvoc_sector_dict(): - # TODO Documentation - """ - - :return: - """ - nmvoc_sectors = {'month': ['ENERGY', 'INDUSTRY_3subsectors', 'RESIDENTIAL', 'TRANSPORT'], - 'year': ['SHIPS', 'AIR_CDS', 'AIR_CRS', 'AIR_LTO']} - return nmvoc_sectors - - -def check_vocs(year): - # TODO Documentation - """ - - :param year: - :return: - """ - from hermesv3_gr.tools.netcdf_tools import extract_vars - for month in range(1, 12 + 1, 1): - for snap in ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT']: - nmvoc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', 'nmvoc_{0}'.format(snap.lower()), - 'nmvoc_{0}{1}.nc'.format(year, str(month).zfill(2))) - [new_voc] = extract_vars(nmvoc_path, ['nmvoc']) - nmvoc_sum = new_voc['data'].sum() - - voc_sum = 0 - for voc in ['voc{0}'.format(str(x).zfill(2)) for x in range(1, 25 + 1, 1)]: - voc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(voc, snap.lower()), - '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) - if os.path.exists(voc_path): - [new_voc] = extract_vars(voc_path, [voc]) - voc_sum += new_voc['data'].sum() - - print('{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum) / nmvoc_sum, month)) - - if __name__ == '__main__': for year in LIST_YEARS: for pollutant_dict in get_pollutant_dict().items(): for current_sector in get_sector_dict()[pollutant_dict[0]]['month']: - # TODO: surface & elevated - # input_name_aux = INPUT_NAME.replace('', current_sector) - input_name_aux = INPUT_NAME.replace('', str(year)) + print('Processing sector: {}'.format(current_sector)) + input_name_aux = INPUT_NAME.replace('', current_sector) + input_name_aux = input_name_aux.replace('', str(year)) files = [os.path.join(INPUT_PATH, input_name_aux),] - # TODO: surf or elevated do_transformation(files, os.path.join(OUTPUT_PATH, 'monthly_mean'), pollutant_dict[0], current_sector, year) - # annual inventories - # for current_sector in get_sector_dict()[pollutant_dict[0]]['year']: - # if current_sector[0:3] == 'AIR': - # input_name_aux = INPUT_NAME_AIR - # else: - # input_name_aux = INPUT_NAME_SHIPS - # input_name_aux = input_name_aux.replace('', current_sector) - # input_name_aux = input_name_aux.replace('', str(year)) - # input_name_aux = input_name_aux.replace('', pollutant_dict[1]) - # input_name_aux = os.path.join(INPUT_PATH, input_name_aux) - # - # do_transformation_annual(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean', ), pollutant_dict[0], - # current_sector, year) - - for current_sector in get_nmvoc_sector_dict()['month']: - if current_sector == 'INDUSTRY_3subsectors': - input_name_aux = INPUT_NAME_NMVOC_INDUSTRY - else: - input_name_aux = INPUT_NAME - input_name_aux = input_name_aux.replace('', 'NMVOC') - input_name_aux = input_name_aux.replace('', current_sector) - input_name_aux = input_name_aux.replace('', str(year)) - files = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) - for aux_month in range(1, 13)] - - if current_sector == 'INDUSTRY_3subsectors': - do_nmvoc_industry_month_transformation(files, os.path.join(OUTPUT_PATH, 'monthly_mean'), - current_sector, year) - else: - do_nmvoc_month_transformation(files, os.path.join(OUTPUT_PATH, 'monthly_mean'), current_sector, year) - # for current_sector in get_nmvoc_sector_dict()['year']: - # if current_sector[0:3] == 'AIR': - # input_name_aux = INPUT_NAME_AIR - # else: - # input_name_aux = INPUT_NAME_SHIPS - # input_name_aux = input_name_aux.replace('', 'NMVOC') - # input_name_aux = input_name_aux.replace('', current_sector) - # input_name_aux = input_name_aux.replace('', str(y)) - # input_name_aux = os.path.join(INPUT_PATH, input_name_aux) - # print(input_name_aux) - # do_nmvoc_year_transformation(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean'), current_sector, y) -- GitLab