From 084dba5255d9e31f38833d011e12968db7037138 Mon Sep 17 00:00:00 2001 From: ctena Date: Thu, 2 Sep 2021 09:47:09 +0200 Subject: [PATCH 1/3] WIP uchile preproc --- preproc/uchile_inema_preproc.py | 113 ++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100755 preproc/uchile_inema_preproc.py diff --git a/preproc/uchile_inema_preproc.py b/preproc/uchile_inema_preproc.py new file mode 100755 index 0000000..e64760c --- /dev/null +++ b/preproc/uchile_inema_preproc.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python + +# Copyright 2018 Earth Sciences Department, BSC-CNS +# +# This file is part of HERMESv3_GR. +# +# HERMESv3_GR is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# HERMESv3_GR is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with HERMESv3_GR. If not, see . + + +import os +import timeit +from netCDF4 import Dataset + +# ============== README ====================== +""" +downloading website: ¿¿¿??? +reference: ¿¿¿??? +Besides citing HERMESv3_GR, users must also acknowledge the use of the corresponding emission inventories in their works +""" + +# ============== CONFIGURATION PARAMETERS ====================== +INPUT_PATH = '/esarchive/recon/uchile/inema_v1/original_files///inema___0.01x0.01_.nc' +OUTPUT_PATH = '/esarchive/recon/uchile/inema_v1/' +SECTOR_LIST = {'Transport': None, + 'Energy': None, + 'Industry': None, + 'Mining': None, + 'Residential': ['Residential', 'Rural', 'Urban']} +POLLUTANT_INFO = {'bc': 'BC', 'c6h6': 'C6H6', 'ch4': 'CH4', 'co2': 'CO2', 'co': 'CO', 'hg': 'Hg', 'nh3': 'NH3', + 'nmvoc': 'NMVOC', 'no': 'NO', 'no2': 'NO2', 'nox': 'NOx', 'pb': 'Pb', 'pcddf': 'PCDDF', + 'pm10': 'PM10', 'pm25': 'PM25', 'pm': 'PM', 'so2': 'SO2', 'tol': 'Toluene', 'voc': 'VOC'} + +YEAR_LIST = [2017] +# ============================================================== + + +def do_transformation(): + from hermesv3_gr.tools.netcdf_tools import write_netcdf, get_grid_area + from hermesv3_gr.tools.coordinates_tools import create_bounds + import calendar + import numpy as np + from datetime import datetime + + lats = lons = blats = blons = cell_area = None + + for sector in SECTOR_LIST.keys(): + if SECTOR_LIST[sector] is None: + subsector_list = [sector] + else: + subsector_list = SECTOR_LIST[sector] + for subsector in subsector_list: + for year in YEAR_LIST: + for pollutant in POLLUTANT_INFO.keys(): + out_file_name = os.path.join(OUTPUT_PATH, 'yearly_mean', + '{0}_{1}'.format(pollutant, subsector), + "{0}_{1}.nc".format(pollutant, year)) + file_name = INPUT_PATH.replace('', str(year)).replace('', sector).replace( + '', POLLUTANT_INFO[pollutant]).replace('', subsector) + + netcdf = Dataset(file_name, mode='r') + if lats is None: + lat_nc = netcdf.variables['lat'][:] + lon_nc = netcdf.variables['lon'][:] + lat_inc = round(lat_nc[1] - lat_nc[0], 5) + lon_inc = round(lon_nc[1] - lon_nc[0], 5) + + lats = np.linspace(lat_nc[0], lat_nc[0] + (lat_inc * (len(lat_nc) - 1)), len(lat_nc), + dtype=np.float) + lons = np.linspace(lon_nc[0], lon_nc[0] + (lon_inc * (len(lon_nc) - 1)), len(lon_nc), + dtype=np.float) + del lat_nc, lon_nc + blats = create_bounds(lats) + blons = create_bounds(lons) + cell_area = get_grid_area(file_name) + + var = netcdf.variables[POLLUTANT_INFO[pollutant]][:] + print(var) + + # if calendar.isleap(year): + # days = 366 + # else: + # days = 365 + # + # # from ton/yr to kg/m2.s + # var = (var / cell_area) * (1000 / (days * 24 * 60 * 60)) + # + # data = [{'name': pollutant, 'units': 'kg.m-2.s-1', 'data': var.reshape((1,) + var.shape)}] + # + # if not os.path.exists(os.path.dirname(out_file_name)): + # os.makedirs(os.path.dirname(out_file_name)) + # print(out_file_name) + # write_netcdf(out_file_name, lats, lons, data, date=datetime(year=year, month=1, day=1), + # boundary_latitudes=blats, boundary_longitudes=blons, cell_area=cell_area, + # regular_latlon=True) + + +if __name__ == '__main__': + starting_time = timeit.default_timer() + + do_transformation() + + print('Time(s):', timeit.default_timer() - starting_time) -- GitLab From f2d65586acbb512859e18d6cad8c36ba9ebed1fb Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 2 Sep 2021 11:13:43 +0200 Subject: [PATCH 2/3] Testing uChule emission inventory --- preproc/uchile_inema_preproc.py | 76 +++++++++++++++++---------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/preproc/uchile_inema_preproc.py b/preproc/uchile_inema_preproc.py index e64760c..a6b8bb1 100755 --- a/preproc/uchile_inema_preproc.py +++ b/preproc/uchile_inema_preproc.py @@ -31,7 +31,7 @@ Besides citing HERMESv3_GR, users must also acknowledge the use of the correspon # ============== CONFIGURATION PARAMETERS ====================== INPUT_PATH = '/esarchive/recon/uchile/inema_v1/original_files///inema___0.01x0.01_.nc' -OUTPUT_PATH = '/esarchive/recon/uchile/inema_v1/' +OUTPUT_PATH = '/esarchive/scratch/ctena/uchile/inema_v1/' SECTOR_LIST = {'Transport': None, 'Energy': None, 'Industry': None, @@ -67,42 +67,44 @@ def do_transformation(): "{0}_{1}.nc".format(pollutant, year)) file_name = INPUT_PATH.replace('', str(year)).replace('', sector).replace( '', POLLUTANT_INFO[pollutant]).replace('', subsector) - - netcdf = Dataset(file_name, mode='r') - if lats is None: - lat_nc = netcdf.variables['lat'][:] - lon_nc = netcdf.variables['lon'][:] - lat_inc = round(lat_nc[1] - lat_nc[0], 5) - lon_inc = round(lon_nc[1] - lon_nc[0], 5) - - lats = np.linspace(lat_nc[0], lat_nc[0] + (lat_inc * (len(lat_nc) - 1)), len(lat_nc), - dtype=np.float) - lons = np.linspace(lon_nc[0], lon_nc[0] + (lon_inc * (len(lon_nc) - 1)), len(lon_nc), - dtype=np.float) - del lat_nc, lon_nc - blats = create_bounds(lats) - blons = create_bounds(lons) - cell_area = get_grid_area(file_name) - - var = netcdf.variables[POLLUTANT_INFO[pollutant]][:] - print(var) - - # if calendar.isleap(year): - # days = 366 - # else: - # days = 365 - # - # # from ton/yr to kg/m2.s - # var = (var / cell_area) * (1000 / (days * 24 * 60 * 60)) - # - # data = [{'name': pollutant, 'units': 'kg.m-2.s-1', 'data': var.reshape((1,) + var.shape)}] - # - # if not os.path.exists(os.path.dirname(out_file_name)): - # os.makedirs(os.path.dirname(out_file_name)) - # print(out_file_name) - # write_netcdf(out_file_name, lats, lons, data, date=datetime(year=year, month=1, day=1), - # boundary_latitudes=blats, boundary_longitudes=blons, cell_area=cell_area, - # regular_latlon=True) + try: + netcdf = Dataset(file_name, mode='r') + if lats is None: + lat_nc = netcdf.variables['lat'][:] + lon_nc = netcdf.variables['lon'][:] + lat_inc = round(lat_nc[1] - lat_nc[0], 5) + lon_inc = round(lon_nc[1] - lon_nc[0], 5) + + lats = np.linspace(lat_nc[0], lat_nc[0] + (lat_inc * (len(lat_nc) - 1)), len(lat_nc), + dtype=np.float) + lons = np.linspace(lon_nc[0], lon_nc[0] + (lon_inc * (len(lon_nc) - 1)), len(lon_nc), + dtype=np.float) + del lat_nc, lon_nc + blats = create_bounds(lats) + blons = create_bounds(lons) + + var = netcdf.variables[POLLUTANT_INFO[pollutant]][:] + var = var.data + var[var == netcdf.variables[POLLUTANT_INFO[pollutant]].getncattr('_FillValue')] = 0 + + if calendar.isleap(year): + days = 366 + else: + days = 365 + + # from ktonne/km2 year to kg/m2.s + var = var * (1000000 / (days * 24 * 60 * 60 * 1000000)) + + data = [{'name': pollutant, 'units': 'kg.m-2.s-1', 'data': var.reshape((1,) + var.shape)}] + + if not os.path.exists(os.path.dirname(out_file_name)): + os.makedirs(os.path.dirname(out_file_name)) + print(out_file_name) + write_netcdf(out_file_name, lats, lons, data, date=datetime(year=year, month=1, day=1), + boundary_latitudes=blats, boundary_longitudes=blons, cell_area=cell_area, + regular_latlon=True) + except FileNotFoundError: + print("Variable {0} not found in {1}: {2}".format(pollutant, subsector, file_name)) if __name__ == '__main__': -- GitLab From b153ad4f7a2f198ea1c66f81f0c6109794668105 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Fri, 3 Sep 2021 10:14:58 +0200 Subject: [PATCH 3/3] done uChule emission inventory --- preproc/uchile_inema_preproc.py | 43 ++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/preproc/uchile_inema_preproc.py b/preproc/uchile_inema_preproc.py index a6b8bb1..a57068b 100755 --- a/preproc/uchile_inema_preproc.py +++ b/preproc/uchile_inema_preproc.py @@ -31,29 +31,27 @@ Besides citing HERMESv3_GR, users must also acknowledge the use of the correspon # ============== CONFIGURATION PARAMETERS ====================== INPUT_PATH = '/esarchive/recon/uchile/inema_v1/original_files///inema___0.01x0.01_.nc' -OUTPUT_PATH = '/esarchive/scratch/ctena/uchile/inema_v1/' +OUTPUT_PATH = '/esarchive/recon/uchile/inema_v1/' SECTOR_LIST = {'Transport': None, 'Energy': None, 'Industry': None, 'Mining': None, 'Residential': ['Residential', 'Rural', 'Urban']} POLLUTANT_INFO = {'bc': 'BC', 'c6h6': 'C6H6', 'ch4': 'CH4', 'co2': 'CO2', 'co': 'CO', 'hg': 'Hg', 'nh3': 'NH3', - 'nmvoc': 'NMVOC', 'no': 'NO', 'no2': 'NO2', 'nox': 'NOx', 'pb': 'Pb', 'pcddf': 'PCDDF', + 'nmvoc': 'NMVOC', 'no': 'NO', 'no2': 'NO2', 'nox_no2': 'NOx', 'pb': 'Pb', 'pcddf': 'PCDDF', 'pm10': 'PM10', 'pm25': 'PM25', 'pm': 'PM', 'so2': 'SO2', 'tol': 'Toluene', 'voc': 'VOC'} -YEAR_LIST = [2017] +YEAR_LIST = [2016, 2017] # ============================================================== def do_transformation(): - from hermesv3_gr.tools.netcdf_tools import write_netcdf, get_grid_area + from hermesv3_gr.tools.netcdf_tools import write_netcdf from hermesv3_gr.tools.coordinates_tools import create_bounds import calendar import numpy as np from datetime import datetime - lats = lons = blats = blons = cell_area = None - for sector in SECTOR_LIST.keys(): if SECTOR_LIST[sector] is None: subsector_list = [sector] @@ -69,23 +67,30 @@ def do_transformation(): '', POLLUTANT_INFO[pollutant]).replace('', subsector) try: netcdf = Dataset(file_name, mode='r') - if lats is None: - lat_nc = netcdf.variables['lat'][:] - lon_nc = netcdf.variables['lon'][:] - lat_inc = round(lat_nc[1] - lat_nc[0], 5) - lon_inc = round(lon_nc[1] - lon_nc[0], 5) - - lats = np.linspace(lat_nc[0], lat_nc[0] + (lat_inc * (len(lat_nc) - 1)), len(lat_nc), - dtype=np.float) - lons = np.linspace(lon_nc[0], lon_nc[0] + (lon_inc * (len(lon_nc) - 1)), len(lon_nc), - dtype=np.float) - del lat_nc, lon_nc - blats = create_bounds(lats) - blons = create_bounds(lons) + lat_nc = netcdf.variables['lat'][:] + lon_nc = netcdf.variables['lon'][:] + lat_inc = round(lat_nc[1] - lat_nc[0], 5) + lon_inc = round(lon_nc[1] - lon_nc[0], 5) + + lats = np.linspace(lat_nc[0], lat_nc[0] + (lat_inc * (len(lat_nc) - 1)), len(lat_nc), + dtype=np.float) + lons = np.linspace(lon_nc[0], lon_nc[0] + (lon_inc * (len(lon_nc) - 1)), len(lon_nc), + dtype=np.float) + if lons[-1] < lons[0]: + flip = True + lons = np.flip(lons) + else: + flip = False + + del lat_nc, lon_nc + blats = create_bounds(lats) + blons = create_bounds(lons) var = netcdf.variables[POLLUTANT_INFO[pollutant]][:] var = var.data var[var == netcdf.variables[POLLUTANT_INFO[pollutant]].getncattr('_FillValue')] = 0 + if flip: + var = np.flip(var, axis=1) if calendar.isleap(year): days = 366 -- GitLab