From 3c57bdd15260f0040af8376e1c28939829614b27 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 21 Nov 2019 14:49:12 +0100 Subject: [PATCH 1/6] Added option to do not fail when find unrecognized arguments --- hermesv3_bu/config/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hermesv3_bu/config/config.py b/hermesv3_bu/config/config.py index c3fccf8..f99e3e4 100755 --- a/hermesv3_bu/config/config.py +++ b/hermesv3_bu/config/config.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from warnings import warn from configargparse import ArgParser import os from mpi4py import MPI @@ -668,7 +669,9 @@ class Config(ArgParser): p.add_argument('--solvents_speciation_profiles', required=False, help="Defines the path to the CSV file that contains the speciation profiles.") - arguments = p.parse_args() + arguments, unknown = p.parse_known_args() + if len(unknown) > 0: + warn("Unrecognized arguments: {0}".format(unknown)) for item in vars(arguments): is_str = isinstance(arguments.__dict__[item], str) -- GitLab From e5ef6cb4ea43331018b58516c56908704e2a3588 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 21 Nov 2019 15:22:45 +0100 Subject: [PATCH 2/6] updating gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2849b44..4ea8663 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .idea .directory -*.pyc \ No newline at end of file +*.pyc +dist +build +hermesv3_bu.egg* \ No newline at end of file -- GitLab From dd349f106d68d0b0ec3f458443d3c24a152014d0 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 5 Dec 2019 10:21:47 +0100 Subject: [PATCH 3/6] Drop unuser library from dependencies --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 9524d79..6d9ebe9 100755 --- a/setup.py +++ b/setup.py @@ -38,7 +38,6 @@ setup( 'pytz', 'timezonefinder', 'mpi4py', - 'pytest', 'shapely', 'rasterio', ], -- GitLab From e86c5ef04637d9bf69512e3353418b69c0713ca4 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 19 Dec 2019 13:19:50 +0100 Subject: [PATCH 4/6] Solvents: solved bug on low processors run (timezonefinder) --- hermesv3_bu/sectors/solvents_sector.py | 51 +++++++++++++++++++++----- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/hermesv3_bu/sectors/solvents_sector.py b/hermesv3_bu/sectors/solvents_sector.py index 8468fb7..07786ef 100755 --- a/hermesv3_bu/sectors/solvents_sector.py +++ b/hermesv3_bu/sectors/solvents_sector.py @@ -583,20 +583,53 @@ class SolventsSector(Sector): proxies = GeoDataFrame( proxies, geometry=self.grid.shapefile.loc[proxies.index.get_level_values('FID'), 'geometry'].values, crs=self.grid.shapefile.crs) - IoShapefile(self.comm).write_shapefile_serial(proxies.reset_index(), proxy_path) + # IoShapefile(self.comm).write_shapefile_serial(proxies.reset_index(), proxy_path) else: proxies = None + proxies = IoShapefile(self.comm).split_shapefile(proxies) + proxies = self.add_timezone(proxies) + IoShapefile(self.comm).write_shapefile_parallel(proxies.reset_index(), proxy_path) + else: - if self.comm.Get_rank() == 0: - proxies = IoShapefile(self.comm).read_shapefile_serial(proxy_path) - proxies.set_index(['FID', 'nut_code'], inplace=True) - else: - proxies = None - proxies = IoShapefile(self.comm).split_shapefile(proxies) + proxies = IoShapefile(self.comm).read_shapefile_parallel(proxy_path) + + proxies.set_index(['FID', 'nut_code'], inplace=True) self.logger.write_time_log('SolventsSector', 'get_proxy_shapefile', timeit.default_timer() - spent_time) return proxies + def add_dates(self, dataframe, drop_utc=True): + """ + Add the 'date' and 'tstep' column to the dataframe. + + The dataframe will be replicated as many times as time steps to calculate. + + :param dataframe: Geodataframe to be extended with the dates. + :type dataframe: GeoDataFrame + + :return: Geodataframe with the dates. The length of the new dataframe is the length of the input dataframe + multiplied by the number of time steps. + :rtype: GeoDataFrame + """ + spent_time = timeit.default_timer() + # dataframe = self.add_timezone(dataframe) + df_list = [] + + for tstep, date in enumerate(self.date_array): + df_aux = dataframe.copy() + df_aux['date'] = pd.to_datetime(date, utc=True) + df_aux['date_utc'] = pd.to_datetime(date, utc=True) + df_aux['tstep'] = tstep + # df_aux = self.to_timezone(df_aux) + df_list.append(df_aux) + dataframe = pd.concat(df_list, ignore_index=True) + dataframe = self.to_timezone(dataframe) + if drop_utc: + dataframe.drop('date_utc', axis=1, inplace=True) + self.logger.write_time_log('Sector', 'add_dates', timeit.default_timer() - spent_time) + + return dataframe + def calculate_hourly_emissions(self, yearly_emissions): """ Disaggrate to hourly level the yearly emissions. @@ -629,7 +662,7 @@ class SolventsSector(Sector): spent_time = timeit.default_timer() self.logger.write_log('\tHourly disaggregation', message_level=2) - emissions = self.add_dates(yearly_emissions, drop_utc=True) + emissions = self.add_dates(yearly_emissions.reset_index(), drop_utc=True) emissions['month'] = emissions['date'].dt.month emissions['weekday'] = emissions['date'].dt.weekday @@ -682,7 +715,7 @@ class SolventsSector(Sector): self.proxies_map.loc[snap, 'proxy_name']], axis=1) emis.set_index(['FID', 'snap'], inplace=True) - emis_list.append(emis[['P_month', 'P_week', 'P_hour', 'P_spec', 'nmvoc', 'geometry']]) + emis_list.append(emis[['P_month', 'P_week', 'P_hour', 'P_spec', 'nmvoc', 'geometry', 'timezone']]) emis = pd.concat(emis_list).sort_index() emis = emis[emis['nmvoc'] > 0] -- GitLab From a24b133a01027139c6d0349fb57c8960ba1654b9 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 19 Dec 2019 13:28:09 +0100 Subject: [PATCH 5/6] Solvents: solved bug on low processors run (timezonefinder) --- hermesv3_bu/sectors/solvents_sector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hermesv3_bu/sectors/solvents_sector.py b/hermesv3_bu/sectors/solvents_sector.py index 07786ef..e3afac6 100755 --- a/hermesv3_bu/sectors/solvents_sector.py +++ b/hermesv3_bu/sectors/solvents_sector.py @@ -776,5 +776,6 @@ class SolventsSector(Sector): emissions['layer'] = 0 emissions = emissions.groupby(['FID', 'layer', 'tstep']).sum() + self.logger.write_log('\tSolvents emissions calculated', message_level=2) self.logger.write_time_log('SolventsSector', 'calculate_emissions', timeit.default_timer() - spent_time) return emissions -- GitLab From 9be9c31f387cf661357ecf905b406ce285675cb2 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 7 Jan 2020 18:46:58 +0100 Subject: [PATCH 6/6] Upgrading version --- hermesv3_bu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_bu/__init__.py b/hermesv3_bu/__init__.py index 485f44a..b3f4756 100755 --- a/hermesv3_bu/__init__.py +++ b/hermesv3_bu/__init__.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.1.2" -- GitLab