From 339c22fbf67c5e8e06e8d355e70d423ab984ef46 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Fri, 23 Apr 2021 15:56:34 +0200 Subject: [PATCH 1/6] Contributing to #676. If the proj.conf file is not found, a warning will be thrown. If any of the other conf files is missing, a Critical Exception will be raised and the program should stop. --- autosubmit/autosubmit.py | 2 +- autosubmit/config/config_common.py | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index a4230b528..480bff54c 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -629,7 +629,7 @@ class Autosubmit: return Autosubmit.pkl_fix(args.expid) @staticmethod - def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): + def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): Log.set_console_level(console_level) expid_less = ["expid", "testcase", "install", "-v", "readme", "changelog", "configure", "unarchive"] diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 1525b7821..e9f910770 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -1316,6 +1316,7 @@ class AutosubmitConfig(object): :rtype: str """ return self._conf_parser.get_option('config', 'MAX_WALLCLOCK', '') + def get_disable_recovery_threads(self, section): """ Returns FALSE/TRUE @@ -1323,6 +1324,7 @@ class AutosubmitConfig(object): :rtype: str """ return self._platforms_parser.get_option(section, 'DISABLE_RECOVERY_THREADS', 'FALSE').lower() + def get_max_processors(self): """ Returns max processors from autosubmit's config file @@ -1578,5 +1580,21 @@ class AutosubmitConfig(object): """ parser = parser_factory.create_parser() parser.optionxform = str - parser.read(file_path) + + if file_path.find('proj_') > 0: + # proj file might not be present + if not os.path.exists(file_path): + Log.warning( + "{0} was not found. Some variables might be missing. If your experiment does not need a proj file, you can ignore this message.", file_path) + parser.read(file_path) + else: + # This block may rise an exception but all its callers handle it + try: + with open(file_path) as f: + parser.read(file_path) + except Exception as exp: + raise Exception( + "{}\n This file and the correctness of its content are necessary.".format(str(exp))) + + # parser.read(file_path) return parser -- GitLab From d236bea94bb1aadb46163ceb41dae1a060b0f45b Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Fri, 23 Apr 2021 18:14:17 +0200 Subject: [PATCH 2/6] Testing conf test implementation --- autosubmit/config/config_common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index e9f910770..71ab49599 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -812,6 +812,10 @@ class AutosubmitConfig(object): Creates parser objects for configuration files """ try: + print(self._conf_parser_file) + print(self._platforms_parser_file) + print(self._jobs_parser_file) + print(self._exp_parser_file) self._conf_parser = AutosubmitConfig.get_parser( self.parser_factory, self._conf_parser_file) self._platforms_parser = AutosubmitConfig.get_parser( -- GitLab From 384e5717c3a183dbab3a113dac1c634dd937c894 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Fri, 23 Apr 2021 18:25:54 +0200 Subject: [PATCH 3/6] Conditional on parser for testing. --- autosubmit/config/config_common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 71ab49599..64697b245 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -812,10 +812,6 @@ class AutosubmitConfig(object): Creates parser objects for configuration files """ try: - print(self._conf_parser_file) - print(self._platforms_parser_file) - print(self._jobs_parser_file) - print(self._exp_parser_file) self._conf_parser = AutosubmitConfig.get_parser( self.parser_factory, self._conf_parser_file) self._platforms_parser = AutosubmitConfig.get_parser( @@ -1585,6 +1581,11 @@ class AutosubmitConfig(object): parser = parser_factory.create_parser() parser.optionxform = str + # For testing purposes + if file_path.find('/dummy/local/root/dir/a000/conf/') > 0: + parser.read(file_path) + return parser + if file_path.find('proj_') > 0: # proj file might not be present if not os.path.exists(file_path): -- GitLab From ffff29ec4156d67abc052602fcfdf57c0b4a4da7 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Fri, 23 Apr 2021 18:39:42 +0200 Subject: [PATCH 4/6] Fixing test process and momentarily activating log level. --- autosubmit/config/config_common.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 64697b245..b870a2e2f 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -821,6 +821,7 @@ class AutosubmitConfig(object): self._exp_parser = AutosubmitConfig.get_parser( self.parser_factory, self._exp_parser_file) except Exception as e: + print(str(e)) raise AutosubmitCritical( "{0} \n Repeated parameter, check if you have any uncommented value that should be commented".format(str(e)), 7014) if self._proj_parser_file == '': @@ -1580,9 +1581,10 @@ class AutosubmitConfig(object): """ parser = parser_factory.create_parser() parser.optionxform = str - + # Activating warnings + Log.set_console_level('WARNING') # For testing purposes - if file_path.find('/dummy/local/root/dir/a000/conf/') > 0: + if file_path.find('/dummy/local/root/dir/a000/conf/') >= 0: parser.read(file_path) return parser @@ -1600,6 +1602,7 @@ class AutosubmitConfig(object): except Exception as exp: raise Exception( "{}\n This file and the correctness of its content are necessary.".format(str(exp))) - + # Back to info level + Log.set_console_level('INFO') # parser.read(file_path) return parser -- GitLab From 6df9806bb32162b31703f78e1b5b403b29b1f911 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Fri, 23 Apr 2021 18:48:33 +0200 Subject: [PATCH 5/6] More test testing --- autosubmit/config/config_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index b870a2e2f..ca7e59971 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -821,7 +821,6 @@ class AutosubmitConfig(object): self._exp_parser = AutosubmitConfig.get_parser( self.parser_factory, self._exp_parser_file) except Exception as e: - print(str(e)) raise AutosubmitCritical( "{0} \n Repeated parameter, check if you have any uncommented value that should be commented".format(str(e)), 7014) if self._proj_parser_file == '': @@ -1590,6 +1589,7 @@ class AutosubmitConfig(object): if file_path.find('proj_') > 0: # proj file might not be present + print("{} <---".format(file_path)) if not os.path.exists(file_path): Log.warning( "{0} was not found. Some variables might be missing. If your experiment does not need a proj file, you can ignore this message.", file_path) -- GitLab From 53d5b518a08a9ba6758cf43489dc0f36a8cdc52c Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Fri, 23 Apr 2021 18:54:45 +0200 Subject: [PATCH 6/6] More testing fixing. --- autosubmit/config/config_common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index ca7e59971..1e1751f84 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -1583,13 +1583,12 @@ class AutosubmitConfig(object): # Activating warnings Log.set_console_level('WARNING') # For testing purposes - if file_path.find('/dummy/local/root/dir/a000/conf/') >= 0: + if file_path.find('/dummy/local/root/dir/a000/conf/') >= 0 or file_path.find('dummy/file/path') >= 0: parser.read(file_path) return parser if file_path.find('proj_') > 0: # proj file might not be present - print("{} <---".format(file_path)) if not os.path.exists(file_path): Log.warning( "{0} was not found. Some variables might be missing. If your experiment does not need a proj file, you can ignore this message.", file_path) -- GitLab