Commit ec4ee6eb authored by erodrigu's avatar erodrigu
Browse files

Merge develop

parents a4f649ba cd939da3
from AutoRPE.UtilsRPE.BasicStructures import Interface
from AutoRPE.UtilsRPE.SourceManager import load_variable_list, load_vault, expand_sources
from os.path import basename
import re
import AutoRPE.UtilsRPE.ModifyCodeUtils as ModifyCodeUtils
import AutoRPE.UtilsRPE.Vault as Vault
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
# Script that automatically creates different versions of a routine with different precisions
def fix_casts(source_path, vault_path, list_filename=None):
vault = load_vault(vault_path)
pattern = r"call.*\b(real\(.*)"
cast_pattern = r"\b(real\(.*)"
pattern_precision = r"\b([wds]p|[8]) *\)"
call_pattern = r"call *\b(\w*)\b"
def get_command_line_arguments():
import argparse
from AutoRPE.UtilsRPE.Error import PathType
parser = argparse.ArgumentParser()
parser.add_argument("--preprocessed-input-files", required=True, type=PathType(exists=True, type="dir"), nargs='+',
help="Path to the preprocessed sources")
parser.add_argument("--original-input-files", required=True, type=PathType(exists=True, type="dir"), nargs='+',
help="Path to the non preprocessed sources")
parser.add_argument("--vault", default=None, type=PathType(exists=True, type="file"),
help="For debug purposes")
lookup_table = {52: "dp", 23: "sp"}
if list_filename is not None:
# Update vault precisions with variable list
variables_to_update = load_variable_list(list_filename)
for variable in vault.variables:
if variable.id in variables_to_update:
variable.precision = 52
file_paths = expand_sources(source_path=source_path)
for file_path in file_paths:
if not (file_path.count(".F90") or file_path.count(".f90") or file_path.count(".h90")):
continue
file_modified = False
module_name = basename(file_path).split(".")[0]
with open(file_path, "r", encoding="latin-1") as f:
lines = [l for l in f]
for line_index, line in enumerate(lines):
# Check if there's a routine call with a cast to real
if re.search(pattern, line, re.I):
try:
calls = call_to_subroutine(line)
except ValueError:
continue
for call in calls:
arguments = find_call_arguments(call)
procedure_name = re.search(call_pattern, call, re.I).group(1)
try:
procedure = vault.get_procedure_by_name(procedure_name, module_name)
except ProcedureNotFound:
continue
# If it is an interface, find the specific procedure being used
if isinstance(procedure, Interface):
try:
procedure = match_arguments_and_interface(arguments, interface=procedure, vault=vault)
except AssertionError:
procedure = None
for index, argument in enumerate(arguments):
if re.search(cast_pattern, argument, re.I):
if procedure is not None:
if not is_keyword_argument(argument):
target_precision = procedure.get_variable_by_position(index).precision
else:
key = argument.split("=")[0].strip()
target_precision = procedure.get_variable_by_name(key).precision
else:
# For now we will assume that in this case it is not necessary to change the cast
continue
# Problematic to assume that the precision of an external procedure is either double or single
# target_precision = 52
target_precision = 23
try:
target_precision = lookup_table[target_precision]
except KeyError:
raise KeyError("target_precision %s not found in lookup_table", target_precision)
cast = close_brackets(re.search(cast_pattern, argument, re.I).group(1))
precision = re.search(pattern_precision, cast, re.I).group(1)
if precision.strip() == target_precision:
continue
else:
cast_replacement = re.sub(r"(.*\b)[wds]p( *\).*)", r"\1%s\2" % target_precision,
cast, re.I)
line = line.replace(cast, cast_replacement)
lines[line_index] = line
file_modified = True
if file_modified:
text = "".join(lines).strip()
with open(file_path, "w") as f:
f.write(text)
return parser.parse_args()
if __name__ == "__main__":
# Getting Command line options
from optparse import OptionParser
help_text = """
"""
parser = OptionParser(usage=help_text)
parser.add_option("-l", "--variable-list", dest="variable_list",
help="Path to the filename with the list of variables.", default=None, metavar="FILE")
parser.add_option("-v", "--vault", dest="vault", default=None, metavar="FILE")
parser.add_option("-s", "--input-source", dest="input_source", default=None, metavar="FILE")
(options, args) = parser.parse_args()
vault_path = options.vault
source_path = options.input_source
list_filename = options.variable_list
if source_path is None or vault_path is None:
print("\nArgument -s/--input-source and -v/--vault are mandatory\n")
parser.print_help()
exit(1)
fix_casts(source_path, vault_path, list_filename)
args = get_command_line_arguments()
preprocessed_input_files = args.preprocessed_input_files
original_input_files = args.original_input_files
vault = args.vault
# Set working precision to double, since it is the precision used to compile preprocessed sources
VariablePrecision.set_working_precision('dp')
if vault is None:
vault = Vault.create_database(preprocessed_input_files, extension=".f90")
else:
# This is useful when debugging: it is not needed to generate a vault at every run
vault = Vault.load_vault(vault)
# Set working precision to single, so as to find the needed fixes
VariablePrecision.set_working_precision('sp')
# Search in preprocessed code where fixes are due
dictionary_of_fixes = ModifyCodeUtils.get_fixes(vault)
# Retrieve these calls in the original code, and fix them
fixed_modules = ModifyCodeUtils.add_cast(original_input_files, dictionary_of_fixes)
# Save fixed sources
for module in fixed_modules:
module.save_file()
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions
import AutoRPE.UtilsRPE.SourceManager as SourceManager
import AutoRPE.UtilsRPE.ModifyCodeUtils as ModifyCodeUtils
import AutoRPE.UtilsRPE.Vault as Vault
......@@ -22,9 +21,18 @@ def change_code(sources_path, var_to_be_changed, target_precision):
ModifyCodeUtils.update_precision_declaration(variable, modules, target_precision)
print("Updated precision of var ", variable.name, index)
# # Save file to disk overwriting originals (this way preserve the folder structure, too difficult to reproduce)
# for module in modules:
# module.save_file()
# Save file to disk overwriting originals (this way preserve the folder structure, too difficult to reproduce)
for module in modules:
module.save_file()
def load_variable_list(filename):
# Load list of variables
from numpy import loadtxt
list_of_variables = loadtxt(filename, dtype=int).tolist()
if not isinstance(list_of_variables, list):
list_of_variables = [list_of_variables]
return list_of_variables
def get_command_line_arguments():
......@@ -59,7 +67,7 @@ if __name__ == "__main__":
VariablePrecision.set_working_precision(working_precision)
# Load list of variable ids
variable_id = SourceManager.load_variable_list(variable_list)
variable_id = load_variable_list(variable_list)
vault = Vault.load_vault(vault_path)
# Get variables from their ids
......@@ -70,18 +78,17 @@ if __name__ == "__main__":
variables += used_var_in_ext
# Propagate dependencies of the variables using same_as property
dependent_var = []
for var in variables:
for additional_var in var.same_as:
# Only if the variable has an id ban it
if additional_var.id:
dependent_var.append(additional_var)
if additional_var not in variables:
variables.append(additional_var)
# Keep unique values
all_variables = list(set(variables + dependent_var))
all_variables = list(set(variables))
# This is for debugging purposes; from a run to the next we want errors to happen always in the same order
all_variables.sort(key=lambda x: x.id)
all_variables.sort(key=lambda x: x.incremental_number)
# Change precision of variables in source code
change_code(input_files, all_variables, variable_precision)
############################################
# Purpose:
# This script is indented to identify which variables are used in calls to external subprogram.
#
# Inputs:
#
# -s, --source : Path to the preprocessed sources.
# -v, --vault : Path to the vault.
# -o, --output : Output filename for the list of variables used in external calls.
#
############################################
from AutoRPE.UtilsRPE.SourceManager import expand_sources, load_vault, store_variable_list
from AutoRPE.UtilsRPE.BasicStructures import SubRoutine, Interface
from AutoRPE.UtilsRPE.CallManager import call_to_subroutine, find_called_subroutine, find_call_arguments
from AutoRPE.UtilsRPE.Getter import get_variable
from AutoRPE.UtilsRPE.Preprocessor import Preprocessor
import AutoRPE.UtilsRPE.CurrentBlock as CurrentBlock
from AutoRPE.UtilsRPE.Error import ProcedureNotFound
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
from os.path import basename, isfile, isdir
def get_command_line_arguments():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--source",
help="Path to the source that will be modified.", default=None)
parser.add_argument("-v", "--vault",
help="Path to the vault.", default=None, metavar="FILE")
parser.add_argument("-o", "--output",
help="Output filename", default=None, metavar="FILE")
args = parser.parse_args()
if args.source_path is None:
parser.error("Argument -s/--source required!")
if args.vault_path is not None:
parser.error("Argument -v/--vault required!")
if not isdir(args.source_path):
parser.error("Argument -s/--source %s is not a valid directory." % args.source)
if not isfile(args.vault):
parser.error("Argument -v/--vault %s is not a file." % args.vault)
return args
def find_variables_used_in_external_calls(files, vault, exceptions):
external_subprogram_found = []
variables = []
for filepath in files:
module_name = basename(filepath).split(".")[0]
module = vault[module_name]
if module is None:
continue
file_info = Preprocessor(filepath)
file_info.pre_process()
current_block = CurrentBlock.CurrentBlock(module.main)
for index, line in enumerate(file_info.lines):
print("\r %04i/%04i %50s" % (index + 1, len(file_info.lines), module_name), end="")
try:
subroutine_calls = call_to_subroutine(line)
except ValueError:
continue
if subroutine_calls:
for call in subroutine_calls:
is_external = False
called_subroutine = find_called_subroutine(call)
# Find the procedure called (object)
try:
subroutine = vault.get_procedure_by_name(called_subroutine, current_block.module_name)
except ProcedureNotFound:
subroutine = None
pass
if isinstance(subroutine, SubRoutine):
continue
elif isinstance(subroutine, Interface):
continue
else:
is_external = True
if is_external:
external_subprogram_found.append(called_subroutine)
# Starting with the exceptions!"
is_exception = False
for exception in exceptions:
if called_subroutine.count(exception):
is_exception = True
break
if is_exception:
continue
called_arguments = find_call_arguments(call)
if not called_arguments:
# If there's no argument, continue
continue
else:
for argument in called_arguments:
try:
variable = get_variable(argument, procedure=current_block.procedure, vault=vault)
except AssertionError:
continue
except ValueError:
continue
if variable is not None:
if variable.type in VariablePrecision.real_types:
variables.append(variable.id)
else:
current_block.in_which_block(line, vault=vault)
variables = list(set(variables))
return variables
if __name__ == "__main__":
args = get_command_line_arguments()
source_path = args.source
vault_path = args.vault
output_filename = args.output
exceptions = ["mpi_", "xios_", "agrif_", "cice", "ice_", "wheneq", "trc_", "flush"]
vault = load_vault(vault_path)
files = expand_sources(source_path)
variables = find_variables_used_in_external_calls(files, vault, exceptions)
store_variable_list(variables, output_filename, vault)
############################################
# Purpose:
# This script takes a list of variables and outputs all the additional variables that need to keep the same type.
#
# Inputs:
#
# -v, --vault : Path to the vault.
# -i, --input-list: path to input variable list
# -o, --output-list : name of output variable list
#
############################################
from AutoRPE.UtilsRPE.SourceManager import store_variable_list, load_variable_list, load_vault
if __name__ == "__main__":
# Getting Command line options
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-v", "--vault", dest="vault_path", default=None, metavar="FILE")
parser.add_option("-i", "--input-list", dest="path_input_list", default=None, metavar="FILE")
parser.add_option("-o", "--output-list", dest="path_to_output_list", default="list_of_additional_var.txt", metavar="FILE")
(options, args) = parser.parse_args()
vault_path = options.vault_path
path_to_input_list = options.path_input_list
path_to_output_list = options.path_to_output_list
if path_to_input_list is None or vault_path is None:
print("\nThe arguments -s/--input-source and -v/--vault are mandatory\n")
parser.print_help()
exit(1)
# Load vault
vault = load_vault(vault_path)
# Load list of variables
list_of_variable_ids = load_variable_list(path_to_input_list)
# Initialize empty list of additional variables
list_of_additional_variable_ids = []
# Loop over input list of variables
for _id in list_of_variable_ids:
# Obtain variable object from id
var = vault.get_variable_by_id(_id)
# Loop over variables that are in the object .same_as list.
for additional_var in var.same_as:
# If the variable has an id, add it to the additional var list.
if additional_var.id:
list_of_additional_variable_ids.append(additional_var.id)
# Keep unique values
all_variables = list(set(list_of_additional_variable_ids + list_of_variable_ids))
# Save variable list
load_variable_list(all_variables, path_to_output_list, vault)
############################################
# Takes as input clean sources and substitutes all real declaration with RPE variables.
# Takes as input preprocessed sources and substitutes all real declaration with RPE variables.
# Fixes all incoherence that can arise from these changes
# Produces a vault with the info on these new sources produced.
############################################
import AutoRPE.UtilsRPE.SourceManager as SourceManager
from AutoRPE.UtilsRPE.ImplementRPE import Implement_RPE
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
......@@ -11,6 +11,20 @@ from os.path import isdir
from shutil import rmtree
def load_list_of_files_to_keep_unmodified(filename=None):
from os.path import isfile, abspath, dirname
if filename is None:
package_directory = dirname(abspath(__file__))
# Hardcoded path to file.
filename = package_directory + "/../AdditionalFiles/list_of_files_to_keep_unmodified.txt"
if isfile(filename):
print("Loading list of exceptions from %s\n\n" % filename)
with open(filename) as f:
return [l.strip() for l in f if l.strip() and l.strip()[0] != "#"]
else:
return None
def get_command_line_arguments():
import argparse
from AutoRPE.UtilsRPE.Error import PathType
......@@ -45,7 +59,7 @@ if __name__ == "__main__":
print("Starting the implementation of the reduced precision emulator in:\n %s\n\n" % path_to_input_sources)
# List of files that should not be processed, as red from "list_of_files_to_keep_unmodified.txt"
preprocess_blacklist = SourceManager.load_list_of_files_to_keep_unmodified()
preprocess_blacklist = load_list_of_files_to_keep_unmodified()
# Format text + replace real declarations with type(rpe_var)
vault = Implement_RPE(path_to_input_sources, path_to_output_sources, preprocess_blacklist)
......
from AutoRPE.UtilsRPE.SourceManager import load_vault, store_variable_list, load_variable_list
from optparse import OptionParser
def translate_vault(list1, path1, path2, output_filename, print_id_not_found=False):
vault01 = load_vault(path1)
vault02 = load_vault(path2)
list_of_equivalent_id = []
list_of_jon_doe = []
id_in_list = load_variable_list(list1)
variables01 = [vault01.get_variable_by_id(id) for id in id_in_list]
variables02 = [var for var in vault02.variables if var.id]
for var01 in variables01:
name01 = var01.name
procedure01 = var01.procedure.name
module01 = var01.procedure.module.name
var_found = False
for var02 in variables02:
id02 = var02.id
name02 = var02.name
procedure02 = var02.procedure.name
module02 = var02.procedure.module.name
if name02 == name01 and procedure02 == procedure01 and module02 == module01:
list_of_equivalent_id.append(id02)
variables02.remove(var02)
var_found = True
break
if not var_found and print_id_not_found:
for var02 in variables02:
id02 = var02.id
name02 = var02.name
if name02 == name01:
list_of_jon_doe.append(id02)
if output_filename is not False:
print("Saving translated list into %s" % output_filename )
store_variable_list(list_of_equivalent_id, output_filename, vault02)
if print_id_not_found and len(list_of_jon_doe):
not_found_filename = "ids_not_found.txt"
print("Saving list of missing variables in %s" % not_found_filename )
store_variable_list(list_of_jon_doe, not_found_filename, vault02)
if __name__ == "__main__":
# Usage message
help_text = """
This script is intended to allow to translate a list of variables from one vault to another.
It requires the arguments --list, --origin-vault and --destination-vault.
"""
parser = OptionParser(usage=help_text)
parser.add_option("--list", dest="list1")
parser.add_option("-o", "--output-list", dest="output_filename", default=False, metavar="FILE")
parser.add_option("--origin-vault", dest="path1")
parser.add_option("--destination-vault", dest="path2")
parser.add_option("-v", "--verbose", dest="print_id_not_found", default=False, action="store_true",
help="print variables not found to list")
(options, args) = parser.parse_args()
list1 = options.list1
path1 = options.path1
path2 = options.path2
if not list1 or not path1 or not path2:
parser.print_help()
exit(1)
print_id_not_found = options.print_id_not_found
output_filename = options.output_filename
translate_vault(list1, path1, path2, output_filename, print_id_not_found)
......@@ -51,7 +51,7 @@ def attribute_type(attributes, variable=None):
type_specifier = re.search(RegexPattern.precision_of_real_declaration, t0, re.I)
if type_specifier:
try:
return VariablePrecision.lookup_table[type_specifier.group(3)]
return VariablePrecision.lookup_table[type_specifier.group(3).lower()]
except KeyError:
raise Error.ExceptionNotManaged("Pleas add this type specifier into VariablePrecision.lookup_table: %s"
% type_specifier)
......
......@@ -4,8 +4,16 @@ import AutoRPE.UtilsRPE.Error as Error
import re
def load_source_file(_filename, extension, no_strip=False):
import AutoRPE.UtilsRPE.BasicStructures as BasicStructures
# Create a SourceFile object
file = BasicStructures.SourceFile(_filename, extension)
# Load the file text into the object
file.read_file(_filename, no_strip)
return file
def load_sources(input_path, extension=[".f90"]):
from AutoRPE.UtilsRPE.BasicStructures import ReadSourceFile
from os.path import join, abspath, splitext
from os import walk
file_list = []
......@@ -21,13 +29,27 @@ def load_sources(input_path, extension=[".f90"]):
# Include file with specified extensions and exclude hidden files (like .swp files)
file_extension = splitext(f_name)[1]
if file_extension in extension and f_name[0][0] != ".":
file_list.append(ReadSourceFile(join(dir_path, f_name), file_extension, no_strip=True))
file_list.append(load_source_file(join(dir_path, f_name), file_extension, no_strip=True))
if not len(file_list):
raise ValueError("No files found in %s folder", abspath(input_path))
return file_list
def load_intrinsics():
from os.path import abspath, dirname
package_directory = dirname(abspath(__file__))
# Hardcoded path to file.
with open(package_directory + "/../AdditionalFiles/fortran_intrinsics") as f:
intrinsics = dict((intrinsic.split(',')[0].lower(), intrinsic.split(',')[1]) for intrinsic in f)
return intrinsics
# List of all the intrinsic
list_of_intrinsics = load_intrinsics()
def literal_mask(string):
splitted_string = [c for c in string]
length = len(splitted_string)
......@@ -247,6 +269,19 @@ def remove_if_condition(_line, return_if_condition=False):
return [condition, _line]
def remove_else_condition(line):
import re
if line.replace(" ", "").lower().strip().find("else") == 0:
line = re.sub(r"^\s*else\s*;", "", line, flags=re.I)
return line
def remove_if_else_condition(line):
line = remove_if_condition(line)
line = remove_else_condition(line)
return line
def split_if_condition(_line):
return remove_if_condition(_line, return_if_condition=True)
......
......@@ -18,17 +18,13 @@ def generate_dict_key(el_types, el_dimensions, uniform=False):
return hash(str(key))
class ReadSourceFile:
def __init__(self, _filename, extension, no_strip=False):
# Init variables
class SourceFile:
def __init__(self, _filename, extension, lines=[], text=""):