Commit cd939da3 authored by sparonuz's avatar sparonuz
Browse files

[Issue#77]: # This is a combination of 42 commits.

[Issue#77]: Fix bug in update_blocks: the father block was not correctely set

[Issue#77]: Remove CodeCleaner.py

[Issue#77]: function read_file removed from ReadSourceFile class. class ReadSourceFile moved to SourceFile

[Issue#77]: Removed SourceManager: it was containing functions that belonged somwhere else.

[Issue#77]: Fix for issue #147: these scripts are now useless

[Issue#77]: Added parameter working precision to function that checks if two types are equivalent

[Issue#77]: FIX: get_type_of_content was giving char for argument that are function with char argument, but the function per se was not char

[Issue#77]: Module class was loading again lines from source file, which is kind of overkill, since this is already done by SourceFile class. This is moreover wrong is some changes were done in between

[Issue#77]: SubprogramCall has now a member to store the block were the call is done

[Issue#77]: Functions to add cast to non coherent calls: for the moment has been tested on the firs calls found

[Issue#77]: Added function to create a regex for name occurrence: was failing if name contained parenthesis

[Issue#77]: Now adds the #  include \"single_precision_substitute.h90\" clause

[Issue#77]: Added fix for reshapes

[Issue#77]: Dependece were not propagated correctely because the id property was used: it did't make much sense

[Issue#77]: For the moment commented out the part that should suggest to create an interface

[Issue#77]: added function to escape string to put them in regex
parent c1f97764
from AutoRPE.UtilsRPE.BasicStructures import Interface import AutoRPE.UtilsRPE.ModifyCodeUtils as ModifyCodeUtils
from AutoRPE.UtilsRPE.SourceManager import load_variable_list, load_vault, expand_sources import AutoRPE.UtilsRPE.Vault as Vault
from os.path import basename import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
import re
# Script that automatically creates different versions of a routine with different precisions def get_command_line_arguments():
def fix_casts(source_path, vault_path, list_filename=None): import argparse
vault = load_vault(vault_path) from AutoRPE.UtilsRPE.Error import PathType
pattern = r"call.*\b(real\(.*)" parser = argparse.ArgumentParser()
cast_pattern = r"\b(real\(.*)" parser.add_argument("--preprocessed-input-files", required=True, type=PathType(exists=True, type="dir"), nargs='+',
pattern_precision = r"\b([wds]p|[8]) *\)" help="Path to the preprocessed sources")
call_pattern = r"call *\b(\w*)\b" parser.add_argument("--original-input-files", required=True, type=PathType(exists=True, type="dir"), nargs='+',
help="Path to the non preprocessed sources")
parser.add_argument("--vault", default=None, type=PathType(exists=True, type="file"),
help="For debug purposes")
lookup_table = {52: "dp", 23: "sp"} return parser.parse_args()
if list_filename is not None:
# Update vault precisions with variable list
variables_to_update = load_variable_list(list_filename)
for variable in vault.variables:
if variable.id in variables_to_update:
variable.precision = 52
file_paths = expand_sources(source_path=source_path)
for file_path in file_paths:
if not (file_path.count(".F90") or file_path.count(".f90") or file_path.count(".h90")):
continue
file_modified = False
module_name = basename(file_path).split(".")[0]
with open(file_path, "r", encoding="latin-1") as f:
lines = [l for l in f]
for line_index, line in enumerate(lines):
# Check if there's a routine call with a cast to real
if re.search(pattern, line, re.I):
try:
calls = call_to_subroutine(line)
except ValueError:
continue
for call in calls:
arguments = find_call_arguments(call)
procedure_name = re.search(call_pattern, call, re.I).group(1)
try:
procedure = vault.get_procedure_by_name(procedure_name, module_name)
except ProcedureNotFound:
continue
# If it is an interface, find the specific procedure being used
if isinstance(procedure, Interface):
try:
procedure = match_arguments_and_interface(arguments, interface=procedure, vault=vault)
except AssertionError:
procedure = None
for index, argument in enumerate(arguments):
if re.search(cast_pattern, argument, re.I):
if procedure is not None:
if not is_keyword_argument(argument):
target_precision = procedure.get_variable_by_position(index).precision
else:
key = argument.split("=")[0].strip()
target_precision = procedure.get_variable_by_name(key).precision
else:
# For now we will assume that in this case it is not necessary to change the cast
continue
# Problematic to assume that the precision of an external procedure is either double or single
# target_precision = 52
target_precision = 23
try:
target_precision = lookup_table[target_precision]
except KeyError:
raise KeyError("target_precision %s not found in lookup_table", target_precision)
cast = close_brackets(re.search(cast_pattern, argument, re.I).group(1))
precision = re.search(pattern_precision, cast, re.I).group(1)
if precision.strip() == target_precision:
continue
else:
cast_replacement = re.sub(r"(.*\b)[wds]p( *\).*)", r"\1%s\2" % target_precision,
cast, re.I)
line = line.replace(cast, cast_replacement)
lines[line_index] = line
file_modified = True
if file_modified:
text = "".join(lines).strip()
with open(file_path, "w") as f:
f.write(text)
if __name__ == "__main__": if __name__ == "__main__":
# Getting Command line options args = get_command_line_arguments()
from optparse import OptionParser preprocessed_input_files = args.preprocessed_input_files
original_input_files = args.original_input_files
help_text = """ vault = args.vault
""" # Set working precision to double, since it is the precision used to compile preprocessed sources
parser = OptionParser(usage=help_text) VariablePrecision.set_working_precision('dp')
parser.add_option("-l", "--variable-list", dest="variable_list", if vault is None:
help="Path to the filename with the list of variables.", default=None, metavar="FILE") vault = Vault.create_database(preprocessed_input_files, extension=".f90")
parser.add_option("-v", "--vault", dest="vault", default=None, metavar="FILE") else:
parser.add_option("-s", "--input-source", dest="input_source", default=None, metavar="FILE") # This is useful when debugging: it is not needed to generate a vault at every run
vault = Vault.load_vault(vault)
(options, args) = parser.parse_args() # Set working precision to single, so as to find the needed fixes
vault_path = options.vault VariablePrecision.set_working_precision('sp')
source_path = options.input_source # Search in preprocessed code where fixes are due
list_filename = options.variable_list dictionary_of_fixes = ModifyCodeUtils.get_fixes(vault)
# Retrieve these calls in the original code, and fix them
if source_path is None or vault_path is None: fixed_modules = ModifyCodeUtils.add_cast(original_input_files, dictionary_of_fixes)
print("\nArgument -s/--input-source and -v/--vault are mandatory\n") # Save fixed sources
parser.print_help() for module in fixed_modules:
exit(1) module.save_file()
fix_casts(source_path, vault_path, list_filename)
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions
import AutoRPE.UtilsRPE.SourceManager as SourceManager
import AutoRPE.UtilsRPE.ModifyCodeUtils as ModifyCodeUtils import AutoRPE.UtilsRPE.ModifyCodeUtils as ModifyCodeUtils
import AutoRPE.UtilsRPE.Vault as Vault import AutoRPE.UtilsRPE.Vault as Vault
...@@ -22,9 +21,18 @@ def change_code(sources_path, var_to_be_changed, target_precision): ...@@ -22,9 +21,18 @@ def change_code(sources_path, var_to_be_changed, target_precision):
ModifyCodeUtils.update_precision_declaration(variable, modules, target_precision) ModifyCodeUtils.update_precision_declaration(variable, modules, target_precision)
print("Updated precision of var ", variable.name, index) print("Updated precision of var ", variable.name, index)
# # Save file to disk overwriting originals (this way preserve the folder structure, too difficult to reproduce) # Save file to disk overwriting originals (this way preserve the folder structure, too difficult to reproduce)
# for module in modules: for module in modules:
# module.save_file() module.save_file()
def load_variable_list(filename):
# Load list of variables
from numpy import loadtxt
list_of_variables = loadtxt(filename, dtype=int).tolist()
if not isinstance(list_of_variables, list):
list_of_variables = [list_of_variables]
return list_of_variables
def get_command_line_arguments(): def get_command_line_arguments():
...@@ -59,7 +67,7 @@ if __name__ == "__main__": ...@@ -59,7 +67,7 @@ if __name__ == "__main__":
VariablePrecision.set_working_precision(working_precision) VariablePrecision.set_working_precision(working_precision)
# Load list of variable ids # Load list of variable ids
variable_id = SourceManager.load_variable_list(variable_list) variable_id = load_variable_list(variable_list)
vault = Vault.load_vault(vault_path) vault = Vault.load_vault(vault_path)
# Get variables from their ids # Get variables from their ids
...@@ -70,18 +78,17 @@ if __name__ == "__main__": ...@@ -70,18 +78,17 @@ if __name__ == "__main__":
variables += used_var_in_ext variables += used_var_in_ext
# Propagate dependencies of the variables using same_as property # Propagate dependencies of the variables using same_as property
dependent_var = []
for var in variables: for var in variables:
for additional_var in var.same_as: for additional_var in var.same_as:
# Only if the variable has an id ban it # Only if the variable has an id ban it
if additional_var.id: if additional_var not in variables:
dependent_var.append(additional_var) variables.append(additional_var)
# Keep unique values # Keep unique values
all_variables = list(set(variables + dependent_var)) all_variables = list(set(variables))
# This is for debugging purposes; from a run to the next we want errors to happen always in the same order # This is for debugging purposes; from a run to the next we want errors to happen always in the same order
all_variables.sort(key=lambda x: x.id) all_variables.sort(key=lambda x: x.incremental_number)
# Change precision of variables in source code # Change precision of variables in source code
change_code(input_files, all_variables, variable_precision) change_code(input_files, all_variables, variable_precision)
############################################
# Purpose:
# This script is indented to identify which variables are used in calls to external subprogram.
#
# Inputs:
#
# -s, --source : Path to the preprocessed sources.
# -v, --vault : Path to the vault.
# -o, --output : Output filename for the list of variables used in external calls.
#
############################################
from AutoRPE.UtilsRPE.SourceManager import expand_sources, load_vault, store_variable_list
from AutoRPE.UtilsRPE.BasicStructures import SubRoutine, Interface
from AutoRPE.UtilsRPE.CallManager import call_to_subroutine, find_called_subroutine, find_call_arguments
from AutoRPE.UtilsRPE.Getter import get_variable
from AutoRPE.UtilsRPE.Preprocessor import Preprocessor
import AutoRPE.UtilsRPE.CurrentBlock as CurrentBlock
from AutoRPE.UtilsRPE.Error import ProcedureNotFound
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
from os.path import basename, isfile, isdir
def get_command_line_arguments():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--source",
help="Path to the source that will be modified.", default=None)
parser.add_argument("-v", "--vault",
help="Path to the vault.", default=None, metavar="FILE")
parser.add_argument("-o", "--output",
help="Output filename", default=None, metavar="FILE")
args = parser.parse_args()
if args.source_path is None:
parser.error("Argument -s/--source required!")
if args.vault_path is not None:
parser.error("Argument -v/--vault required!")
if not isdir(args.source_path):
parser.error("Argument -s/--source %s is not a valid directory." % args.source)
if not isfile(args.vault):
parser.error("Argument -v/--vault %s is not a file." % args.vault)
return args
def find_variables_used_in_external_calls(files, vault, exceptions):
external_subprogram_found = []
variables = []
for filepath in files:
module_name = basename(filepath).split(".")[0]
module = vault[module_name]
if module is None:
continue
file_info = Preprocessor(filepath)
file_info.pre_process()
current_block = CurrentBlock.CurrentBlock(module.main)
for index, line in enumerate(file_info.lines):
print("\r %04i/%04i %50s" % (index + 1, len(file_info.lines), module_name), end="")
try:
subroutine_calls = call_to_subroutine(line)
except ValueError:
continue
if subroutine_calls:
for call in subroutine_calls:
is_external = False
called_subroutine = find_called_subroutine(call)
# Find the procedure called (object)
try:
subroutine = vault.get_procedure_by_name(called_subroutine, current_block.module_name)
except ProcedureNotFound:
subroutine = None
pass
if isinstance(subroutine, SubRoutine):
continue
elif isinstance(subroutine, Interface):
continue
else:
is_external = True
if is_external:
external_subprogram_found.append(called_subroutine)
# Starting with the exceptions!"
is_exception = False
for exception in exceptions:
if called_subroutine.count(exception):
is_exception = True
break
if is_exception:
continue
called_arguments = find_call_arguments(call)
if not called_arguments:
# If there's no argument, continue
continue
else:
for argument in called_arguments:
try:
variable = get_variable(argument, procedure=current_block.procedure, vault=vault)
except AssertionError:
continue
except ValueError:
continue
if variable is not None:
if variable.type in VariablePrecision.real_types:
variables.append(variable.id)
else:
current_block.in_which_block(line, vault=vault)
variables = list(set(variables))
return variables
if __name__ == "__main__":
args = get_command_line_arguments()
source_path = args.source
vault_path = args.vault
output_filename = args.output
exceptions = ["mpi_", "xios_", "agrif_", "cice", "ice_", "wheneq", "trc_", "flush"]
vault = load_vault(vault_path)
files = expand_sources(source_path)
variables = find_variables_used_in_external_calls(files, vault, exceptions)
store_variable_list(variables, output_filename, vault)
############################################
# Purpose:
# This script takes a list of variables and outputs all the additional variables that need to keep the same type.
#
# Inputs:
#
# -v, --vault : Path to the vault.
# -i, --input-list: path to input variable list
# -o, --output-list : name of output variable list
#
############################################
from AutoRPE.UtilsRPE.SourceManager import store_variable_list, load_variable_list, load_vault
if __name__ == "__main__":
# Getting Command line options
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-v", "--vault", dest="vault_path", default=None, metavar="FILE")
parser.add_option("-i", "--input-list", dest="path_input_list", default=None, metavar="FILE")
parser.add_option("-o", "--output-list", dest="path_to_output_list", default="list_of_additional_var.txt", metavar="FILE")
(options, args) = parser.parse_args()
vault_path = options.vault_path
path_to_input_list = options.path_input_list
path_to_output_list = options.path_to_output_list
if path_to_input_list is None or vault_path is None:
print("\nThe arguments -s/--input-source and -v/--vault are mandatory\n")
parser.print_help()
exit(1)
# Load vault
vault = load_vault(vault_path)
# Load list of variables
list_of_variable_ids = load_variable_list(path_to_input_list)
# Initialize empty list of additional variables
list_of_additional_variable_ids = []
# Loop over input list of variables
for _id in list_of_variable_ids:
# Obtain variable object from id
var = vault.get_variable_by_id(_id)
# Loop over variables that are in the object .same_as list.
for additional_var in var.same_as:
# If the variable has an id, add it to the additional var list.
if additional_var.id:
list_of_additional_variable_ids.append(additional_var.id)
# Keep unique values
all_variables = list(set(list_of_additional_variable_ids + list_of_variable_ids))
# Save variable list
load_variable_list(all_variables, path_to_output_list, vault)
############################################ ############################################
# Takes as input clean sources and substitutes all real declaration with RPE variables. # Takes as input preprocessed sources and substitutes all real declaration with RPE variables.
# Fixes all incoherence that can arise from these changes
# Produces a vault with the info on these new sources produced. # Produces a vault with the info on these new sources produced.
############################################ ############################################
import AutoRPE.UtilsRPE.SourceManager as SourceManager
from AutoRPE.UtilsRPE.ImplementRPE import Implement_RPE from AutoRPE.UtilsRPE.ImplementRPE import Implement_RPE
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
...@@ -11,6 +11,20 @@ from os.path import isdir ...@@ -11,6 +11,20 @@ from os.path import isdir
from shutil import rmtree from shutil import rmtree
def load_list_of_files_to_keep_unmodified(filename=None):
from os.path import isfile, abspath, dirname
if filename is None:
package_directory = dirname(abspath(__file__))
# Hardcoded path to file.
filename = package_directory + "/../AdditionalFiles/list_of_files_to_keep_unmodified.txt"
if isfile(filename):
print("Loading list of exceptions from %s\n\n" % filename)
with open(filename) as f:
return [l.strip() for l in f if l.strip() and l.strip()[0] != "#"]
else:
return None
def get_command_line_arguments(): def get_command_line_arguments():
import argparse import argparse
from AutoRPE.UtilsRPE.Error import PathType from AutoRPE.UtilsRPE.Error import PathType
...@@ -45,7 +59,7 @@ if __name__ == "__main__": ...@@ -45,7 +59,7 @@ if __name__ == "__main__":
print("Starting the implementation of the reduced precision emulator in:\n %s\n\n" % path_to_input_sources) print("Starting the implementation of the reduced precision emulator in:\n %s\n\n" % path_to_input_sources)
# List of files that should not be processed, as red from "list_of_files_to_keep_unmodified.txt" # List of files that should not be processed, as red from "list_of_files_to_keep_unmodified.txt"
preprocess_blacklist = SourceManager.load_list_of_files_to_keep_unmodified() preprocess_blacklist = load_list_of_files_to_keep_unmodified()
# Format text + replace real declarations with type(rpe_var) # Format text + replace real declarations with type(rpe_var)
vault = Implement_RPE(path_to_input_sources, path_to_output_sources, preprocess_blacklist) vault = Implement_RPE(path_to_input_sources, path_to_output_sources, preprocess_blacklist)
......
from AutoRPE.UtilsRPE.SourceManager import load_vault, store_variable_list, load_variable_list
from optparse import OptionParser
def translate_vault(list1, path1, path2, output_filename, print_id_not_found=False):
vault01 = load_vault(path1)
vault02 = load_vault(path2)
list_of_equivalent_id = []
list_of_jon_doe = []
id_in_list = load_variable_list(list1)
variables01 = [vault01.get_variable_by_id(id) for id in id_in_list]
variables02 = [var for var in vault02.variables if var.id]
for var01 in variables01:
name01 = var01.name
procedure01 = var01.procedure.name
module01 = var01.procedure.module.name
var_found = False
for var02 in variables02:
id02 = var02.id
name02 = var02.name
procedure02 = var02.procedure.name
module02 = var02.procedure.module.name
if name02 == name01 and procedure02 == procedure01 and module02 == module01:
list_of_equivalent_id.append(id02)
variables02.remove(var02)
var_found = True
break
if not var_found and print_id_not_found:
for var02 in variables02:
id02 = var02.id
name02 = var02.name
if name02 == name01:
list_of_jon_doe.append(id02)
if output_filename is not False:
print("Saving translated list into %s" % output_filename )
store_variable_list(list_of_equivalent_id, output_filename, vault02)
if print_id_not_found and len(list_of_jon_doe):
not_found_filename = "ids_not_found.txt"
print("Saving list of missing variables in %s" % not_found_filename )
store_variable_list(list_of_jon_doe, not_found_filename, vault02)
if __name__ == "__main__":
# Usage message
help_text = """
This script is intended to allow to translate a list of variables from one vault to another.
It requires the arguments --list, --origin-vault and --destination-vault.
"""
parser = OptionParser(usage=help_text)
parser.add_option("--list", dest="list1")
parser.add_option("-o", "--output-list", dest="output_filename", default=False, metavar="FILE")
parser.add_option("--origin-vault", dest="path1")
parser.add_option("--destination-vault", dest="path2")
parser.add_option("-v", "--verbose", dest="print_id_not_found", default=False, action="store_true",
help="print variables not found to list")
(options, args) = parser.parse_args()
list1 = options.list1
path1 = options.path1
path2 = options.path2
if not list1 or not path1 or not path2:
parser.print_help()
exit(1)
print_id_not_found = options.print_id_not_found
output_filename = options.output_filename
translate_vault(list1, path1, path2, output_filename, print_id_not_found)
...@@ -51,7 +51,7 @@ def attribute_type(attributes, variable=None): ...@@ -51,7 +51,7 @@ def attribute_type(attributes, variable=None):
type_specifier = re.search(RegexPattern.precision_of_real_declaration, t0, re.I) type_specifier = re.search(RegexPattern.precision_of_real_declaration, t0, re.I)
if type_specifier: if type_specifier:
try: try:
return VariablePrecision.lookup_table[type_specifier.group(3)] return VariablePrecision.lookup_table[type_specifier.group(3).lower()]
except KeyError: except KeyError:
raise Error.ExceptionNotManaged("Pleas add this type specifier into VariablePrecision.lookup_table: %s" raise Error.ExceptionNotManaged("Pleas add this type specifier into VariablePrecision.lookup_table: %s"
% type_specifier) % type_specifier)
......
...@@ -4,8 +4,16 @@ import AutoRPE.UtilsRPE.Error as Error ...@@ -4,8 +4,16 @@ import AutoRPE.UtilsRPE.Error as Error
import re import re
def load_source_file(_filename, extension, no_strip=False):
import AutoRPE.UtilsRPE.BasicStructures as BasicStructures
# Create a SourceFile object
file = BasicStructures.SourceFile(_filename, extension)
# Load the file text into the object
file.read_file(_filename, no_strip)
return file
def load_sources(input_path, extension=[".f90"]): def load_sources(input_path, extension=[".f90"]):
from AutoRPE.UtilsRPE.BasicStructures import ReadSourceFile
from os.path import join, abspath, splitext from os.path import join, abspath, splitext
from os import walk from os import walk
file_list = [] file_list = []
...@@ -21,13 +29,27 @@ def load_sources(input_path, extension=[".f90"]): ...@@ -21,13 +29,27 @@ def load_sources(input_path, extension=[".f90"]):
# Include file with specified extensions and exclude hidden files (like .swp files) # Include file with specified extensions and exclude hidden files (like .swp files)
file_extension = splitext(f_name)[1] file_extension = splitext(f_name)[1]
if file_extension in extension and f_name[0][0] != ".": if file_extension in extension and f_name[0][0] != ".":
file_list.append(ReadSourceFile(join(dir_path, f_name), file_extension, no_strip=True)) file_list.append(load_source_file(join(dir_path, f_name), file_extension, no_strip=True))
if not len(file_list): if not len(file_list):
raise ValueError("No files found in %s folder", abspath(input_path)) raise ValueError("No files found in %s folder", abspath(input_path))