Commit c1f97764 authored by sparonuz's avatar sparonuz
Browse files

[Issue#147]: # This is a combination of 49 commits.

[Issue#147]: Added flag to mark variables used in external calls

[Issue#147]: Function to fill vault has been excorporated. Now makeVault assigns ids to all real variables: these were lines that got lost in refactoring.

[Issue#147]: Modified main of Modify code. Now 1) read the banned_var list + vault 2) add var used in external calls to ban list 3) propagate dependency of the ban list

[Issue#147]: Moved function change_code form inserter to ModifyCode

[Issue#147]: Moved function from Inseter + var used in external have to be banned just if they are RPE type

[Issue#147]: Added additional parameter "var_type" for tracking function

[Issue#147]: Added incremental_id to variables, to be used during tracking.

[Issue#147]: Added var_type to function that propagate dependency. Now var_identifier uses var.incremental_number

[Issue#147]: Added tracking of RPE variables

[Issue#147]: Modified function load_sources to take extension as input, and to be able to read files in subdirectories

[Issue#147]: Extension is now passed to create ReadSourceFile objects, to properly set module_name

[Issue#147]: Updated rebuild_text, so as to split lines with \n marker in module.lines

[Issue#147]: Moved all function to deal with non preprocessed code into a single file

[Issue#147]: Added parameter to read file to avoid removing spaces when reading. Fixed bug in write_file that was adding a last empty line to every file.

[Issue#147]: Check that, when looking for allocation of a var  we did not match a member with the same name: a_i will also match %a_i

[Issue#147]: Storing precision with and without KIND keyword specification was useless: removed
parent 966cb1c2
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions
import AutoRPE.UtilsRPE.SourceManager as SourceManager
import AutoRPE.UtilsRPE.ModifyCodeUtils as ModifyCodeUtils
import AutoRPE.UtilsRPE.Vault as Vault
def change_code(sources_path, var_to_be_changed, target_precision):
"""
The purpose of this function is to modify the declarations of several variables in a source-code.
- source_path: Path to a folder that contains the source files
- var_to_be_changed: list of variables that will be modified in the code
- vault_path: Path to the database that contains info about source files
"""
# Load files that will be modified
modules = BasicFunctions.load_sources(sources_path, extension=[".F90", ".h90", ".f90"])
# For every variable in the banned, searches declaration through source files and change declaration's precision
for index, variable in enumerate(var_to_be_changed):
ModifyCodeUtils.update_precision_declaration(variable, modules, target_precision)
print("Updated precision of var ", variable.name, index)
# # Save file to disk overwriting originals (this way preserve the folder structure, too difficult to reproduce)
# for module in modules:
# module.save_file()
def get_command_line_arguments():
import argparse
from AutoRPE.UtilsRPE.Error import PathType
parser = argparse.ArgumentParser()
parser.add_argument("--input-files", required=True, type=PathType(exists=True, type="dir"), nargs='+',
help="Path to the non preprocessed sources")
parser.add_argument("--variable-list", required=True, type=PathType(exists=True, type='file'),
help="Path to the filename with the list of variables")
parser.add_argument("--vault", required=True, type=PathType(exists=True, type='file'),
help="Path to vault generated either from RPE implementation")
parser.add_argument("-variable-precision", default="dp",
help="Precision specification at which the variables in the list will be set")
parser.add_argument("--working-precision", default="23", type=int,
help="Precision used for wp variables in the code. i.e. double for standard runs, "
"single for mixed-precision versions")
return parser.parse_args()
if __name__ == "__main__":
args = get_command_line_arguments()
input_files = args.input_files
variable_list = args.variable_list
vault_path = args.vault
variable_precision = args.variable_precision
working_precision = args.working_precision
# Set working precision
VariablePrecision.set_working_precision(working_precision)
# Load list of variable ids
variable_id = SourceManager.load_variable_list(variable_list)
vault = Vault.load_vault(vault_path)
# Get variables from their ids
variables = [vault.get_variable_by_id(identifier) for identifier in variable_id]
# List variables used in external calls, because they have to be change precision too
used_var_in_ext = [v for v in vault.variables if v.is_used_in_external and v.type == "rpe_var"]
variables += used_var_in_ext
# Propagate dependencies of the variables using same_as property
dependent_var = []
for var in variables:
for additional_var in var.same_as:
# Only if the variable has an id ban it
if additional_var.id:
dependent_var.append(additional_var)
# Keep unique values
all_variables = list(set(variables + dependent_var))
# This is for debugging purposes; from a run to the next we want errors to happen always in the same order
all_variables.sort(key=lambda x: x.id)
# Change precision of variables in source code
change_code(input_files, all_variables, variable_precision)
......@@ -68,7 +68,7 @@ if __name__ == "__main__":
# Loading vault
vault = load_vault(vault_path)
else:
vault = Vault.Vault(path_to_input_sources)
vault = Vault.create_database(path_to_input_sources)
# Find all files in sources
all_sources = BasicFunctions.load_sources(path_to_input_sources)
......
......@@ -16,7 +16,7 @@
# * bar(sp)
#
from AutoRPE.UtilsRPE.Inserter import replace_real_declaration
from AutoRPE.UtilsRPE.Inserter import replace_real_precision
import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions
import AutoRPE.UtilsRPE.Vault as Vault
import AutoRPE.UtilsRPE.RegexPattern as RegexPattern
......@@ -88,9 +88,9 @@ def generate_routine_versions(precision_combinations, sensitive_variables, subpr
line_index = \
[idx for idx, l in enumerate(new_subprogram_code) if re.search(r".*::.*\b%s\b" % variable.name, l, re.I)][0]
# Update line
new_subprogram_code[line_index] = replace_real_declaration(variable,
new_subprogram_code[line_index],
combination[index])
new_subprogram_code[line_index] = replace_real_precision(variable,
new_subprogram_code[line_index],
combination[index])
# Update name
new_subprogram_code[0] = new_subprogram_code[0].replace(subprogram.name, new_subprogram_name)
......
......@@ -7,7 +7,7 @@ def get_command_line_arguments():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input-source", default=None, required=True,
parser.add_argument("-i", "--input-source", required=True, type=PathType(exists=True, type="dir"),
help="Path to clean input fortran files")
parser.add_argument("-o", "--output-vault", default="vault.pkl",
help="Name of the output vault file")
......@@ -36,7 +36,7 @@ if __name__ == "__main__":
# Parse input sources and stores info in vault
print("Creating a database from sources found in:\n %s" % path_to_input_sources)
vault = Vault.Vault(path_to_input_sources)
vault = Vault.create_database(path_to_input_sources)
# Store the dependencies btw variables, and create a graph for visualization
TrackDependency.propagate_dependencies(vault, "./graph.gexf")
......
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
from AutoRPE.UtilsRPE.Inserter import change_code
from os.path import isfile, isdir
def get_command_line_arguments():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-l", "--variable--file-list", default=None,
help="Path to the filename with the list of variables.")
parser.add_argument("-t", "--target-precision", default="dp",
help="Precision specification at which the variables in the list will be set")
parser.add_argument("-w", "--working-precision", default="single",
help="Precision used for wp variables in the code. i.e. double for standard runs, "
"single for mixed-precision versions")
parser.add_argument("-s", "--source", default=None,
help="Path to the source that will be modified.")
parser.add_argument("-v", "--vault", default=None,
help="Path to the vault.")
args = parser.parse_args()
if args.variable_file_list is None:
parser.error("Argument -l/--variable-file-list required!")
if not isfile(args.variable_file_list):
parser.error("Argument -l/--variable-list %s is not a file." % args.variable_file_list)
if args.source is None:
parser.error("Argument -s/--source required!")
if not isdir(args.source):
parser.error("Argument -s/--source %s is not a valid directory." % args.source)
if args.vault is None:
parser.error("Argument -v/--vault required!")
if not isfile(args.vault):
parser.error("Argument -v/--vault %s is not a file." % args.vault)
return args
if __name__ == "__main__":
args = get_command_line_arguments()
list_filename = args.variable_file_list
vault_path = args.vault
source_path = args.source
target_precision = args.target_precision
working_precision = args.working_precision
# Setting the working precision
VariablePrecision.wp = working_precision
change_code(list_filename, source_path, vault_path, target_precision)
import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
import AutoRPE.UtilsRPE.RegexPattern as RegexPattern
import AutoRPE.UtilsRPE.Error as Error
import re
......@@ -47,11 +48,10 @@ def attribute_type(attributes, variable=None):
elif var_type == "public":
return None
elif var_type == "real":
pattern = r"real *\(([^\)]*)\)"
if re.search(pattern, t0, re.I):
type_specifier = re.search(pattern, t0, re.I).group(1).strip()
type_specifier = re.search(RegexPattern.precision_of_real_declaration, t0, re.I)
if type_specifier:
try:
return VariablePrecision.lookup_table[type_specifier]
return VariablePrecision.lookup_table[type_specifier.group(3)]
except KeyError:
raise Error.ExceptionNotManaged("Pleas add this type specifier into VariablePrecision.lookup_table: %s"
% type_specifier)
......
......@@ -4,17 +4,27 @@ import AutoRPE.UtilsRPE.Error as Error
import re
def load_sources(path):
def load_sources(input_path, extension=[".f90"]):
from AutoRPE.UtilsRPE.BasicStructures import ReadSourceFile
from os.path import join, abspath
from os import listdir
ord_list = listdir(path)
ord_list.sort()
file_list = [ReadSourceFile(join(path, filename)) for filename in
ord_list if
filename.count(".f90") and filename[0][0] != "."]
from os.path import join, abspath, splitext
from os import walk
file_list = []
# If the passed argument is not a list, convert to avoid splitting a string
if not isinstance(input_path, list):
input_path = [input_path]
# Reads files for top folder, and searches recursively in sub-folders
for path in input_path:
for dir_path, dir_name, file_name in walk(path):
for f_name in file_name:
# Include file with specified extensions and exclude hidden files (like .swp files)
file_extension = splitext(f_name)[1]
if file_extension in extension and f_name[0][0] != ".":
file_list.append(ReadSourceFile(join(dir_path, f_name), file_extension, no_strip=True))
if not len(file_list):
raise ValueError("No files found in %s folder", abspath(path))
raise ValueError("No files found in %s folder", abspath(input_path))
return file_list
......@@ -353,15 +363,3 @@ def merge_line_range(lines, line_index):
return line
# Merge lines in file
def merge_line_in_file(file, line_index):
with open(file, "w") as f:
lines = [l for l in f]
line = merge_line_range(lines, line_index)
lines[line_index] = line
for j in range(1, merge_line_range.number):
lines[line_index + j] = ""
text = "".join(lines)
f.write(text)
import AutoRPE.UtilsRPE.Error as Error
import itertools
def generate_dict_key(el_types, el_dimensions, uniform=False):
......@@ -18,28 +19,44 @@ def generate_dict_key(el_types, el_dimensions, uniform=False):
class ReadSourceFile:
def __init__(self, _filename):
def __init__(self, _filename, extension, no_strip=False):
# Init variables
self.path = _filename
self.filename = self.path.split("/")[-1]
self.module_name = self.filename.replace(".f90", "")
self.module_name = self.filename.replace(extension, "")
self.lines = []
self.text = ""
# Read file
self.read_file(_filename)
self.read_file(_filename, no_strip)
def read_file(self, _filename):
def read_file(self, _filename, no_strip):
try:
with open(_filename) as f:
self.text = f.read()
self.lines = [l.strip() for l in self.text.split("\n")]
if not no_strip:
self.lines = [l.strip() for l in self.text.split("\n")]
else:
self.lines = [l for l in self.text.split("\n")]
except UnicodeDecodeError:
raise UnicodeDecodeError("The file %s contains non UTF-8 character.\n"
" Please clean, and retry.", self.path)
raise Error.ExceptionNotManaged("The file %s contains non UTF-8 character.\n"
" Please clean, and retry." % self.path)
def rebuild_text(self):
self.text = "".join([l + "\n" for l in self.lines[:] if l.strip()])
def rebuild_text(self, no_strip=False):
# self.text = "".join([l + "\n" for l in self.lines[:] if l.strip()])
old_lines = self.lines[:]
self.lines = []
for line in old_lines:
if line.count("\n"):
self.lines.extend(line.split("\n"))
else:
self.lines.append(line)
if no_strip:
self.text = "".join([l + "\n" for l in self.lines[:] if l.strip()])
else:
self.text = "".join([l + "\n" for l in self.lines[:]])
# Remove las line that is an artificial empty line
self.text = self.text[:-1]
def save_file(self, file_path=None):
if file_path:
......@@ -51,8 +68,12 @@ class ReadSourceFile:
class Variable:
# Increment the var number automatically every time a variable is created
incremental_number = itertools.count().__next__
def __init__(self, procedure, _name, _type, _dimension, _module=None):
self.id = None
self.incremental_number = Variable.incremental_number()
self.module = _module
self.procedure = procedure
self.name = _name
......@@ -66,6 +87,7 @@ class Variable:
self.position = None
self.allocatable = False
self.appears_in_namelist = False
self.is_used_in_external = False
self.precision = None
self.is_parameter = None
self.is_member_of = None
......@@ -161,8 +183,8 @@ class DerivedType(Procedure):
class Module(ReadSourceFile):
def __init__(self, file_name):
super(Module, self).__init__(file_name)
def __init__(self, file_name, extension):
super(Module, self).__init__(file_name, extension)
self.name = self.module_name
self.main = None
self.blocks = []
......
......@@ -5,8 +5,6 @@ import AutoRPE.UtilsRPE.RegexPattern as RegexPattern
import AutoRPE.UtilsRPE.Error as Error
import re
Fortran_builtins = ["dimension", "intent", "type", "if", "elseif", "while", "allocate", "deallocate", "write", "read", "open", "inquire", "close", "where", "case", "character", "format", "flush", "rewind"]
def call_to_subroutine(string):
""" Return the list strings corresponding to the list of calls made in line"""
......@@ -20,6 +18,7 @@ def call_to_subroutine(string):
return ret_val
def calls_to_functions(string, procedure, vault):
""" Return the list strings corresponding to the list of functions called in line"""
if procedure:
......@@ -233,15 +232,17 @@ def is_call_to_function_vault(contents, procedure, vault):
return True
def get_procedure_inline(line, vault, procedure, only_subroutine=False):
def get_procedure_inline(line, vault, procedure, only_subroutine=False, only_function=False):
called_subroutines = []
called_functions = []
# Search for calls to subprograms
called_subroutines = get_subroutine(line, procedure, vault=vault)
if not only_function:
called_subroutines = get_subroutine(line, procedure, vault=vault)
if not only_subroutine:
called_functions = get_function(line, procedure, vault=vault)
else:
called_functions = []
return called_subroutines + called_functions
......@@ -261,10 +262,12 @@ def get_subroutine(string, procedure, vault):
def get_subprogram(string, procedure, vault, regex_pattern, procedure_type):
import AutoRPE.UtilsRPE.Getter as Getter
import AutoRPE.UtilsRPE.NatureDeterminer as NatureDeterminer
string = string.strip()
string = BasicFunctions.remove_literals(string)
to_return = []
unknown_subprogram = []
external_subprogram = []
is_unknown_external = False
# Check is not a function or subroutine declaration, to avoid matching with functions syntax
if re.search(RegexPattern.subprogram_declaration, string, re.MULTILINE | re.IGNORECASE):
return to_return
......@@ -275,78 +278,87 @@ def get_subprogram(string, procedure, vault, regex_pattern, procedure_type):
if matches:
for m in matches:
subprogram_name = m.groups()[0]
# Just if is a function we are looking for, check is not a variable or a builtin function
if procedure_type == BasicStructures.Function:
# Match the ret val of a function and fortran Instrinsics
if subprogram_name == procedure.name or subprogram_name.lower() in\
Fortran_builtins + list(SourceManager.list_of_intrinsics):
continue
try:
var = vault.variables_dictionary[subprogram_name]
# Match variables ensuring that it is not the ret_val of a function with the same name
if var[0].procedure.name != var[0].name:
continue
except KeyError:
pass
# check is not a variable, a builtin function or an intrinsic
if not NatureDeterminer.is_subprogram(subprogram_name, procedure, vault):
continue
# Get the actual call
call = re.sub(r'\bcall\b', '', BasicFunctions.close_brackets(string[m.start():]), flags=re.I)
# Find called arguments
called_arguments = find_call_arguments(call)
# We ruled out the possibility of variables or intrinsics: must be a subprogram
try:
subprogram = vault.procedures_dictionary[subprogram_name]
except KeyError:
unknown_subprogram.append(subprogram_name)
continue
# Create an external subprogram to be returned as the error
subprogram = [procedure_type(subprogram_name, module=None, is_external=True)]
is_unknown_external = True
if len(subprogram) > 1:
subprogram = [s for s in subprogram if s.module.name == procedure.module.name]
subprogram = subprogram[0]
call = re.sub(r'\bcall\b', '', BasicFunctions.close_brackets(string[m.start():]), flags=re.I)
subprogram_call = BasicStructures.SubprogramCall(call.strip(), subprogram)
called_arguments = find_call_arguments(subprogram_call.call)
if not isinstance(subprogram, procedure_type):
# Search for the correct subprogram
if isinstance(subprogram_call.subprogram, BasicStructures.Interface):
subprogram_call.subprogram = Getter.get_interface_from_arguments(called_arguments,
interface=subprogram_call.subprogram,
procedure=procedure,
vault=vault)
if isinstance(subprogram, BasicStructures.Interface):
subprogram = Getter.get_interface_from_arguments(called_arguments,
interface=subprogram,
procedure=procedure,
vault=vault)
# If the subprogram is not of the correct type, skip the occurrence
if not isinstance(subprogram_call.subprogram, procedure_type):
if not isinstance(subprogram, procedure_type):
continue
elif isinstance(subprogram, BasicStructures.DerivedType):
continue
else:
raise Error.ProcedureNotFound("The subprogram %s could not be found", subprogram_name)
for index, argument in enumerate(called_arguments):
key = None
try:
if Getter.is_keyword_argument(argument):
[key, argument] = argument.split("=", 1)
key = key.strip()
argument = argument.strip()
var_type = Getter.get_type_of_contents(argument, procedure=procedure, vault=vault)
else:
var_type = Getter.get_type_of_contents(argument, procedure=procedure, vault=vault)
except Error.VariableNotFound:
continue
if not subprogram_call.subprogram.is_external:
if key is not None:
dummy_var = subprogram_call.subprogram.get_variable_by_name(key)
else:
dummy_var = subprogram_call.subprogram.get_variable_by_position(index)
subprogram_call.dummy_arguments.append(dummy_var)
try:
var = Getter.get_variable(argument, procedure=procedure, vault=vault)
except Error.VariableNotFound:
var = None
arg = BasicStructures.Argument(argument, var_type, var)
subprogram_call.arguments.append(arg)
to_return.append(subprogram_call)
# Return all the unknown found in line
if unknown_subprogram:
raise Error.ProcedureNotFound(unknown_subprogram)
subprogram_call = create_SubprogramCall(call, called_arguments, subprogram, procedure, vault)
if is_unknown_external:
external_subprogram.append(subprogram_call)
else:
to_return.append(subprogram_call)
# Raise an error returning all the unknown subprogram calls found in line
if external_subprogram:
raise Error.ProcedureNotFound(external_subprogram)
return to_return
def create_SubprogramCall(call, called_arguments, subprogram, procedure, vault):
import AutoRPE.UtilsRPE.Getter as Getter
subprogram_call = BasicStructures.SubprogramCall(call.strip(), subprogram)
for index, argument in enumerate(called_arguments):
key = None
try:
if Getter.is_keyword_argument(argument):
[key, argument] = argument.split("=", 1)
key = key.strip()
argument = argument.strip()
var_type = Getter.get_type_of_contents(argument, procedure=procedure, vault=vault)
else:
var_type = Getter.get_type_of_contents(argument, procedure=procedure, vault=vault)
except Error.VariableNotFound:
# An external variable has been used
arg = BasicStructures.Argument(argument, type="external", var=None)
subprogram_call.arguments.append(arg)
continue
if not subprogram_call.subprogram.is_external:
if key is not None:
dummy_var = subprogram_call.subprogram.get_variable_by_name(key)
else:
dummy_var = subprogram_call.subprogram.get_variable_by_position(index)
subprogram_call.dummy_arguments.append(dummy_var)
try:
var = Getter.get_variable(argument, procedure=procedure, vault=vault)
except Error.VariableNotFound:
var = None
arg = BasicStructures.Argument(argument, var_type, var)
subprogram_call.arguments.append(arg)
return subprogram_call
......@@ -55,14 +55,17 @@ def get_variable(contents, procedure=None, vault=None):
raise Error.VariableNotFound("Variable %s: not found" % contents)
def get_real_in_line(line, vault, procedure, intent=["out", "inout"]):
def get_type_in_line(line, vault, procedure, intent=["out", "inout"], var_type=VariablePrecision.real_id):
"""
Find all variables of type var_type in calls or pointer assignation, and return them + their corresponding dummies
"""
subprogram_call = CallManager.get_procedure_inline(line, vault, procedure)
ret_dummy = []
ret_var = []
if not subprogram_call:
if NatureDeterminer.is_pointer_assignment(line):
ptr, target = get_pointer_target(line, procedure, vault)
if [ptr, target] != [None, None] and ptr.type in VariablePrecision.real_id:
if [ptr, target] != [None, None] and ptr.type in var_type:
# This is needed because we do not have an argument/dummy but we need to copy the 'else' structure
arg_trg = BasicStructures.Argument(line.split("=>")[1], target.type, target)
return [[ptr], [arg_trg]]
......@@ -71,7 +74,7 @@ def get_real_in_line(line, vault, procedure, intent=["out", "inout"]):
if not s_call.subprogram.is_external:
for dummy_a, arg in zip(s_call.dummy_arguments, s_call.arguments):
if dummy_a.intent in intent:
if dummy_a.type in VariablePrecision.real_id:
if dummy_a.type in var_type:
ret_dummy.append(dummy_a)
ret_var.append(arg)
# Check something was found
......