Commit 5c3abca1 authored by erodrigu's avatar erodrigu
Browse files

Merged develop

parents 47a9d63d 83b5dede
......@@ -34,5 +34,5 @@ if __name__ == "__main__":
rmtree(path_to_output_sources)
mkdir(path_to_output_sources)
Preprocessor.pre_process_sources(path_to_input_sources, path_to_output_sources)
Preprocessor.clean_sources(path_to_input_sources, path_to_output_sources)
print("\nClean sources created in ", abspath(path_to_output_sources))
......@@ -51,7 +51,7 @@ def get_command_line_arguments():
from AutoRPE.UtilsRPE.Error import PathType
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input-sources", required=True, type=PathType(exists=True, type='dir'),
help="Path to a folder that contains the preprocessed cleaned code source")
help="Path to a folder that contains the preprocessed code source")
parser.add_argument("-o", "--output-sources", default="SourcesWithRPE",
help="Path to the folder that will contain the AutoRPE implementation ")
parser.add_argument("--output-vault", default="vault.pkl",
......@@ -70,7 +70,7 @@ if __name__ == "__main__":
path_to_output_sources = args.output_sources
output_vault = args.output_vault
namelist_path = args.namelist_path
# Set working precision to single
# Set working precision
VariablePrecision.set_working_precision('dp')
# Paths to source folders
if isdir(path_to_output_sources):
......
......@@ -116,10 +116,8 @@ def get_pointer_target(line, procedure, vault):
return ptr, target
# Retrieve variable corresponding to a structure member
def get_structure_member(contents, procedure, vault):
"""Retrieve variable corresponding to last member of a structure"""
# The first part will be the root
split_contents = contents.split("%", 1)
......@@ -234,10 +232,15 @@ def get_type_of_contents(contents, procedure, vault):
num_type = NatureDeterminer.is_a_number(contents)
if num_type:
return num_type
# # This function is vault agnostic, but the possibiliy of a function call
# array = NatureDeterminer.is_an_array(contents)
# if array:
# # Now the array will match with a simple variable
# contents = array
# Remove the indexing from array and DT:
# ght_abl <-- ght_abl(2:jpka)
# todelay%z1d <-- todelay(ji)%z1d(:)
# ght_abl(2:jpka+b) --> ght_abl
# todelay(ji)%z1d(:) --> todelay%z1d
clean_contents = remove_indexing(contents)
# Search if is a simple variable
......@@ -468,14 +471,24 @@ def get_type_of_real_cast(string):
if len(arguments) == 1:
return VariablePrecision.real_id['wp']
elif len(arguments) == 2:
argument, type = arguments
return VariablePrecision.lookup_table[type]
argument, arg_type = arguments
# In case the type is specified along with keyword
arg_type = re.sub(r'kind=', '', arg_type, flags=re.I)
return VariablePrecision.lookup_table[arg_type.lower()]
def remove_indexing(content):
# If the variable is already a simple name
if re.match(RegexPattern.variable, content):
# Case of a recursive call
content = content.replace("XXX", "%")
return content
# If is a simple array with indexing
array = NatureDeterminer.is_an_array(content)
if array:
# Case of a recursive call
array = array.replace("XXX", "%")
return array
# Avoid problem with structure used as index
content = content.replace("%", "XXX")
# Used to avoid recursion error
......
import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions
import AutoRPE.UtilsRPE.BasicStructures as BasicStructures
import AutoRPE.UtilsRPE.Splitter as Splitter
import AutoRPE.UtilsRPE.Getter as Getter
import AutoRPE.UtilsRPE.NatureDeterminer as NatureDeterminer
......@@ -12,7 +11,6 @@ import AutoRPE.UtilsRPE.Vault as Vault
import AutoRPE.UtilsRPE.TrackDependency as TrackDependency
import re
# Hardcoded list, this should be solved
list_of_intrinsics_2fix = ["max", "maxloc", "maxval", "min", "minloc", "minval", "epsilon",
"isnan", "count", "abs", "mod", "reshape",
......@@ -243,9 +241,23 @@ class ImplementRPE:
if not read_arguments:
continue
replacement = read_arguments.replace(" ", "")
# Add parenthesis because the read() don't use them
# Add parenthesis because the read() doesn't use them
arguments = CallManager.find_call_arguments("(" + replacement + ")")
# Further separate arguments. Used in case of loops READ(inum,*) ( tpifl (jfl), jfl=1, jpnrstflo)
list_arguments = []
index_to_remove = []
for argument_index, argument in enumerate(arguments):
if BasicFunctions.is_all_inside_parenthesis(argument):
index_to_remove.append(argument_index)
argument = CallManager.find_call_arguments(argument)
list_arguments.extend(argument)
# Remove the original argument
for i in index_to_remove[::-1]:
del arguments[i]
# Add the split argument
arguments.extend([i for i in list_arguments])
# Check the argument type and replace just if is an rpe
for argument_index, argument in enumerate(arguments):
argument_type = Getter.get_type_of_contents(argument, procedure=module.blocks[line_index],
......@@ -262,7 +274,8 @@ class ImplementRPE:
for l_index, line in enumerate(module.lines):
if re.search("(?<!\w)(WRITE) *\(", line, re.IGNORECASE):
wc = BasicFunctions.close_brackets(line[line.lower().find("write"):])
wc = BasicFunctions.close_brackets(line[re.search(RegexPattern.name_occurrence % "write",
line, re.I).start():])
line = line.replace(wc, "WRITE()")
matches = re.search(pattern, line.strip(), re.IGNORECASE)
......@@ -443,6 +456,7 @@ class ImplementRPE:
return line
def fix_subprogram_calls(self, module):
# Returns arguments of subprograms call and corresponding type and dummy
for _index, line in enumerate(module.lines):
if_condition, conditional_piece = BasicFunctions.split_if_condition(line)
for piece in if_condition, conditional_piece:
......@@ -456,7 +470,6 @@ class ImplementRPE:
line = line.replace(piece, fix)
module.lines[_index] = line
def _find_kind_of_parameter_declaration(self, argument, procedure):
"""Return the nature of the parameter declaration. Depending on the type, the fix will be different"""
# These are real, but need some manipulation before adding the RPE constructor
......@@ -530,7 +543,9 @@ class ImplementRPE:
for m in matches:
fun_name = m.groups()[0]
try:
return self.vault.get_derived_type_constructor(fun_name)
derived_type = self.vault.get_derived_type_constructor(fun_name)
if derived_type:
return derived_type
except Error.DerivedTypeNotFound:
pass
return None
......@@ -12,12 +12,14 @@ import re
import itertools
from os.path import join, isfile
def insert_use_rp_emulator_in_function(func_name, module):
for idx, line in enumerate(module.lines):
if re.search(RegexPattern.subprogram_name_declaration % func_name, line, flags=re.IGNORECASE):
module.lines.insert(idx + 1, "USE rp_emulator")
return
def insert_use_statement(vault):
# Adding module rp_emulator (in theory adding this to par_kind.f90 should be enough,
# but theory fails and compiler complains so I'll add it everywhere
......@@ -37,6 +39,7 @@ def insert_use_statement(vault):
interface.module.rebuild_text()
break
def create_read_precisions_module(path_to_sources, vault=None, target_module=None):
from os.path import dirname, abspath
print("\n\nCreating module to read precision namelist...")
......@@ -121,7 +124,8 @@ def _add_argument_truncation(line, subprogram_call, counter):
arg.variable.mutable = True
counter.up()
variables_to_truncate.append([arg.name.replace("%val", "").strip(), arg.variable.id, arg.variable.is_pointer])
variables_to_truncate.append(
[arg.name.replace("%val", "").strip(), arg.variable.id, arg.variable.is_pointer])
else:
raise Error.IntentOutError("The dummy argument %s of procedure %s has intent %s but is used "
"like %s "
......@@ -137,12 +141,21 @@ def _add_argument_truncation(line, subprogram_call, counter):
arg.variable.mutable = True
counter.up()
# Add it to the array of variable to truncate
variables_to_truncate.append([arg.name.replace("%val", "").strip(), arg.variable.id, arg.variable.is_pointer])
variables_to_truncate.append(
[arg.name.replace("%val", "").strip(), arg.variable.id, arg.variable.is_pointer])
for var, id_num, is_pointer in variables_to_truncate:
call_to_subroutine += "\n"
if is_pointer:
call_to_subroutine += "if(associated(%s)) then\n" % var
# Remove indexing to avoid
# error #7835: Record fields or array elements or sections of pointers are not themselves pointers.
if var.count("%"):
indexed_member = var.split("%")[-1]
member_name = Getter.remove_indexing(indexed_member)
deindexed_var = var.replace(indexed_member, member_name)
else:
deindexed_var = Getter.remove_indexing(var)
call_to_subroutine += "if(associated(%s)) then\n" % deindexed_var
call_to_subroutine += if_condition + " %s%%sbits = emulator_variable_precisions(%i) \n" % (var, id_num)
call_to_subroutine += if_condition + " CALL apply_truncation(%s)" % var
if is_pointer:
......@@ -158,8 +171,9 @@ def insert_variable_precision_specification(variable, module, insertion_line, co
if variable.is_pointer and not variable.allocatable:
module.lines[insertion_line] += "\nif(associated(%s)) then" % var_name
# Add sbits
module.lines[insertion_line] += "\n" + condition + " %s%%sbits = emulator_variable_precisions(%i)" % (var_name, variable.id)
# Add sbits
module.lines[insertion_line] += "\n" + condition + " %s%%sbits = emulator_variable_precisions(%i)" % (
var_name, variable.id)
# Add apply_truncation
module.lines[insertion_line] += "\n" + condition + " CALL apply_truncation(%s)" % var_name
......@@ -172,6 +186,9 @@ def insert_variable_precision_specification(variable, module, insertion_line, co
def _get_member_name(arg, module, line_index, vault, variable):
# Remove leading spaces
arg = arg.replace(" ", "").strip()
# Get variable name without indexing (foo(n)%var --> foo%var)
clean_contents = Getter.remove_indexing(arg)
member_type = Getter.get_type_of_contents(clean_contents, module.blocks[line_index], vault)
......@@ -207,7 +224,7 @@ def insert_variable_precision_specification_to_allocatable(variable, counter_obj
continue
else:
# Check that we did not match a member with the same name: a_i will also match %a_i
if re.search(RegexPattern.allocation_member_name % variable.name, line.replace(" ",""), re.I):
if re.search(RegexPattern.allocation_member_name % variable.name, line.replace(" ", ""), re.I):
continue
variable.id = counter_object.count
......@@ -519,6 +536,35 @@ def find_call_in_original_file(subprogram, argument, argument_index, _file, targ
return False
def find_subprogram_lines(subprogram, module):
indices = []
# if module is None:
# module = subprogram.module
begin_pattern = RegexPattern.subprogram_name_declaration % subprogram.name
end_pattern = RegexPattern.subprogram_name_end_declaration % subprogram.name
# First find the subprogram body inside the module
for index, line in enumerate(module.lines):
# If it's the start or end of the subprogram
line = BasicFunctions.remove_comments(line)
if re.search(begin_pattern, line, re.I):
indices.append(index)
# End of the program was hit
if re.search(end_pattern, line, re.I):
break
if not len(indices):
raise Error.ProcedureNotFound
# In case more than one start/end index was found:
#
# #if defined key_agrif
# RECURSIVE SUBROUTINE stp_MLF( )
# INTEGER :: kstp ! ocean time-step index
# #else
# SUBROUTINE stp_MLF( kstp )
# INTEGER, INTENT(in) :: kstp ! ocean time-step index
# #endif
#
# return the broader range
return indices[0], indices[-1]
## PRIVATE
......@@ -530,74 +576,60 @@ class Counter:
def up(self, how_much=1):
self.count += how_much
def find_first_use(variable):
procedure = variable.procedure
module = procedure.module
subprogram = variable.procedure
module = subprogram.module
# pattern = '(^|\W+)%s(\W+|$)' % variable.name
pattern = r'\W*%s\W*' % variable.name
searching = False
loop_level = 0
loop_start = None
in_where_statement = False
where_level = 0
where_statement_start = None
# Scroll all the lines
for line_index, line in enumerate(module.lines):
line = line.strip().lower()
# Until it finds the subroutine to which the variables belongs (routine.name)
if not searching:
match = re.search(r"(subroutine|function)\s+\b%s\b" % procedure.name, line, re.IGNORECASE)
if match:
searching = True
# In the subroutine
else:
# Scroll all the lines searching for the first use
if re.search(pattern, line, re.IGNORECASE):
# skip declarations: bad workaround, but avoids false positive that will block compilation.
if line.count("::") or line.lower().count("implicit none"):
continue
# Skip the first line
elif line.count("subroutine") or line.count("function"):
# Reach the end: the variable has not been used
if line.count("end subroutine") or line.count("end function"):
break
continue
# Skip import statements
elif line.find("use ") == 0:
continue
# Other statement
elif line.find("data ") == 0:
continue
elif _check_if_condition(line, variable.name):
continue
# Find start and end of subprogram declaration
indices = find_subprogram_lines(subprogram, module)
elif in_where_statement:
# Inside a where statement
return module, where_statement_start
elif loop_level > 0:
# In a (nested) loop
return module, loop_start
else:
return module, line_index
# Search for first use, avoiding subprogram declaration and end lines
# We will need to add indices[0] - 1 to the line_index return value since that's our starting index
for line_index, line in enumerate(module.lines[indices[0] + 1:indices[1]]):
# Skip all declarations to avoids false positive that will block compilation.
if line.count("::") or line.lower().count("implicit none") or line.find("use ") == 0:
continue
elif line.count("end subroutine"):
break
# Whenever a 'do loop' starts increment nesting index
elif re.search(r"^ *do *[\w]* *=", line, re.IGNORECASE):
if loop_level == 0:
loop_start = line_index
loop_level += 1
# Whenever a 'do loop' ends decrement nesting index
# This way match bot the expression 'end do' and 'enddo' but excludes matches like 'enddo loop_name'
elif re.search(r"^ *end\s*do", line, re.IGNORECASE):
loop_level -= 1
# Do the same for where statement
elif line.find("where") == 0:
in_where_statement = True
# Scroll subprogram's lines searching for the first use
if re.search(RegexPattern.name_occurrence % variable.name, line, re.IGNORECASE):
# Check the variable is not mentioned in a ".NOT.PRESENT" statement
if _check_if_condition(line, variable.name):
continue
# The variable is found: : return line -1, because it will be inserted at the end of that line
if where_level > 0:
# Inside a where statement
return module, indices[0] + where_statement_start
else:
# In a (nested) loop
if loop_level > 0:
return module, indices[0] + loop_start
# On a normal line
else:
return module, indices[0] + line_index
# Whenever a 'do loop' starts increment nesting index
elif re.search(r"^ *do *[\w]* *=", line, re.IGNORECASE):
if loop_level == 0:
loop_start = line_index
loop_level += 1
# Whenever a 'do loop' ends decrement nesting index
# This way match bot the expression 'end do' and 'enddo' but excludes matches like 'enddo loop_name'
elif re.search(r"^ *end\s*do", line, re.IGNORECASE):
loop_level -= 1
# Do the same for where statement
elif line.lower().strip().find("where") == 0:
if where_level == 0:
where_statement_start = line_index
elif line.find("end where") == 0:
in_where_statement = False
where_level += 1
elif line.lower().strip().find("end where") == 0:
where_level -= 1
# The variable was not found in the subroutine
return None, None
......
......@@ -226,41 +226,10 @@ def find_structure_declarations_index(structure, module):
return [start, end]
def find_subprogram_lines(subprogram, module):
indices = []
# if module is None:
# module = subprogram.module
begin_pattern = RegexPattern.subprogram_name_declaration % subprogram.name
end_pattern = RegexPattern.subprogram_name_end_declaration % subprogram.name
# First find the subprogram body inside the module
for index, line in enumerate(module.lines):
# If it's the start or end of the subprogram
line = BasicFunctions.remove_comments(line)
if re.search(begin_pattern, line, re.I):
indices.append(index)
# End of the program was hit
if re.search(end_pattern, line, re.I):
break
if not len(indices):
raise Error.ProcedureNotFound
# In case more than one start/end index was found:
#
# #if defined key_agrif
# RECURSIVE SUBROUTINE stp_MLF( )
# INTEGER :: kstp ! ocean time-step index
# #else
# SUBROUTINE stp_MLF( kstp )
# INTEGER, INTENT(in) :: kstp ! ocean time-step index
# #endif
#
# return the broather range
return indices[0], indices[-1]
def find_subprogram_declarations_index(subprogram, module):
"""Find start and end indexes of variables declaration in subprogram code """
# Find start and end of subprogram declaration
indices = find_subprogram_lines(subprogram, module)
indices = Inserter.find_subprogram_lines(subprogram, module)
start = -1
end = 0
......@@ -452,7 +421,7 @@ def arguments_to_cast(subprogram_call):
if dummy_a.intent == "in":
# If is wp, maintain wp
# If is sp for sure there is a dp version of this, otherwise the code would not work in dp
if dummy_a.type in [VariablePrecision.real_id['sp'], VariablePrecision.real_id['wp']]:
if dummy_a.type in [VariablePrecision.real_id['sp'], VariablePrecision.real_id['wp']]:
cast_precision = "wp"
else:
# There is no sp/dp interface (otherwise we would be in the other branch of the if/else)
......
......@@ -46,8 +46,8 @@ def is_subprogram(subprogram_name, procedure, vault):
return True
# Check if a string is a number and returns the type
def is_a_number(string):
"""Check if a string is a number and returns the type"""
# Check if is just white spaces
if not string.strip():
......@@ -88,6 +88,18 @@ def is_a_number(string):
return False
def is_an_array(string, vault=None):
""" Check if the string is an array with indexing"""
starts_with_array = re.search(r'^\s*(\w+)\(', string, re.I)
if starts_with_array:
# Check if the entire expression is just an array
array = BasicFunctions.close_brackets(string[starts_with_array.start():])
if array == string:
# Returns the name of the array or the function
return starts_with_array.group(1)
return None
def is_an_hardcoded_array(string):
match = re.match(RegexPattern.hardcoded_array, string)
if match:
......@@ -138,6 +150,7 @@ def has_operations(string):
return False
if CallManager.is_call_to_function(string):
return False
# Remove the unary operators
if string[0] == '-' or string[0] == '+':
string = string[1:]
......@@ -227,6 +240,10 @@ def remove_arrays_and_structures(string):
replacement_index = 0
arrays = True
replacement_list = []
# Check if the entire string is an array
array = is_an_array(string)
if array:
return "REPLACEMENT"
# Checks for all array pattern, and substitute with string
while arrays:
arrays = re.findall(RegexPattern.array, string)
......
......@@ -35,7 +35,7 @@ def parse_sources(file_list, vault, extension):
for index, module in enumerate(vault.modules):
find_interfaces(module, vault)
# # Fill vault variable list:
# Fill vault variable list:
for procedures in vault.procedures_dictionary.values():
for proc in procedures:
if not isinstance(proc, BasicStructures.Interface):
......@@ -136,7 +136,8 @@ def find_subprogram_declaration(module, _subprogram_start, _subprogram_end, obj_
stop_index = [index
for index, line in enumerate(module.lines[_subprogram_start:_subprogram_end + 1])
if re.search(RegexPattern.contains_declaration, line, re.I) or re.search(RegexPattern.interface_declaration, line, re.I)]
if re.search(RegexPattern.name_occurrence % 'contains', line, re.I)
or re.search(RegexPattern.name_occurrence % 'interface', line, re.I)]
# A declaration found stop the search there
if stop_index:
_subprogram_end = _subprogram_start - 1 + stop_index[0]
......@@ -146,7 +147,7 @@ def find_subprogram_declaration(module, _subprogram_start, _subprogram_end, obj_
# Lines that are variable declarations
_all_declaration = [_l.strip() for _l in _subprogram_lines if
re.search(obj_to_find, _l, re.I) and not
re.search(obj_to_find, BasicFunctions.remove_literals(_l), re.I) and not
(re.search(RegexPattern.general_allocation, _l, re.I) or
re.search(RegexPattern.string_array_initialization, _l, re.I))]
......@@ -296,11 +297,11 @@ def find_subroutines(module, vault):
used_modules, subprogram_type=BasicStructures.SubRoutine)
# Check if subroutines are defined inside this scope
subroutine_lines = module.lines[subroutine_start+1:subroutine_end-1]
internal_sbr = [re.search(RegexPattern.subroutine_declaration, l, re.I) for l in subroutine_lines
if re.search(RegexPattern.subroutine_declaration, l, re.I)]
if internal_sbr:
sbr_names = [i_s.group(1) for i_s in internal_sbr]
subroutine.declares = sbr_names
internal_subprogram = [re.search(RegexPattern.subprogram_declaration, l, re.I) for l in subroutine_lines
if re.search(RegexPattern.subprogram_declaration, l, re.I)]
if internal_subprogram:
subprogram_names = [i_s.group(2) for i_s in internal_subprogram]
subroutine.declares = subprogram_names
vault.add_procedure(subroutine)
......@@ -326,16 +327,17 @@ def find_external_functions(module, vault):
for index, line in enumerate(module.lines):
# Check if there is a call to function in line
# the replace spaces is a workaround for the regex to work: it contains a lookahead
line = BasicFunctions.remove_if_condition(line)
func_call = re.search(RegexPattern.call_to_function, re.sub('\\s\\s+', r' ', line.strip()), re.I)
if_condition, conditional_piece = BasicFunctions.split_if_condition(line)
for piece in if_condition, conditional_piece:
func_call = re.search(RegexPattern.call_to_function, re.sub('\\s\\s+', r' ', piece.strip()), re.I)
if func_call and not (line.count("::") or NatureDeterminer.is_function_declaration_statement(line)):
# Check if the function called is external
try:
CallManager.get_procedure_inline(line, vault, module.blocks[index], only_function=True)
if func_call and not (piece.count("::") or NatureDeterminer.is_function_declaration_statement(piece)):
# Check if the function called is external
try:
CallManager.get_procedure_inline(piece, vault, module.blocks[index], only_function=True)
except Error.ProcedureNotFound as unknown_function_calls:
add_external_subprogram(unknown_function_calls, vault)
except Error.ProcedureNotFound as unknown_function_calls:
add_external_subprogram(unknown_function_calls, vault)
def add_external_subprogram(unknown_calls, vault):
......@@ -353,7 +355,6 @@ def add_external_subprogram(unknown_calls, vault):
def find_types(module, vault):
# Looks into the module lines for type declaration
import AutoRPE.UtilsRPE.RegexPattern as RegexPatter
# Look into each line
in_type_declaration = False
derived_types = []
......@@ -363,7 +364,7 @@ def find_types(module, vault):
# Look if routine starts with subroutine
if not in_type_declaration:
# check if a type has been declared
m = re.search(RegexPatter.simple_type_declaration, line, re.IGNORECASE)
m = re.search(RegexPattern.simple_type_declaration, line, re.IGNORECASE)
if m:
in_type_declaration = True
variable_counter = 0
......@@ -372,7 +373,7 @@ def find_types(module, vault):
derived_types.append(derived_type)
# Case in which a type is extending another type
match_extend = re.search(RegexPatter.extended_type_declaration, line, re.IGNORECASE)
match_extend = re.search(RegexPattern.extended_type_declaration, line, re.IGNORECASE)
if match_extend:
in_type_declaration = True
variable_counter = 0
......@@ -382,7 +383,7 @@ def find_types(module, vault):
derived_types.append(derived_type)
else:
# Check if we are exiting a type declaration
m = re.search(RegexPatter.end_type_statement, line, re.IGNORECASE)
m = re.search(RegexPattern.end_type_statement, line, re.IGNORECASE)
if m: