Commit 31cb9c07 authored by sparonuz's avatar sparonuz
Browse files

# This is a combination of 3 commits.

[Issue#104]: Function merge_lines was basically defined twice: removed one and update the one inside Inserter. Removed more useless functions inside InterfaceGenerator.py.

[Issue#104]: Starting from the question "Is it Finder.py the best file to have that function?" I decided that the file was useless, and ridistribute the function in it.
parent 2412c1e5
......@@ -16,179 +16,117 @@
# * bar(sp)
#
# Import classes and functions from UtilsRPE
from AutoRPE.UtilsRPE.CurrentBlock import CurrentBlock
from AutoRPE.UtilsRPE.Inserter import replace_real_declaration
from AutoRPE.UtilsRPE.SourceManager import preprocess_sources, parse_sources, load_vault
from AutoRPE.UtilsRPE.Cleaner import remove_comments
from AutoRPE.UtilsRPE.Finder import find_declaration_line
import AutoRPE.UtilsRPE.RegexPattern as re_pattern
import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions
import AutoRPE.UtilsRPE.Vault as Vault
import AutoRPE.UtilsRPE.RegexPattern as RegexPattern
from argparse import ArgumentParser
from os import mkdir
from os.path import join, isdir
from shutil import rmtree
from os.path import join, isdir, abspath
import re
import itertools
def get_command_line_arguments():
from argparse import ArgumentParser
help_text = """
This script takes as arguments the input source (-s/--input-source)
and the name of a routine (-r/--routine) and creates different versions
of this routine accouting for all different precision combinations that can happen.
of this routine accounting for all different precision combinations that can happen.
An additional argument for the vault (-v/--vault) allows to skip the
creation of the vault if this is provided. Optional output (-o/--output-folder) for creating
the files there, if not given the outputs are printed out.
"""
parser = ArgumentParser(usage=help_text)
parser.add_argument("-v", "--vault", dest="vault", required=False, help="Path to the vault")
parser.add_argument("-o", "--output-folder", dest="output_folder", required=False, help="Path to outputs folder")
parser.add_argument("-s", "--input-source", dest="input_source", required=True,
help="Path to the processed sources")
parser.add_argument("-r", "--routine", dest="routine_name", required=True,
help="Name of the routine to create different interfaces")
parser.add_argument("-v", "--vault-path", required=True,
help="Path to a vault generated with MakeVault")
parser.add_argument("-i", "--input-sources", required=True,
help="Path to the processed sources, i.e. sources with comments")
parser.add_argument("-s", "--subprogram-name", required=True,
help="Name of the subroutine/function to create different interfaces")
parser.add_argument("-o", "--output-folder", default="./",
help="Path to folder in which new code will be stored")
args = parser.parse_args()
return args.vault, args.input_source, args.routine_name, args.output_folder
# Few functions
def merge_declaration_lines(code_text):
lines = [l for l in code_text.split("\n")]
for _index, _line in enumerate(lines):
line = remove_comments(_line).strip()
if line.count("&") and line.count("::"):
i = 1
while line and line.strip()[-1] == "&":
line1 = line
line2 = remove_comments(lines[_index + i]).strip()
line = line1.replace("&", "").rstrip() + line2
i += 1
line = line.replace("&", "")
line = re.sub(", +", ", ", line)
line = re.sub(": +", ": ", line)
lines[_index] = line
for j in range(1, i):
lines[_index + j] = ""
return "\n".join(lines).strip()
def replace_subroutine_name(old_name, new_name, code_text):
pattern = re_pattern.subroutine_declaration % old_name
pat = re.compile(pattern, re.I)
replacement = r"\1 %s" % new_name
return pat.sub(replacement, code_text)
def obtain_routine_source(routine, sources_path):
source_file = join(sources_path, "%s.f90" % routine.module.name)
return args
def obtain_subprogram_source(subprogram, sources_path):
source_file = join(sources_path, subprogram.module.filename)
indices = []
with open(source_file) as f:
pattern = re_pattern.subroutine_declaration % routine.name
lines = [l for l in f]
for index, line in enumerate(lines):
line = remove_comments(line).strip()
# if it's the start or end of the routine
if re.search(pattern, line, re.I):
indices.append(index)
routine_code = "".join(lines[indices[0]:indices[1]+1])
routine_code = merge_declaration_lines(routine_code)
return routine_code, indices
def replace_routine_name(old_name, new_name, code_text):
pattern = re_pattern.subroutine_declaration % old_name
pat = re.compile(pattern, re.I)
replacement = r"\1 %s" % new_name
return pat.sub(replacement, code_text)
def prepare_vault(vault_path=None):
path_to_preprocessed_sources = "./tmp_processedsources/"
# In case no vault has been provided it will create it from the sources in the input sources folder
if vault_path is None:
# Paths to source folders
# Path_to_input_sources
if not isdir(path_to_preprocessed_sources):
mkdir(path_to_preprocessed_sources)
# Format text + replace real declarations with type(rpe_var)
preprocess_sources(path_to_input_sources, path_to_preprocessed_sources)
# Obtain source information
vault = parse_sources(path_to_preprocessed_sources, save_vault=True)
# Remove generated folder
rmtree(path_to_preprocessed_sources)
else:
vault = load_vault(vault_path)
return vault
def get_routine(vault, routine_name=None):
# Find procedure with the routine name
routine = [p for p in vault.procedures if p.name == routine_name]
# Check if we found it
if len(routine) == 0:
print("Routine %s not found in vault." % routine_name)
exit(0)
return routine[0]
def update_line(version_lines, variable, filepath, vault, routines_indices, variable_precision):
# Find the declaration/original lines
d_lines = find_declaration_line(variable, filepath, vault)
# Search in declaration lines
for d_line in d_lines:
# Update line if it's between routine lines routine
if d_line >= routines_indices[0] and d_line <= routines_indices[1]:
original_line = version_lines[d_line - routines_indices[0]]
new_line = replace_real_declaration(variable, original_line, variable_precision)
version_lines[d_line - routines_indices[0]] = new_line
def generate_routine_versions(precision_combinations, sensitive_variables, vault, routine_lines, routine, filepath, indices):
code_versions = {}
pattern = RegexPattern.subprogram_name_declaration % subprogram.name
lines = [l for l in f]
for index, line in enumerate(lines):
line = BasicFunctions.remove_comments(line).strip()
# if it's the start or end of the subprogram
if re.search(pattern, line, re.I):
indices.append(index)
if line.lower().count("end"):
break
subprogram_code = lines[indices[0]:indices[1] + 1]
for index, line in enumerate(subprogram_code):
if line.count("::") and line.count("&"):
subprogram_code[index] = BasicFunctions.merge_line_range(subprogram_code, index)
for i in range(BasicFunctions.merge_line_range.number-1):
# Remove the merged line from the list
subprogram_code.pop(index + 1)
return subprogram_code
def generate_routine_versions(precision_combinations, sensitive_variables, subprogram, subprogram_code, output_folder):
code_versions = []
for c_index, combination in enumerate(precision_combinations):
new_routine_name = "%s_%04i" % (routine.name, c_index)
new_subprogram_name = "%s_%04i" % (subprogram.name, c_index)
new_subprogram_code = subprogram_code[:]
# Search and update real declaration
for index, variable in enumerate(sensitive_variables):
update_line(routine_lines, variable, filepath, vault, indices, combination[index])
# Join the routine lines and uodate nane
version_text = "\n".join(routine_lines).strip()
version_text = replace_routine_name(routine.name, new_routine_name, version_text)
code_versions[new_routine_name] = version_text
return code_versions
def write_routine_versions(code_versions, output_folder):
if output_folder is None:
# If there's no folder, print the outputs to terminal
for key, code in code_versions.items():
print(code)
else:
# Create folder if it doesn't exists
if not isdir(output_folder):
mkdir(output_folder)
# Create a file for each subroutine interface
for key, code in code_versions.items():
with open(output_folder + key, 'w') as f:
f.write(code)
# Find the declaration lines
line_index = \
[idx for idx, l in enumerate(new_subprogram_code) if re.search(r".*::.*\b%s\b" % variable.name, l, re.I)][0]
# Update line
new_subprogram_code[line_index] = replace_real_declaration(variable,
new_subprogram_code[line_index],
combination[index])
# Update name
new_subprogram_code[0] = new_subprogram_code[0].replace(subprogram.name, new_subprogram_name)
new_subprogram_code[-1] = new_subprogram_code[-1].replace(subprogram.name, new_subprogram_name)
code_versions.append("".join(new_subprogram_code))
if not isdir(output_folder):
mkdir(output_folder)
# Create a file for each subroutine interface
with open(output_folder + subprogram_name + "_interfaces.f90", 'w') as f:
for versions in code_versions:
f.write(versions)
f.write("\n\n")
print("\nAlternative procedure for %s have been written to %s" % (subprogram_name, abspath(output_folder)))
if __name__ == "__main__":
# Get command line arguments
vault_path, path_to_input_sources, routine_name, output_folder = get_command_line_arguments()
# Get the vault
vault = prepare_vault(vault_path)
# Get routine from vault
routine = get_routine(vault, routine_name)
# Get the routine code
routine_code, indices = obtain_routine_source(routine, path_to_input_sources)
args = get_command_line_arguments()
vault_path = args.vault_path
path_to_input_sources = args.input_sources
subprogram_name = args.subprogram_name
output_folder = args.output_folder
# Load vault and retrive the object
vault = Vault.load_vault(vault_path)
subprogram = vault.get_procedure_by_name(subprogram_name)
# Get the subprogram code
subprogram_code = obtain_subprogram_source(subprogram, path_to_input_sources)
# All possible combinations of precisions x sensitive variables
sensitive_variables = [v for v in routine.variables if v.is_dummy_argument and v.type == "double"]
sensitive_variables = [v for v in subprogram.variables if v.is_dummy_argument and v.type == "double"]
precision_combinations = list(itertools.product(["dp", "sp"], repeat=len(sensitive_variables)))
# Get filepath
filepath = join(path_to_input_sources, "%s.f90" % routine.module.name)
# Generate the different routine versions
code_versions = generate_routine_versions(precision_combinations, sensitive_variables, vault, routine_code.split("\n"), routine, filepath, indices)
generate_routine_versions(precision_combinations, sensitive_variables, subprogram, subprogram_code, output_folder)
write_routine_versions(code_versions, output_folder)
......@@ -3,33 +3,33 @@ import unittest
class TestFinder(unittest.TestCase):
def test_find_function_type_from_header_1(self):
import AutoRPE.UtilsRPE.Finder as Finder
import AutoRPE.UtilsRPE.ObtainSourceFileInfo as ObtainSourceFileInfo
header = 'RECURSIVE FUNCTION nodal_factort( kformula ) RESULT( zf )'
var_declaration = ['INTEGER, INTENT(in) :: kformula', 'type(rpe_var) :: zf', 'type(rpe_var) '
' :: zs',
'type(rpe_var) :: zf1', 'type(rpe_var) :: zf2', 'CHARACTER(LEN=3) '
' :: clformula']
ret_type, ret_val = Finder.find_function_type_from_header(header, var_declaration)
ret_type, ret_val = ObtainSourceFileInfo.find_function_type_from_header(header, var_declaration)
expected_retval = [None, 'zf']
self.assertEqual([ret_type, ret_val], expected_retval)
def test_find_function_type_from_header_2(self):
import AutoRPE.UtilsRPE.Finder as Finder
import AutoRPE.UtilsRPE.ObtainSourceFileInfo as ObtainSourceFileInfo
header = ' type(rpe_var) function bdy_segs_surf(phu, phv)'
var_declaration = ['type(rpe_var), DIMENSION(jpi,jpj), INTENT(in) :: phu',
'type(rpe_var), DIMENSION(jpi,jpj), INTENT(in) :: phv', 'INTEGER :: igrd',
'INTEGER :: ib_bdy', 'INTEGER :: ib', 'INTEGER , POINTER :: nbi',
'INTEGER , POINTER :: nbj', 'type(rpe_var), POINTER :: zflagu',
'type(rpe_var), POINTER :: zflagv']
ret_type, ret_val = Finder.find_function_type_from_header(header, var_declaration)
ret_type, ret_val = ObtainSourceFileInfo.find_function_type_from_header(header, var_declaration)
expected_retval = ['rpe_var', 'bdy_segs_surf']
self.assertEqual([ret_type, ret_val], expected_retval)
def test_find_function_type_from_header_3(self):
import AutoRPE.UtilsRPE.Finder as Finder
import AutoRPE.UtilsRPE.ObtainSourceFileInfo as ObtainSourceFileInfo
header = ' function dia_ptr_alloc()'
var_declaration = ['INTEGER :: dia_ptr_alloc', 'INTEGER, DIMENSION(3) :: ierr']
ret_type, ret_val = Finder.find_function_type_from_header(header, var_declaration)
ret_type, ret_val = ObtainSourceFileInfo.find_function_type_from_header(header, var_declaration)
expected_retval = [None, 'dia_ptr_alloc']
self.assertEqual([ret_type, ret_val], expected_retval)
......
......@@ -306,3 +306,37 @@ def merge_lines(_lines):
_lines[_index + j] = ""
_lines = remove_empty_lines(_lines)
return _lines
# Merge calls split on multiple lines and remove comments
def merge_line_range(lines, line_index):
line = remove_comments(lines[line_index]).strip()
merge_line_range.number = 1
if line.count("&"):
while line and line.strip()[-1] == "&":
line1 = line
while remove_comments(lines[line_index + merge_line_range.number]).strip() == "":
merge_line_range.number = merge_line_range.number + 1
line2 = remove_comments(lines[line_index + merge_line_range.number])
line = line1.replace("&", "").rstrip() + line2
merge_line_range.number += 1
line = line.replace("&", "")
line = re.sub(", +", ", ", line)
line = re.sub(": +", ": ", line)
line = line.strip() + "\n"
return line
# Merge lines in file
def merge_line_in_file(file, line_index):
with open(file, "w") as f:
lines = [l for l in f]
line = merge_line_range(lines, line_index)
lines[line_index] = line
for j in range(1, merge_line_range.number):
lines[line_index + j] = ""
text = "".join(lines)
f.write(text)
import AutoRPE.UtilsRPE.CurrentBlock as CurrentBlock
import AutoRPE.UtilsRPE.BasicFunctions as BasicFunctions
import AutoRPE.UtilsRPE.VariablePrecision as VariablePrecision
import AutoRPE.UtilsRPE.Error as Error
import warnings
import re
def find_function_type_from_header(header, var_declaration):
# Get the function name
fun_name = header.lower().split('function')[1]
fun_name = fun_name.split("(")[0]
fun_name = fun_name.replace("(", "")
fun_name = fun_name.strip()
# Check if a return val is declared
if header.lower().count("result"):
ret_val = header.lower().split('result')[1]
ret_val = ret_val.replace('results', '')
ret_val = ret_val.replace('(', '')
ret_val = ret_val.replace(')', '').strip()
else:
ret_val = fun_name
# Check if the return_val typ is specified among the function variables
declaration_of_retval = [v.split("::")[1].lower().strip() for v in var_declaration
if v.split("::")[1].lower().strip() in [fun_name, ret_val]]
if declaration_of_retval:
return None, declaration_of_retval[0]
# The type is specified in front of the function
pattern = "(.*)function"
match = re.search(pattern, header.strip(), re.IGNORECASE)
fun_type = remove_brackets_content(match.groups(0)[0].strip().lower())
if fun_type == "real":
real_declaration = match.group(1)
real_type_pattern = r"real\((.*)\)"
nt = re.match(real_type_pattern, real_declaration, re.I)
try:
fun_type = VariablePrecision.lookup_table[nt.group(1)]
except AttributeError:
fun_type = VariablePrecision.wp
if match.group(0).count('rpe_var'):
return 'rpe_var', fun_name
else:
return fun_type, fun_name
def find_var_elsewhere(variable, file_objects):
pattern = r"[^\w]%s[^\w]"
files = [module.name for module in variable.procedure.module.used_by]
files.append("%s.f90" % variable.procedure.module.name)
files_to_search = []
for f in file_objects:
if f in files:
files_to_search.append(file_objects[f])
if len(files_to_search) == files:
break
return [f for f in files_to_search if re.search(pattern % variable.name, f.text)]
def find_module_and_line_of_allocation(variable):
module_where_var_appears = variable.procedure.module.used_by
for module in [variable.module] + module_where_var_appears:
for line_index, line in enumerate(module.lines):
if re.search(r"\ballocate\b\s*\(.*\b"+variable.name+r"\b.*", line, re.I):
return module, line_index
warnings.warn("Allocation of variable %s not found" % variable.name)
return None, None
def find_declaration_line(variable, filepath, vault, additional_files=None):
# Finds line at which the variable is declared
# WARNING: if the line of the declaration is split, it merges the line in the file
# and therefore this function can modify the file
import re
# Declaration patterns
# Standard pattern
pattern = r"::.*\b%s\b" % variable.name
# Exception pattern (line is split)
exception_pattern = r"&.*\b%s\b" % variable.name
# Common real variable declaration pattern
real_pattern = r"real.*::"
def load_lines(filepath):
with open(filepath, "r") as f:
# Sometimes people decide that is ok to copy and paste non UTF-8 character in comments. It isn't.
try:
lines = [l for l in f]
except UnicodeDecodeError:
raise UnicodeDecodeError("The file %s contains non UTF-8 character.\n "
"Please clean, and retry.", filepath)
return lines
# Check if the file contains exceptions and fix them
check_for_exceptions = True
while check_for_exceptions:
# Setting this to False, will only activate this again in case a case is found
check_for_exceptions = False
lines = load_lines(filepath=filepath)
current_block = CurrentBlock.CurrentBlock(vault[variable.procedure.module.name].main)
for index, line in enumerate(lines):
line = BasicFunctions.remove_comments(line).strip()
if current_block.procedure == variable.procedure:
m = re.search(exception_pattern, line)
if m:
declaration_displacement = 1
if BasicFunctions.remove_comments(lines[index - 1]).strip()[0] == "&":
while BasicFunctions.remove_comments(lines[index - declaration_displacement]).strip()[0] == "&":
declaration_displacement = declaration_displacement + 1
if re.search(real_pattern, lines[index - declaration_displacement], re.I):
merge_line_range(filepath, index - declaration_displacement)
check_for_exceptions = True
break
current_block.in_which_block(line, vault=vault)
# Lines should be already updated since in case being modified it checks for exceptions again.
# Search for declarations
declarations = []
current_block = CurrentBlock.CurrentBlock(vault[variable.procedure.module.name].main)
type_def = None
for index, line in enumerate(lines):
line = BasicFunctions.remove_comments(line).strip()
type_def = _in_which_type_definition(line, type_def)
if re.search(pattern, line):
if current_block.procedure == variable.procedure:
if variable.is_member_of is None:
declarations.append(index)
elif variable.is_member_of == type_def:
declarations.append(index)
else:
pass
else:
current_block.in_which_block(line, vault=vault)
if declarations:
return declarations
include_declaration = [line for line in lines if line.count("#include")]
# If no line has been found in this file searches for possible additional files
if additional_files is not None:
for declaration in include_declaration:
h90_file = declaration.split("#include")[1]
if h90_file.lower().count("h90"):
h90_file = h90_file.strip().replace("\"", "")
additional_files.append(h90_file)
raise Error.LineNotFound
def find_file_and_line_of_namelist_reading(variable):
pattern = r"namelist.*[^\w]%s[^\w]" % variable.name
try:
# Find the module where the namelist is read
module = [m for m in [variable.module] + variable.procedure.module.used_by
if m.name == variable.appears_in_namelist][0]
except IndexError:
raise Error.ModuleNotFound("Unable to find module %s in used module of var %s"
% (variable.appears_in_namelist % variable.name))
namelist_name = None
for line in module.lines:
if re.search(pattern, line, re.IGNORECASE):
pattern2 = r"namelist/(.*)/.*%s[^\w]" % variable.name
match = re.search(pattern2, line, re.IGNORECASE)
namelist_name = match.group(1)
break
if namelist_name:
for index, line in enumerate(module.lines):
pattern3 = r"read *\(.*[^\w]%s[^\w]" % namelist_name
if re.search(pattern3, line, re.IGNORECASE):
return module, index
return None, None
def find_first_use(variable):
procedure = variable.procedure
module = procedure.module
# pattern = '(^|\W+)%s(\W+|$)' % variable.name
pattern = r'\W*%s\W*' % variable.name
searching = False
loop_level = 0
loop_start = None
in_where_statement = False
where_statement_start = None
# Scroll all the lines
for line_index, line in enumerate(module.lines):
line = line.strip().lower()
# Until it finds the subroutine to which the variables belongs (routine.name)
if not searching:
match = re.search(r"(subroutine|function)\s+\b%s\b" % procedure.name, line, re.IGNORECASE)
if match:
searching = True
# In the subroutine
else:
# Scroll all the lines searching for the first use
if re.search(pattern, line, re.IGNORECASE):
# skip declarations: bad workaround, but avoids false positive that will block compilation.
if line.count("::") or line.lower().count("implicit none"):
continue
# Skip the first line
elif line.count("subroutine"):
# Reach the end: the variable has not been used
if line.count("end subroutine"):
break
continue
# Skip import statements
elif line.find("use ") == 0:
continue
# Other statement
elif line.find("data ") == 0:
continue
else:
# The variable is found
if in_where_statement:
# Inside a where statement
return module, where_statement_start
else:
# In a (nested) loop
if loop_level > 0:
return module, loop_start
# On a normal line
else:
return module, line_index
elif line.count("end subroutine"):
break
# Whenever a 'do loop' starts increment nesting index
elif re.search(r"^ *do *[\w]* *=", line, re.IGNORECASE):
if loop_level == 0:
loop_start = line_index
loop_level += 1
# Whenever a 'do loop' ends decrement nesting index
# This way match bot the expression 'end do' and 'enddo' but excludes matches like 'enddo loop_name'
elif re.search(r"^ *end\s*do", line, re.IGNORECASE):
loop_level -= 1