#=====================================
# Functions to plot temporal series of data measured by tall towers:
# Fun. 1: || plot_raw_data || Plot all data, with original resolution
# Fun. 2: || hourly.avgs || Plot hourly averaged data
# Fun. 3: || monthly.avgs || Plot monthly averaged data 
# Fun. 4: || plot_flagged_data || Plot QC'ed data with flags depending on the results of the QC suite
#=====================================
# jramon - November 2017
#=====================================
# all.data: data table in which wind variables and other meteo variables must be saved in columns
# fileplot: file name to save the plot
# dir:  =0 do not plot wind direction variables
#       =1 plot both wind speed and wind direction variables
# temp: =0 do not plot temperature
#       =1 plot ONLY temperatures at several heights
# ini	: zoom period (beginning of interval)
# end	: zoom period (end of interval)
# rmarkdown: special plot settings to do the summary
#=====================================
library(ggplot2)
library(data.table)
library(lubridate)
library(gridExtra)
library(scales)
source('/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/times_functions.R')
source('/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/plotting/circular_avgs.R')

plot_raw_data <- function(all.data,tower_name,fileplot="",dir=0,ini=NULL,end=NULL,temp=0,rmarkdown=0) {
	#-------------------------------------
        # Path where the plot will be saved 
        #-------------------------------------
        plot.path <- config::get("plot.path",file = "/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml")

	#-------------------------------------
        # Data resolution and measurement heights 
        #-------------------------------------
        freq <- attr(all.data,"freq")
        heights <- attr(all.data,"heights")

	#-------------------------------------
        # Clean up data table. Only prepared to plot wind speed, wind direction or temperature 
        #-------------------------------------
	all.data <- all.data[,c("times",grep("(^windagl|^wdiragl|^taagl)",names(all.data),value=T)),with=F]
	
	#-------------------------------------
        # Reduce time period if requested
        #-------------------------------------
        if ( !is.null(ini) & !is.null(end) ) {
                all.data <- all.data[all.data$times >= ymd_hm(ini) & all.data$times <= ymd_hm(end)]
        }
        else if (!is.null(ini) & is.null(end) ) {
                all.data <- all.data[all.data$times >= ymd_hm(ini)]
        }
        else if (is.null(ini) & !is.null(end) ) {
                all.data <- all.data[all.data$times <= ymd_hm(end)]
        }
                
        #-------------------------------------
        # By default, we only plot wind speed at several levels 
        # Names vars: windagl (disable wdiragl)
	#-------------------------------------
        names_vars <- hgt_check(heights,tower_name,dir,temp)
		
        #-------------------------------------
        # Read tower name from metadata excel file 
        #-------------------------------------
        tower_name_long <- readRDS("/esarchive/scratch/jramon/INDECIS/Tall_towers_work/tall_towers_list/towers_metadata.rds")[tower_name,][["long_name"]]

	plots <- list()
	for (i in seq(names_vars)) {

		gral.var <- names_vars[i]
      		hgts <- heights[[gral.var]]

                #-------------------------------------
                # Prepare data to plot
                #-------------------------------------
                subset.var <- all.data[,grep(paste0("(",gral.var,"|times)"),names(all.data),value=TRUE),with=FALSE]
                melted.var = melt(subset.var, id.vars="times")
                names(melted.var) <- c("times","fullname","value")
		melted.var$height <- factor(sub(paste0(gral.var,"(\\d+)S.*"),"\\1",melted.var$fullname))
					    #sort(unique(hgts),decreasing=T))
		
		melted.var$sens <- factor(sub(paste0(gral.var,".*S(\\d{1})"),"\\1",melted.var$fullname)) 
		
                ws.lim = c(0,ceiling(max(melted.var$value,na.rm=TRUE)/10)*10)
                wd.lim = c(0,360)

		if (grepl("windagl",gral.var) == TRUE) {
                	ylim <- ws.lim
                        ytit <- expression(Wind~speed~(m~s^-1))
                        long_name <- "Wind speed"
                	point.color <- "blue"
		}
                else if (grepl("wdiragl",gral.var) == TRUE) {
                	ylim <- wd.lim
                        ytit <- "Wind direction (deg)"
                        long_name <- "Wind direction"
			point.color <- "blue"
		}
		else if (grepl("taagl",gral.var) == TRUE) {
			ylim <- NULL
			long_name <- "Temperature"
			ytit <- "Air temperature (ºC)"
			melted.var$value <- melted.var$value - 273.15
			point.color <- "black"
		}
	
		#-------------------------------------
		# Name levels for the facetting
		# Suppose we have less than 10 anemos per height
		#-------------------------------------
		melted.var$hgtsens <- factor(as.numeric((paste0(melted.var$height,melted.var$sens))))
		
		#---------------------------------
                # Name levels in order
                #---------------------------------
		for (hgtsen in sort(unique(melted.var$hgtsens),decreasing=T)) {
			levels(melted.var$hgtsens)[levels(melted.var$hgtsens)==hgtsen] <- 
			 	paste0(long_name," at ",sub("(\\d+).","\\1",as.character(hgtsen)),
					 " m agl of sensor ",sub(".*(\\d{1})","\\1",as.character(hgtsen)))
		}

		melted.var$hsreord <- factor(melted.var$hgtsens,levels=rev(levels(melted.var$hgtsens)))

		#-------------------------------------
        	# Order time stamps. Sometimes necessary, specially when having 2 different frequences 
        	#-------------------------------------
	        melted.var <- melted.var[order(melted.var$times),]

		#---------------------------------
	        # Create directory to save the plots, if it does not exit
        	#---------------------------------
        	ifelse(!dir.exists(file.path(config::get("plot.path",file="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml"),tower_name)),
                	dir.create(file.path(config::get("plot.path",file="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml"),tower_name)), FALSE)
		
		#---------------------------------
                # Read some plotting characteristics, that may vary depending on the length of the time period considered
                # The function is in times_functions.R
		#---------------------------------
		plot.feat <- plotting_features(melted.var$times,gral.var,rmarkdown) 
	
		#-------------------------------------
                # Plotting
                #-------------------------------------
                p <- ggplot(data=melted.var, aes(x=times,y=value)) 
              	p <- p + geom_line(color = "grey100", alpha = plot.feat$alpha)
		p <- p + geom_point(color=plot.feat$point.color, size=plot.feat$point.size)
                p <- p + facet_wrap(~hsreord,ncol=plot.feat$ncol)
                p <- p + labs(title = paste0(long_name," at several heights at ",tower_name_long," site"),
			      subtitle = paste0("Resolution of data is ",freq),
			      x = "UTC time", y = ytit, color="Heights\n(meters a.g.l.)",size=1) 
                p <- p + scale_x_datetime(labels = date_format(plot.feat$time_label),date_breaks=plot.feat$interv_date,
                                          expand=c(0,0),limits=c(plot.feat$min,plot.feat$max))
                p <- p + scale_y_continuous(expand = c(0, 0),limits=ylim) 
                p <- p + guides(colour = guide_legend(override.aes = list(size=4))) 
		p <- p + theme(plot.title = element_text(hjust=0.5,size=24),
                               panel.spacing = unit(1.5, "lines"),
                               strip.text = element_text(size=10),
			       axis.title=element_text(size=20),axis.text=element_text(size=16),
			       strip.text.x = element_text(size=15,margin = margin(.05,0,.05,0, "cm"))) 

		plots[[gral.var]] <- p		

		#-------------------------------------
                # Save plot only if indicated
	        #-------------------------------------
		if (! fileplot == "") {
   			 ggsave(plot=p,path=paste(plot.path,tower_name,sep="/"),
				filename=paste0(gral.var,"_",fileplot),width=16,height=9,device="pdf")
  		}
	}
	                
	return(plots)
}

#=====================================
# Hourly averages for each variable
# all.data: data table in which each column is one variable
# filtvars: full names of the variables (names of all.data data table except "times")
# height: different measurement levels
#=====================================
hourly.avgs <- function(all.data,tower_name,fileplot="",dir=0) {
      	#-------------------------------------
        # Path where the plot will be saved 
        #-------------------------------------
        plot.path <- config::get("plot.path",file = "/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml")

	#-------------------------------------
        # Data resolution and variables
        #-------------------------------------
        freq <- attr(all.data,"freq")
	filtvars <- attr(all.data,"filtvars")

	#-------------------------------------
        # Do the averages only if we have data with resolution higher than 1 hour. 
        #-------------------------------------
	if (freq2sec(freq) < 3600) {
		#-------------------------------------
        	# Data hourly averaged 
        	#-------------------------------------
		list_dfs_hr <- list()
        
		for (i in seq(filtvars))
        	{       var <- filtvars[i]
                
                	if (grepl("windagl",var) == TRUE) {
                        	vals <- aggregate(all.data[[var]],list(hour = cut(all.data$times, breaks="hour")),
						  mean,na.rm = TRUE)
                	}

                	#-------------------------------------
                	# Transform data into circular format and average angle values
                	#-------------------------------------
                	else if (grepl("wdiragl",var) == TRUE) {
                		vals <- aggregate(all.data[[var]],list(hour = cut(all.data$times, breaks="hour")),
						  circular.avg)
                	}
                	
			names(vals) <- c("times",var)

                	#-------------------------------------
                	# This is a list of data frames. 
			# Useful to name each column by the name of the corresponding variable
                	#-------------------------------------
                	list_dfs_hr[[i]] <- data.frame(vals)
        	}

        	#-------------------------------------
        	# Create a big data table with hourly-averaged values
        	#-------------------------------------
        	hr.means.dt <- Reduce(function(a,b) merge(a, b, by = "times", all=TRUE), list_dfs_hr)

        	#-------------------------------------
        	# Times in POSIXct and UTC time zone 
        	#-------------------------------------
	       	freq <- paste0("hourly averaged from ",freq," data")
		hr.means.dt$times <- as.POSIXct(hr.means.dt$times,tz="UTC")
	}

	else {
		print("Time stamp sampling is equal or higher than 1 hour. Plotting data with their original resolution")
		hr.means.dt <- all.data
	}
         
	attr(hr.means.dt,"heights") <- attr(all.data,"heights")
	attr(hr.means.dt,"filtvars") <- attr(all.data,"filtvars")
	attr(hr.means.dt,"freq") <- freq

	#-------------------------------------
        # Data frame to data table 
        #-------------------------------------
	hr.means.dt <- as.data.table(hr.means.dt)

	#-------------------------------------
        # Plot all data hourly averaged 
        #-------------------------------------
	plots <- plot_raw_data(hr.means.dt,tower_name,"hourly_averaged.pdf")
  
	return(plots)     
}

#=====================================
# Monthly averages for each variable monthly grouped
# all.data: data table in which each column is one variable
# filtvars: full names of the variables (names of all.data data table except "times"
# height: different measurement levels
#=====================================
monthly.avgs <- function(all.data,tower_name,fileplot="",dir=0)
{ 	#-------------------------------------
        # Path where the plot will be saved 
        #-------------------------------------
        plot.path <- config::get("plot.path",file = "/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml")

        #-------------------------------------
        # Data resolution and variables
        #-------------------------------------
        freq <- attr(all.data,"freq")
	heights <- attr(all.data,"heights")
        filtvars <- attr(all.data,"filtvars")
   	mth.freq <- paste0("monthly averaged from ",freq," data")
	
	#-------------------------------------
        # Data monthly averaged 
        #-------------------------------------
	list_dfs_mth <- list()
        for (i in seq(filtvars))
        {       var <- filtvars[i]
                
		if (grepl("windagl",var) == TRUE) {
                        vals <- aggregate(all.data[[var]],list(month = cut(all.data$times, breaks="month")),
					  mean,na.rm = TRUE)
                }

                #-------------------------------------
                # Transform data into circular format and average angle values
                #-------------------------------------
                else if (grepl("wdiragl",var) == TRUE) {
                        vals <- aggregate(all.data[[var]],list(month = cut(all.data$times, breaks="month")),
					  circular.avg)
                }
                
		names(vals) <- c("times",var)

                #-------------------------------------
                # This is a list of data frames. 
		# Useful to name each column by the name of the corresponding variable
                #-------------------------------------
                list_dfs_mth[[i]] <- data.frame(vals)
        }

        #-------------------------------------
        # Create a big data table with monthly-averaged values
        #-------------------------------------
        mth.means.dt <- Reduce(function(a,b) merge(a, b, by = "times", all=TRUE), list_dfs_mth)
        mth.means.dt$times <- as.POSIXct(mth.means.dt$times,tz="UTC")

        #-------------------------------------
        # Extremes of time intervals 
        #-------------------------------------
        max <- mth.means.dt$times[length(mth.means.dt$times)]
        min <- mth.means.dt$times[1]

	#-------------------------------------
        # Axis labels depending on the length of time series.
        # If the time series is between 8 and 16 years, annual labels
        # if it is more than 16 year, 5-annual labels
        #-------------------------------------
        por.in.days <- as.numeric(max - min)
        if (por.in.days >= 1825 & por.in.days < 5840) {
                time_label <- "%Y"
                interv_date <- "1 year"
        }

        else if (por.in.days >= 5840 & por.in.days < 8760)  {
                time_label <- "%Y"
                interv_date <- "3 year"
        }

        else if (por.in.days >= 8760) {
                time_label <- "%Y"
                interv_date <- "5 year"
        }

        else {
                time_label <- "%b %Y"
                interv_date <- "6 month"
        }

	#---------------------------------
        # Create directory to save the plots, if it does not exit
        #---------------------------------
        ifelse(!dir.exists(file.path(config::get("plot.path",file="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml"),tower_name)),
                dir.create(file.path(config::get("plot.path",file="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml"),tower_name)), FALSE)

        #-------------------------------------
        # Plot only wind speed.
	# dir = 1 to enable plotting wind direction 
        #-------------------------------------
	names_vars <- hgt_check(heights,tower_name,dir) 
                
	p <- list()
        for (i in seq(names_vars)) {

        	gral.var <- names_vars[i]
		hgts <- heights[[gral.var]]
	
                #-------------------------------------
                # Prepare data to plot
                #-------------------------------------
                subset.var <- mth.means.dt[,grep(paste0("(",gral.var,"|times)"),names(mth.means.dt))]
                melted.var = melt(subset.var, id.vars="times")
                names(melted.var) <- c("times","fullname","value")
                melted.var$height <- sub(paste0(gral.var,"(\\d+)S.*"),"\\1",melted.var$fullname)

                ws.lim = c(0,5+ceiling(max(melted.var$value,na.rm=TRUE)/10)*10)
                wd.lim = c(0,360)

                if (grepl("windagl",gral.var) == TRUE) {
                	ylim <- ws.lim
                        ytit <- expression(Monthly~averaged~wind~speed~(m~s^-1))
                        titl <- paste0("Wind speed at several heights at ",tower_name," site")
                }
                
	        else if (grepl("wdiragl",gral.var) == TRUE) {
                	ylim <- wd.lim
                        ytit <- "Monthly wind direction (deg)"
                        titl <- "Mensual means"
                }

                p[[gral.var]] <- ggplot(data=melted.var, aes(x=times,y=value,
				 color=factor(height,sort(as.numeric(hgts),decreasing=TRUE))),group=height) +
                          geom_line(size=0.25) +
                          labs(title = titl, subtitle = paste0("Resolution of data is ",mth.freq),
			       x = "UTC time", y = ytit, color="Heights\n(meters a.g.l.)") +
                          scale_x_datetime(labels = date_format(time_label),date_breaks =interv_date,expand=c(0,0),
					   limits=c(min,max)) +
                          scale_y_continuous(expand = c(0, 0),limits=ylim) + 
			  theme(axis.text.x=element_text(angle=0,hjust=0.5),
				axis.title=element_text(size=26),axis.text=element_text(size=16),
				plot.title = element_text(hjust=0.5,size=24))
	
		#-------------------------------------
        	# Save plot only if indicated
        	#-------------------------------------
        	if ( fileplot != "") {
        		ggsave(plot=p[[i]],path=paste(plot.path,tower_name,sep="/"),
		               filename=paste0(gral.var,"_",fileplot),width=16,height=9,device = "pdf")
       		 }
	}

	return(p)
}

#=====================================
# Plot all data with flags after being evaluated by the QC suite.
# If one observation fails to pass one test, this observation will be flagged as "Fail", whichever this test is.
# For each observation, a general flag is assigned. This flag pretends to be a summary of the results
# of the different QC routines that have been carried out.
# all.data -> is a the output data table of any of the QC functions.
# ini: beginning of the time period in format yyyymmdd hhmm (if requested)
# end: end of the time period in format yyyymmdd hhmm (if requested)
#=====================================
plot_flagged_data <- function(all.data,tower_name,fileplot="",dir=0,ini=NULL,end=NULL) {
	#-------------------------------------
        # Path where the plot will be saved 
        #-------------------------------------
        plot.path <- config::get("plot.path",file ="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml")
	
	all.data$freq <- NULL

	#-------------------------------------
        # Variables to be plotted
        #-------------------------------------
	heights <- attr(all.data,"heights")
	names_vars <- hgt_check(heights,tower_name,dir)
	filtvars <- attr(all.data,"filtvars")
	freq <- attr(all.data,"freq")	

	#-------------------------------------
        # Reduce time period if requested
        #-------------------------------------
	if ( !is.null(ini) & !is.null(end) ) {
		all.data <- all.data[all.data$times >= ymd_hm(ini) & all.data$times <= ymd_hm(end)]	
	} else if (!is.null(ini) & is.null(end) ) {
		all.data <- all.data[all.data$times >= ymd_hm(ini)]
	} else if (is.null(ini) & !is.null(end) ) {
                all.data <- all.data[all.data$times <= ymd_hm(end)]
	}
	
	#-------------------------------------
        # Tower name
        #-------------------------------------
	tower_name_long <- readRDS("/esarchive/scratch/jramon/INDECIS/Tall_towers_work/tall_towers_list/towers_metadata.rds")[tower_name,][["long_name"]]

	#-------------------------------------
       	# Wrap up flags imposed by each QC procedure
       	#-------------------------------------
	for (var in filtvars) {
       		all.data[[paste0("f_",var)]] <- 1 #A priori, all data is flagged as "Pass"

       		#-------------------------------------
       		# Data not evaluated by any of the QC routines
       		#-------------------------------------
       		all.data[[paste0("f_",var)]][apply(all.data[,grep(paste0("flag.*",var),names(all.data),value=T),
						with=F],1, function(x) {sum(x == 0) > 3})] <- 10
       
       		#-------------------------------------
       		# Calms
       		#-------------------------------------
       		all.data[[paste0("f_",var)]][apply(all.data[,grep(paste0("flag.*",var),names(all.data),value=T),
						with=F],1, function(x) {any(x == 5)})] <- 5
       
       		#-------------------------------------
       		# If there is one "Suspect" value, the value is flagged as "Suspect" no matter what test is
       		#-------------------------------------
       		all.data[[paste0("f_",var)]][apply(all.data[,grep(paste0("flag.*",var),names(all.data),value=T),
						with=F],1, function(x) {any(x == 2)})] <- 2

       		#-------------------------------------
       		# But, if there is one "Fail" value, the value is flagged as "Fail", overwriting the "Suspect"
       		#-------------------------------------
       		all.data[[paste0("f_",var)]][apply(all.data[,grep(paste0("flag.*",var),names(all.data),value=T),
						with=F],1, function(x) {any(x == 4)})] <- 4

       		#-------------------------------------
       		# Missing data
       		#-------------------------------------
       		all.data[[paste0("f_",var)]][apply(all.data[,grep(paste0("flag.*",var),names(all.data),value=T),
						with=F],1, function(x) {all(x == 9)})] <- 9
	}
	
	#-------------------------------------
	# Clean the data table
	#-------------------------------------
	all.data[,grep("flag.*",names(all.data),value=T)] <- NULL
		
	#-------------------------------------
	# Prepare data to plot 
	#-------------------------------------
	melted.data = melt(all.data,id.vars=c("times",grep("f.*",names(all.data),value=T)))
	melted.data[["flag"]] <- 1

	for (var in filtvars) {
       		melted.data[melted.data[["variable"]] == var][["flag"]] <-
                       	                melted.data[melted.data[["variable"]] == var][[paste0("f_",var)]]
	}

	melted.data$height <- as.factor(sub("(windagl|^wdiragl)(\\d+)S.*","\\2",melted.data$variable))
	melted.data$varname <- sub("(windagl|^wdiragl)(\\d+)S.*","\\1",melted.data$variable)

	melted.data$sens <- factor(sub(paste0(".*S(\\d{1})"),"\\1",melted.data$variable))
	
	#-------------------------------------
        # Define name levels for the facetting
        # Suppose we have less than 10 anemos per height
        #-------------------------------------
        melted.data$hgtsens <- factor(as.numeric((paste0(melted.data$height,melted.data$sens))))

	#-------------------------------------
        # Order time stamps. Sometimes necessary, specially when having 2 different frequences 
        #-------------------------------------
	melted.data <- melted.data[order(melted.data$times),]

	#-------------------------------------
	# Compute the % of the flagged data
	#-------------------------------------
	perc.neval      <- round(100*nrow(melted.data[melted.data$flag == 10])/nrow(melted.data),1)
	perc.pass       <- round(100*nrow(melted.data[melted.data$flag == 1])/nrow(melted.data),1)
	perc.suspect    <- round(100*nrow(melted.data[melted.data$flag == 2])/nrow(melted.data),1)
	perc.fail       <- round(100*nrow(melted.data[melted.data$flag == 4])/nrow(melted.data),1)
	perc.calm       <- round(100*nrow(melted.data[melted.data$flag == 5])/nrow(melted.data),1)
	perc.missing    <- round(100*nrow(melted.data[melted.data$flag == 9])/nrow(melted.data),1)

	#-------------------------------------
	# Set levels for heights and flags
	#-------------------------------------
	for (hgt in sort(heights$windagl)) {
       		levels(melted.data$height)[levels(melted.data$height)==hgt] <- 
									paste0("Wind speed at ",hgt, " m agl")
	}	

	melted.data$flag <- as.factor(melted.data$flag)
	levels(melted.data$flag)[levels(melted.data$flag)==1] <- "Pass"
	levels(melted.data$flag)[levels(melted.data$flag)==2] <- "Suspect"
	levels(melted.data$flag)[levels(melted.data$flag)==4] <- "Fail"
	levels(melted.data$flag)[levels(melted.data$flag)==5] <- "Calm"
	levels(melted.data$flag)[levels(melted.data$flag)==10] <- "Not evaluated"

	#---------------------------------
        # Create directory to save the plots, if it does not exit
        #---------------------------------
        ifelse(!dir.exists(file.path(config::get("plot.path",file="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml"),tower_name)),
                dir.create(file.path(config::get("plot.path",file="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml"),tower_name)), FALSE)

	#-------------------------------------
	# Define the scale colour
	#-------------------------------------
	myColors <- c("darkgreen","orange","red2","green4","blue")
	names(myColors) <- c("Pass","Suspect","Fail","Calm","Not evaluated")
	colScale <- scale_color_manual(values = myColors, drop=TRUE) #Change to DROP=TRUE to remove non-used cathegories
	
	#-------------------------------------
        # Plotting wind speed and wind direction separately
        #-------------------------------------
	plots <- list()
	for (i in seq(names_vars)) {

                gral.var <- names_vars[i]

                if (gral.var == "windagl") {long_name = "Wind speed"}
                else if (gral.var == "wdiragl") {long_name = "Wind direction"}

		#---------------------------------
                # Name levels in order
                #---------------------------------
                for (hgtsen in sort(unique(melted.data$hgtsens),decreasing=T)) {
                        levels(melted.data$hgtsens)[levels(melted.data$hgtsens)==hgtsen] <-
                                paste0(long_name," at ",sub("(\\d+).","\\1",as.character(hgtsen)),
                                         " m agl of sensor ",sub(".*(\\d{1})","\\1",as.character(hgtsen)))
                }

		melted.data$hsreord <- factor(melted.data$hgtsens,levels=rev(levels(melted.data$hgtsens)))

		#---------------------------------
                # Read some plotting characteristics, that may vary depending on the length of the time period considered
                # The function is in times_functions.R
                #---------------------------------
                plot.feat <- plotting_features(melted.data$times,gral.var)
	
		p <- ggplot(melted.data[melted.data$flag != 9 & melted.data$varname == gral.var],aes(x=times,y=value,color=flag))
		p <- p + geom_line(color = "grey100", alpha = plot.feat$alpha)
		p <- p + geom_point(size=plot.feat$point.size)
		p <- p + facet_wrap(~hsreord, ncol=1)
		p <- p + colScale
		p <- p + scale_x_datetime(labels = date_format(plot.feat$time_label),date_breaks=plot.feat$interv_date,
                                          expand=c(0,0),limits=c(plot.feat$min,plot.feat$max))
		p <- p + guides(colour = guide_legend(override.aes = list(size=10))) # Enlarge legend points
		p <- p + ylab(expression(Wind~speed~(m~s^-1))) + xlab("UTC time")
		p <- p + labs(title=paste0(long_name," at several heights at ",tower_name_long," site"), 
			      subtitle = paste0("Flagged data. Resolution of data is ",freq),
        	      	      caption = paste0(paste0(perc.suspect, " % of data is flagged as suspect\n"),
                	      		paste0(perc.fail, " % of data is flagged as fail\n"),
                        		paste0(perc.calm, " % of data corresponds to calm periods\n"),
                        		paste0(perc.neval, " % of data have not been evaluated by 3 or more tests\n"),
                        		paste0(perc.missing," % of data corresponding to these time series is missing\n")))
		p <- p + theme(legend.title=element_blank(),legend.text=element_text(size=20),
			       legend.key.size=unit(0.005,"in"),axis.title=element_text(size=20),
			       axis.text=element_text(size=16),plot.title = element_text(hjust = 0.5,size=24),
			       plot.caption = element_text(hjust=0),strip.text.x = element_text(size=15,
										margin = margin(.05,0,.05,0, "cm")))
	
		if ( fileplot != "") {
               		ggsave(plot=p,path=paste(plot.path,tower_name,sep="/"),
                       	       filename=fileplot,width=16,height=9,device="pdf")
		}
		
		plots[[gral.var]] <- p	
	}

	return(plots)
}

#=====================================
# Plot all data with flags after filling gaps using data from other levels.
# A multi-linear regression has been used
# This flag pretends to be a summary of the results
# of the preliminary filling procedures applied to tall tower data
# all.data -> quality controlled and filled data table containing wind variables in columns and flags 
#	      indicating whether the observation has been filled or not 
#=====================================
plot_filled_data <- function(all.data,tower_name,fileplot="",dir=0,ini=NULL,end=NULL) {
	#-------------------------------------
        # Path where the plot will be saved 
        #-------------------------------------
        plot.path <- config::get("plot.path",file ="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml")

        all.data$freq <- NULL

        #-------------------------------------
        # Variables to be plotted
        #-------------------------------------
        heights <- attr(all.data,"heights")
        names_vars <- hgt_check(heights,tower_name,dir)
        filtvars <- attr(all.data,"filtvars")
        freq <- attr(all.data,"freq")

        #-------------------------------------
        # Reduce time period if requested
        #-------------------------------------
        if ( !is.null(ini) & !is.null(end) ) {
                all.data <- all.data[all.data$times >= ymd_hm(ini) & all.data$times <= ymd_hm(end)]
        }
        else if (!is.null(ini) & is.null(end) ) {
                all.data <- all.data[all.data$times >= ymd_hm(ini)]
        }
        else if (is.null(ini) & !is.null(end) ) {
                all.data <- all.data[all.data$times <= ymd_hm(end)]
        }

        #-------------------------------------
        # Tower name
        #-------------------------------------
        tower_name_long <- readRDS("/esarchive/scratch/jramon/INDECIS/Tall_towers_work/tall_towers_list/towers_metadata.rds")[tower_name,][["long_name"]]

	#-------------------------------------
        # Prepare data to plot 
        #-------------------------------------
        melted.data = melt(all.data,id.vars=c("times",grep("^flagfill_.*",names(all.data),value=T)))
        melted.data[["flag"]] <- 0 # Initialize to 0, as wind direction data have not been filled

        for (var in grep("^windagl_*",filtvars,value=T)) {
                melted.data[melted.data[["variable"]] == var][["flag"]] <-
                                        melted.data[melted.data[["variable"]] == var][[paste0("flagfill_",var)]]
        }

        melted.data$height <- as.factor(sub("(windagl|^wdiragl)(\\d+)S.*","\\2",melted.data$variable))
        melted.data$varname <- sub("(windagl|^wdiragl)(\\d+)S.*","\\1",melted.data$variable)

        melted.data$sens <- factor(sub(paste0(".*S(\\d{1})"),"\\1",melted.data$variable))

	#-------------------------------------
        # Define name levels for the facetting
        # Suppose we have less than 10 anemos per height
        #-------------------------------------
        melted.data$hgtsens <- factor(as.numeric((paste0(melted.data$height,melted.data$sens))))

        #-------------------------------------
        # Order time stamps. Sometimes necessary, specially when having 2 different frequences 
        #-------------------------------------
        melted.data <- melted.data[order(melted.data$times),]

	#-------------------------------------
        # Set levels for heights and flags only for wind speed data
        #-------------------------------------
        #for (hgt in sort(heights$windagl)) {
        #        levels(melted.data$height)[levels(melted.data$height)==hgt] <- paste0("Wind speed at ",hgt, " m agl")
        #} 

	#-------------------------------------
        # Compute the % of the filled data for ALL time series
        #-------------------------------------
        perc.filled <- round(100*nrow(melted.data[melted.data$varname == "windagl" & melted.data$flag == 1])/nrow(melted.data[melted.data$varname == "windagl"]),1)
	perc.missing <- round(100*sum(is.na(melted.data[melted.data$varname == "windagl"]))/nrow(melted.data[melted.data$varname == "windagl"]),1)

	#-------------------------------------
        # Set levels: original value, filled value
        #-------------------------------------
	melted.data$flag <- as.factor(melted.data$flag)
        levels(melted.data$flag)[levels(melted.data$flag)==1] <- "Filled"
        levels(melted.data$flag)[levels(melted.data$flag)==0] <- "Original"
	levels(melted.data$flag)[levels(melted.data$flag)==2] <- "Averaged from \nredundant sensors"

	#---------------------------------
        # Create directory to save the plots, if it does not exit
        #---------------------------------
        ifelse(!dir.exists(file.path(config::get("plot.path",file="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml"),tower_name)),
                dir.create(file.path(config::get("plot.path",file="/esarchive/scratch/jramon/GitLab_jramon/TallTowers/generic/config.yml"),tower_name)), FALSE)

	#-------------------------------------
        # Define the scale colour
        #-------------------------------------
        myColors <- c("blue","orange","palegreen4")
        names(myColors) <- c("Original","Filled","Averaged from \nredundant sensors")
        colScale <- scale_color_manual(values = myColors, drop=TRUE) #Change to DROP=TRUE to remove non-used cathegorie

        #-------------------------------------
        # Plotting wind speed and wind direction separately
        #-------------------------------------
        plots <- list()
        for (i in seq(names_vars)) {

                gral.var <- names_vars[i]

                if (gral.var == "windagl") {long_name = "Wind speed"}
                else if (gral.var == "wdiragl") {long_name = "Wind direction"}

                #---------------------------------
                # Name levels in order
                #---------------------------------
                for (hgtsen in sort(unique(melted.data$hgtsens),decreasing=T)) {
                        levels(melted.data$hgtsens)[levels(melted.data$hgtsens)==hgtsen] <-
                                paste0(long_name," at ",sub("(\\d+).","\\1",as.character(hgtsen)),
                                         " m agl of sensor ",sub(".*(\\d{1})","\\1",as.character(hgtsen)))
                }

                melted.data$hsreord <- factor(melted.data$hgtsens,levels=rev(levels(melted.data$hgtsens)))

                #---------------------------------
                # Read some plotting characteristics, that may vary depending on the length of 
		# the time period considered
                # The function is in times_functions.R
                #---------------------------------
                plot.feat <- plotting_features(melted.data$times,gral.var)

                p <- ggplot(melted.data[melted.data$varname == gral.var],aes(x=times,y=value,color=flag))
                p <- p + geom_line(color = "grey100", alpha = plot.feat$alpha)
                p <- p + geom_point(size=plot.feat$point.size)
                p <- p + facet_wrap(~hsreord, ncol=1)
                p <- p + colScale
                p <- p + scale_x_datetime(labels = date_format(plot.feat$time_label),date_breaks=plot.feat$interv_date,
                                          expand=c(0,0),limits=c(plot.feat$min,plot.feat$max))
                p <- p + guides(colour = guide_legend(override.aes = list(size=10))) # Enlarge legend points
                p <- p + ylab(expression(Wind~speed~(m~s^-1))) + xlab("UTC time")
                p <- p + labs(title=paste0(long_name," at several heights at ",tower_name_long," site"),
                              subtitle = paste0("Filled data. Resolution of data is ",freq),
                              caption = paste0(paste0(perc.filled, " % of data have been filled\n"),
                                        paste0(perc.missing," % of data corresponding to these time series is missing\n")))
                p <- p + theme(legend.title=element_blank(),legend.text=element_text(size=20),
                               legend.key.size=unit(0.005,"in"),axis.title=element_text(size=20),
                               axis.text=element_text(size=16),plot.title = element_text(hjust = 0.5,size=24),
                               plot.caption = element_text(hjust=0),strip.text.x = element_text(size=15,
                                                                                margin = margin(.05,0,.05,0, "cm")))

                if ( fileplot != "") {
                        ggsave(plot=p,path=paste(plot.path,tower_name,sep="/"),
                               filename=fileplot,width=16,height=9,device="pdf")
                }

                plots[[gral.var]] <- p
        }

        return(plots)
}


#=====================================
# AUXILIARY FUNCTIONS
#=====================================
# Check if we have any site without anemos or vanes
# 'heights' list with the elements 'wdiragl' and 'windagl' with anemo and vane heights, 
# if there is any. If not, it is set to NULL 
# dir:	1 <- use wind speed and wind direction values, if possible
#	0 <- use only wind speed values, if possible
# temp: 1 <- plot ONLY temperature series
#	0 <- do not plot temperature	
#=====================================
hgt_check <- function(heights,tower_name,dir=0,temp=0) {
	#-------------------------------------
        # If the tower does not have anemo or vanes. Cannot work without wind speed data.
        #-------------------------------------
	if (temp != 0 & !is.null(heights$taagl)) {
                names_vars <- "taagl"
        }
	
	else if (temp != 0 & is.null(heights$taagl)) {
		stop("Temperature data have not been loaded.")
	}

	else if (!is.null(heights$windagl) & !is.null(heights$wdiragl) & dir == 1 & temp == 0) {
                names_vars <- c("windagl","wdiragl")
        } 
	
	else if (!is.null(heights$windagl) & !is.null(heights$wdiragl) & dir == 0 & temp == 0) {
		names_vars <- "windagl"
	}

        else if (!is.null(heights$windagl) & is.null(heights$wdiragl)) {
                print(paste0("The tower ",tower_name," does not have wind direction data"))
                names_vars <- "windagl"
        }
	
        else {
                stop(paste0("The tower ",tower_name," does not have wind speed or temperature data"))
        }
	
	return(names_vars)
}
