#########################################
# qc_tests.R		                #
# Each function is one QC test		#
# Returns data.table with flagged values#
# These test must be applied only with  #
# data with high resolution (i.e.       #
# resolution higher than daily)         #
# jramon - January 2018                 #
#########################################
#----------------------------------------
# Main arguments, included in ALL QC tests:
#   all.data: data table in which wind variables and other meteo variables must be saved in columns. 
#     Must contain the following attributes:
#       attr(all.data, "heights"): list specifying sensor heights
#       attr(all.data, "filtvars"): wind variables to be quality controlled
#       attr(all.data, "freq"): time interval sampling of the data 
#       attr(all.data, "sensors"): indicates the maximum number of sensors per level
#----------------------------------------

#----------------------------------------
# Name: Plausible values test.
# Aim:  Remove unrealistic values 
# Arguments:
#   max.value.fail: data above this values will be flagged as 'fail'. Default: 113.2 m s-1, Australia wind gust world record
#   max.value.suspect: data above this values will be flagged as 'suspect'. Default: 75 m s-1, Vaisala's sensors highest recordable value
# Flags:
#   Fail: wind speed values outside [0,max.value.fail] range are flagged as fail
#         wind direction values out of [0,360) range are flagged as fail
#   Suspect: wind speed values in the range (max.value-suspect,max.value.fail] are flagged as suspect
#   Pass: other cases
#----------------------------------------
plausible_values <- function(all.data,max.value.fail=113.2,max.value.suspect=75) 
{	#---------------------------------
        # Variables to be quality controlled
        #---------------------------------
	filtvars <- attr(all.data,"filtvars")
	
	flag_name <- "flagpv_"

	#---------------------------------
        # Wind speed
        #---------------------------------
	for (var in grep("^windagl",filtvars,value=TRUE)) {
		all.data[[paste0(flag_name,var)]] <- factor(qc_values$not.eval,levels=unlist(qc_values))
		all.data[[paste0(flag_name,var)]][all.data[[var]] <= max.value.fail] <- qc_values$pass	    
		all.data[[paste0(flag_name,var)]][all.data[[var]] > max.value.suspect] <- qc_values$suspect
		all.data[[paste0(flag_name,var)]][all.data[[var]] > max.value.fail | all.data[[var]] < 0] <- qc_values$fail
		all.data[[paste0(flag_name,var)]][is.na(all.data[[var]]) == TRUE] <- qc_values$missing
	}
	
	#---------------------------------
        # Wind direction
        #---------------------------------
	for (var in grep("^wdiragl",filtvars,value=TRUE)) { 
                all.data[[paste0(flag_name,var)]] <- factor(qc_values$not.eval,levels=unlist(qc_values))
                all.data[[paste0(flag_name,var)]][all.data[[var]] < 360 & all.data[[var]] >= 0] <- qc_values$pass
                all.data[[paste0(flag_name,var)]][all.data[[var]] >= 360 | all.data[[var]] < 0] <- qc_values$fail
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]]) == TRUE] <- qc_values$missing
       	}

	return(all.data)
}

#----------------------------------------
# Name: Flat line test.
# Aim:  Identify and flag sequences of constant observations
# Flags:
#   Fail:    6 or more consecutive wind speed observations are equal 
#	     40 or more consecutive wind direction observations are equal
#            periods of 30 days in which the maximum wind speed value is 0 
#   Suspect: 3 or more consecutive wind speed observations are equal 
#            20 or more consecutive wind direction observations are equal
#----------------------------------------
flat_line <- function(all.data) {
	
	flag_name <- "flagfl_"

	#---------------------------------
        # Variables to be quality controlled
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
	freqs <- attr(all.data,"freq")

	for (var in filtvars) {
		#---------------------------------
        	# PART 1: Detect and remove sequences of consecutively repeated values
		# How many constant values in a row are allowed?
	        #---------------------------------
		if (grepl("windagl",var)) {
			consec.fail <- 6
		        consec.suspect <- 3
		}
		if (grepl("wdiragl",var)) {
			consec.fail <- 40
                        consec.suspect <- 20
		}
		
		all.data[[paste0(flag_name,var)]] <- factor(qc_values$pass,levels=unlist(qc_values))
			
		#---------------------------------
	        # Find the start and end point of the repeated sequence
		# Original code from: http://masterr.org/r/how-to-find-consecutive-repeats-in-r/
        	#---------------------------------
		#---------------------------------
                # Detect sequences that will be flagged as FAIL data
                #---------------------------------
		runs_f = rle(all.data[[var]])
		myruns_f = which(runs_f$lengths >= consec.fail)
		runs_f.lengths.cumsum = cumsum(runs_f$lengths)
		ends_f = runs_f.lengths.cumsum[myruns_f]
		newindex_f = ifelse(myruns_f>1, myruns_f-1, 0)
		starts_f = runs_f.lengths.cumsum[newindex_f] + 1
		if (0 %in% newindex_f) starts_f = c(1,starts_f)		
			
		if (length(starts_f) != 0) {
			for (j in length(starts_f)) {
				all.data[[paste0(flag_name,var)]][starts_f[j]:ends_f[j]] <- qc_values$fail
			}
		}
	
		#---------------------------------
                # Detect sequences that will be flagged as SUSPECT data
                #---------------------------------
		runs_s = rle(all.data[[var]])
                myruns_s = which(runs_s$lengths == consec.suspect)
                runs_s.lengths.cumsum = cumsum(runs_s$lengths)
                ends_s = runs_s.lengths.cumsum[myruns_s]
                newindex_s = ifelse(myruns_s>1, myruns_s-1, 0)
                starts_s = runs_s.lengths.cumsum[newindex_s] + 1
                if (0 %in% newindex_s) starts_s = c(1,starts_s)               
        	
		if (length(starts_s) != 0) {       
	                 for (j in length(starts_s)) {
         	               all.data[[paste0(flag_name,var)]][starts_s[j]:ends_s[j]] <- qc_values$suspect
                	}
		}
	
		#---------------------------------
                # PART 2: detect and remove sequences of zeros (could be NAs between them!)
                #---------------------------------
                for (freq in freqs) {

                        freq.in.sec <- freq2sec(freq)
	
			#---------------------------------
        	        # 30-day moving window, to flag long flat lines mixed with NA's 
			#---------------------------------
			N <- 30*24*3600/freq.in.sec

			#---------------------------------
                	# Compute max min differences in periods of 30 days
                	#---------------------------------
			idx_fail <- rollapply(all.data[[var]][all.data$freq == freq],N,function(x)
                        	    {ifelse(all(is.na(x)),NA, max(x,na.rm=T)-min(x,na.rm=T))}) == 0
                	   
			#---------------------------------
                        # Inspect the last 30-day period manually
                        #---------------------------------
			len <- length(all.data[[var]][all.data$freq == freq])
			idx_fail_end <- rollapply(all.data[[var]][all.data$freq == freq][(len-N+2):len],N-1,function(x)
                                    {ifelse(all(is.na(x)),NA, max(x,na.rm=T)-min(x,na.rm=T))}) == 0

			idx_fail <- c(idx_fail,rep(idx_fail_end,N-1))

                	#---------------------------------
                	# Flag all the period
                	#---------------------------------
                	idx_fail <- apply(do.call(rbind,shift(idx_fail,0:(N-1),T)),2,function(x) {all(x)})

                	all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_fail] <- qc_values$fail
		}
		
		#---------------------------------
                # Missing data
                #---------------------------------
		all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing
	}

	return(all.data)
}
                
#----------------------------------------
# Name: Step test.
# Aim : Detect and flag those pairs of consecutive wind speed values which difference exceeds a certain threshold
#	This threshold depends on the time stamp interval sampling
# Flags:
#   Fail:    For 2-minutely data or lower, the difference between two consecutive wind speed values exceeds 20 m s-1 (WM0 standard)
#            For higher time stamp intervals, the threshold to flag data as 'fail' is set to 10 m s-1 (Jimenez et al 2010 uses 10 m s-1 for 10- and 30-minutely)
#-----------------------------------------
step_test <- function(all.data) {
	
	flag_name <- "flagst_"

	#---------------------------------
        # Variables to be quality controlled
	# Data resolution
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
	freqs <- unique(all.data[["freq"]])

	#---------------------------------
        # Loop all wind speed variables
        #---------------------------------
	for (var in grep("^windagl",filtvars,value=TRUE)) {
		#---------------------------------
	        # Loop all frequencies.
       		# Note that the threshold value in this test depends on data time resolution
        	#---------------------------------
		for (freq in freqs) {
	
	                freq.in.sec <- freq2sec(freq)
	
                	#---------------------------------
                	# Maximum threshold value between two consecutive values
                	#---------------------------------
                	if (freq.in.sec <= 120) {high_threshold <- 20}
                	else {high_threshold <- 10}
			
			all.data[[paste0(flag_name,var)]][all.data$freq == freq] <- qc_values$pass

			#---------------------------------
	        	# Compute differences between two consecutive values
        		#---------------------------------
			diff_df <- abs(diff(all.data[[var]][all.data$freq == freq]))
			idx_fail <- diff_df>=high_threshold
			idx_fail <- c(idx_fail,FALSE) | c(FALSE,idx_fail)

			all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_fail] <- qc_values$fail
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][!idx_fail] <- qc_values$pass
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][is.na(idx_fail)] <- qc_values$not.eval
		}
	
		#---------------------------------
                # Missing data
                #---------------------------------
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing
	}

	return(all.data)
}

#----------------------------------------
# Name: Persistence test.
# Aim:  Detect and flag sequences of short periods with extremely low variability. 
# Flags: 
#   Suspect: For 1-minutely observations, wind speed does not change more than 0.5 m s -1 in 60 consecutive values. (based on a WMO standard for instantaneous values)
#            For more than 1-minutely averaged observations, wind speed does not change more than 0.7 m s -1 in 60 consecutive values.
#   	     For 1-minutely observations, wind direction does not change more than 10 degrees in 60 consecutive values. (based on a WMO standard for instantaneous values)
#	     For more than 1-minutely averaged observations, wind direction does not change more than 5 degrees in 60 consecutive values.
#   Calms:   Wind speeds lower than 0.5 m s-1 are flagged as 'calm' and the persistence test should not be applied over these observations. 
#----------------------------------------
persistence_test <- function(all.data) {
	
	flag_name <- "flagpt_"	

	#---------------------------------
        # Variables to be quality controlled
	# Data resolution
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
	freqs <- unique(all.data[["freq"]])
	calm_threshold <- 0.5 # m s-1 

	#---------------------------------
        # Moving window width
        #---------------------------------
        N <- 60

	#---------------------------------
        # Loop all frequencies.
        # Note that the threshold value in this test depends on data time resolution
        #---------------------------------
        for (freq in freqs) {
		
		freq.in.sec <- freq2sec(freq)

		#---------------------------------
                # Threshold value between two consecutive values
                #---------------------------------
                if (freq.in.sec <= 60) {
			ws_threshold <- 0.5 #m s-1
			wd_threshold <- 10 # deg
		}
                
		#---------------------------------
                # And for higher interval averages
                #---------------------------------
		else {
			ws_threshold <- 0.7 #m s-1
			wd_threshold <- 5 # deg
		}
		
		#---------------------------------
        	# Loop all wind speed variables
        	#---------------------------------
		for (var in grep("^windagl",filtvars,value=TRUE)) {

			all.data[[paste0(flag_name,var)]][all.data$freq == freq] <- qc_values$not.eval 

			#---------------------------------
                	# Get difference between lowest and highest wind value.
			# If more than 75% of the data within the interval is NA, we do not take the difference 
			#---------------------------------
			idx_pass <- rollapply(all.data[[var]][all.data$freq == freq],N,function(x)
				 {ifelse(sum(is.na(x))/length(x)>0.75,NA, max(x,na.rm=T)-min(x,na.rm=T))}) > ws_threshold
			idx_pass <- c(idx_pass,rep(T,(N-1)))
		
			#---------------------------------
                	# Flag all the period
                	#---------------------------------
			idx_pass <- apply(do.call(rbind,shift(idx_pass,0:(N-1),T)),2,function(x) {all(x)})
			
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_pass] <- qc_values$pass
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][!idx_pass] <- qc_values$suspect
			
			#---------------------------------
                	# Change those flags corresponding to wind calms
                	#---------------------------------
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][all.data[[var]] < calm_threshold] <- qc_values$calm
									
			#---------------------------------
                	# Missing data
                	#---------------------------------
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][is.na(all.data[[var]])] <- qc_values$missing
		}
		
		#---------------------------------
        	# Loop all wind direction variables
        	#---------------------------------
        	for (var in grep("^wdiragl",filtvars,value=TRUE)) {

                	all.data[[paste0(flag_name,var)]][all.data$freq == freq] <- qc_values$not.eval

        		#---------------------------------
                	# Get difference between lowest and highest wind value.
			# If more than 75% of the data of the interval is NA, we do not take the difference  
                	#---------------------------------
                	idx_pass <- rollapply(all.data[[var]][all.data$freq == freq],N,function(x) 
				{ifelse(sum(is.na(x))/length(x)>0.75,NA, max(x,na.rm=T)-min(x,na.rm=T))}) > wd_threshold
                	idx_pass <- c(idx_pass,rep(T,N-1))
                
			#---------------------------------
               		# Flag all the period
                	#---------------------------------
			idx_pass <- apply(do.call(rbind,shift(idx_pass,0:(N-1),T)),2,function(x) {all(x)})
	
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_pass] <- qc_values$pass
                	all.data[[paste0(flag_name,var)]][all.data$freq == freq][!idx_pass] <- qc_values$suspect
                			
			#---------------------------------
                	# Missing data
                	#---------------------------------
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][is.na(all.data[[var]])] <- qc_values$missing
		}
	}		

	return(all.data)
}

#----------------------------------------
# Name: Differences between extreme values (of the empirical distribution)
# Aim:  Detect and flag abnormally extreme values of the time series. Checks the difference between the maximum and the second maximum values
#	of the empirical distribution
# Flags:
#   Suspect: the difference betweeen the maximum and the second maximum exceeds the absolute value of the second maximum 
#----------------------------------------
diff_extreme_values <- function(all.data) {

	flag_name <- "flagdev_"

	#---------------------------------
        # Variables in the data table. This test will be only applied over wind speed data
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")

	#---------------------------------
        # Loop all wind speed variables
        #---------------------------------
        for (var in grep("^windagl",filtvars,value=TRUE)) {
		
		all.data[[paste0(flag_name,var)]] <- factor(qc_values$pass,levels=unlist(qc_values))

        	#---------------------------------
		# Loop forever until there is no extreme value flagged by the test
        	#---------------------------------
		sorted.vals <- sort(all.data[[var]],decreasing=TRUE)
		while(T) {
        	        first_max <- sorted.vals[1]
                	second_max <- sorted.vals[2]

	                if ((!is.na(first_max) & !is.na(second_max)) & ((first_max - second_max) > second_max)) {
				all.data[[paste0(flag_name,var)]][which(all.data[[var]]==first_max)] <- qc_values$suspect
				sorted.vals <- sorted.vals[2:length(sorted.vals)]
	                } else { 
				break
			} 
		}

		#---------------------------------
                # Missing data
                #---------------------------------
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]]) == TRUE] <- qc_values$missing
	}

	return(all.data)
}

#----------------------------------------
# Name: Rate of change.
# Aim:  Detect and flag those pairs of consecutive wind speed values which difference exceeds a certain threshold
#	This threshold depends on the statistical distribution of the observations.
# Flags:
#   Fail: The difference between two consecutive observations is higher than 3*IQR
#	  (where IQR is the interquartile range of the time series)
#   Suspect: The difference between two consecutive values is higher than 2*IQR
#----------------------------------------
rate_change <- function(all.data) {
	
	flag_name <- "flagrc_"	

	#---------------------------------
        # Variables in the data table
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")

	#---------------------------------
        # Group data by months
        #---------------------------------
	all.data$yearmon <- as.Date(as.yearmon(all.data$times))
	
	#---------------------------------
        # Loop all wind speed variables
        #---------------------------------
	for (var in grep("windagl",filtvars,value=TRUE)) {
		
		all.data[[paste0(flag_name,var)]] <- factor(qc_values$not.eval,levels=unlist(qc_values))

		#---------------------------------
                # Compute the Interquartile Range grouped by months
                #---------------------------------
		all.data[,iqr:=ifelse((sum(is.na(all.data[[var]]))/(length(all.data[[var]])))>0.75,NA,
				IQR(base::get(var),na.rm=T)),by=yearmon]
	
		#---------------------------------
                # Compute differences between two consecutive values
                #---------------------------------
                all.data[["diff_df"]] <- c(abs(diff(all.data[[var]])),NA)
                
		idx_susp <- all.data$diff_df >= 2*all.data$iqr
		
		#---------------------------------
                # Flag all the suspect period and eliminate the last repeated NA
                #---------------------------------
		idx_susp <- head(c(idx_susp,FALSE) | c(FALSE,idx_susp),-1)
		
		idx_fail <- all.data$diff_df >= 3*all.data$iqr
	
		#---------------------------------
                # Flag all the fail period and eliminate the last repeated NA
                #---------------------------------
                idx_fail <- head(c(idx_fail,FALSE) | c(FALSE,idx_fail),-1)

		all.data[[paste0(flag_name,var)]][idx_susp] <- qc_values$suspect
                all.data[[paste0(flag_name,var)]][idx_fail] <- qc_values$fail
                all.data[[paste0(flag_name,var)]][!idx_fail & !idx_susp] <- qc_values$pass
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing
	
		all.data$iqr <- NULL
	}
	
	#---------------------------------
        # Finally, clean the data table
        #---------------------------------
        all.data$yearmon <- NULL
	all.data$diff_df <- NULL

	return(all.data)
}

#----------------------------------------
# Name: Repeated sequences. 
# Aim:  Detect and flag sequences of consecutive observations that appear two times or more within the same time series
# Flags:
#   Fail: 20 or 30 repeated measurements (depending on the resolution of data) of wind speed in the same time series
#         30 repeated measurements of wind direction in the same time series
#----------------------------------------
repeated_sequences <- function(all.data) {

	flag_name <- "flagrs_"
	
	#---------------------------------
        # Variables to be quality controlled
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")

	#---------------------------------
        # 2 Auxiliary functions that will be used later
	# decimalplaces: get the decimal positions of data
	#   (From: https://stackoverflow.com/questions/5173692/how-to-return-number-of-decimal-places-in-r)
        # getmode: compute the mode of a numeric vector
	#---------------------------------
	decimalplaces <- function(x) {
		return(ifelse((x %% 1) != 0, nchar(strsplit(sub('0+$', '', as.character(x)), ".", fixed=TRUE)[[1]][[2]]), 0))
	}

	decplaces <- function(x) {
		for (i in 0:100) {
			if ((x %% 1) == 0) {
				return(i)
			}
			else {
				x <- x*10
			}
		}
	}

	getmode <- function(v) {
   		uniqv <- unique(v)
   		uniqv[which.max(tabulate(match(v, uniqv)))]
	}
	
	#---------------------------------
        # Auxliary function to detect repeated sequences of length N
        #   (From: https://stackoverflow.com/questions/48660606/find-a-sequence-of-numbers-in-a-vector)
        #---------------------------------
        identRptSeq <- function(x, N) {
	        #---------------------------------
                # Create groups to split input vector in
                #---------------------------------
                splits <- ceiling(seq_along(x) / N)

                #---------------------------------
                # Use data.table shift to create overlapping windows
                #---------------------------------
                foo <- lapply(data.table::shift(x, 0:(N-1), type = "lead"), function(i) {
           		res <- split(i, splits)
                        res[lengths(res) == N]})

                foo <- na.omit(t(as.data.frame(foo)))

                #---------------------------------
                # Find duplicated windows
                #---------------------------------
                rep.seqs <- as.data.frame(foo[duplicated(foo), ])

                #---------------------------------
                # Remove duplicated repeated sequences (useful when we have long sequences of repeated values)
                #---------------------------------
                rep.seqs <- unique(rep.seqs)

                #---------------------------------
		# Remove flat line seqs
                #---------------------------------
		rep.seqs <- rep.seqs[apply(rep.seqs,1,function(x) { length(unique(x)) } ) >1,]

                return(rep.seqs)
        }

	#---------------------------------
        # Loop both wind speed and wind direction variables
        #---------------------------------
	for (var in filtvars) {
		#---------------------------------
                # Set maximum number of repeated elements allowed
		# To do that, select 10 random elements and see how many decimals do the data have
                #---------------------------------
		rdm.data <- sample(all.data[[var]][!is.na(all.data[[var]])],10)
		ord.data <- unique(sort(all.data[[var]][!is.na(all.data[[var]])]))
		dec <- -log10(getmode(round(diff(ord.data),4)))
		#dec <- getmode(as.numeric(lapply(rdm.data,function(x) decimalplaces(x))))

		#---------------------------------
	        # Set maximum number of repeated elements allowed
	      	#---------------------------------
		if (grepl("windagl",var) == TRUE & dec >= 1) { N <- 20 }
		else if (grepl("windagl",var) == TRUE & dec == 0) { N <- 30 } 	
	
		# For wind direction
		else {N <- 30}	
			
		all.data[[paste0(flag_name,var)]] <- factor(qc_values$pass,levels=unlist(qc_values))	

		#---------------------------------
                # 'rep.seqs' is a data frame in which each row is one repeated sequence
                #---------------------------------
                rep.seqs <- identRptSeq(all.data[[var]],N)
	
		#---------------------------------
               	# Find the repeated sequences in the dataset. Only if we have repeated sequences
               	#---------------------------------
		if (nrow(rep.seqs) != 0) {
			for (j in seq(nrow(rep.seqs))) {      	
				x <- as.numeric(rep.seqs[j,])
        
				idx <- which(all.data[[var]] == x[1])
       				
				beg.seq <- which(rowSums(mapply('==',shift(all.data[[var]], 
						type = 'lead', n = 0:(length(x) - 1)),x)) == length(x))
						
				for (k in seq(length(beg.seq))) {	
					all.data[[paste0(flag_name,var)]][beg.seq[k]:(beg.seq[k]+(N-1))] <- 
													qc_values$fail
				}
			}
		}
		
		#---------------------------------
                # Missing data
                #---------------------------------
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing
	}

	return(all.data)
}
		
#----------------------------------------
# Name: Abnormal variations
# Aim:  Detect inadequate variations (abnormally high or abnormally low) within the time series. This tests computes 30-day moving variances over wind speed time series
# Flags:
#   Suspect: periods with standard deviation out of the range: mean(SD_30day)+-4*SD(SD_30day)
#	     Where:
# 	       mean(SD_30day) stands for the mean standard deviation of all the 30-day moving standard deviations in the time series
#	       SD(SD_30day) stands for the standard deviation of all the 30-day moving standard deviations in the time series
# Note: this test will require more memory as the time stamp sampling decreases. It could take several minutes to run this routine depending on the amount of data.
#----------------------------------------
abnormal_variations <- function(all.data) {
	
	flag_name <- "flagav_"

	#---------------------------------
        # Variables in the dataset and time stamp sampling
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
	freqs <- unique(all.data[["freq"]])

	#---------------------------------
        # Loop all wind speed variables
        #---------------------------------
        for (var in grep("^windagl",filtvars,value=TRUE)) {
			
		#---------------------------------
	        # Given a variable, loop all the frequencies
        	#---------------------------------
		for (freq in freqs) {
		
		        freq.in.sec <- freq2sec(freq)

                	#---------------------------------
                	# 30-day moving window width.
                	# See how many time stamps do we have within the 30 day period
                	#---------------------------------
                	N <- 30*24*3600/freq.in.sec

			all.data[[paste0(flag_name,var)]][all.data$freq == freq] <- qc_values$not.eval
		
			#---------------------------------
	        	# Compute 30-day moving variances
			# If more than half of the data is missing, do not compute the SD
        		#---------------------------------
			all.data[[paste0("SD_",var)]][all.data$freq == freq] <- 
				rollapply(all.data[[var]][all.data$freq == freq],N,
				function(x) {ifelse(sum(is.na(x))/length(x)>0.5,NA,sd(x,na.rm=T))},
						fill=NA,align=c("left"))
			
        		#---------------------------------
			# Compute thresholds for strange SD values (i.e mean(sd)+-3*sd(sd)
        		#---------------------------------
			mean.sd <- mean(all.data[[paste0("SD_",var)]][all.data$freq == freq],na.rm=TRUE)
			sd.sd <- sd(all.data[[paste0("SD_",var)]][all.data$freq == freq],na.rm=TRUE)
			max.4sd <- mean.sd + 4*sd.sd
			min.4sd <- mean.sd - 4*sd.sd

        		#---------------------------------
			# Set values above/below threshold to suspect
        		#---------------------------------
			idx_susp <- all.data[[paste0("SD_",var)]][all.data$freq == freq] < min.4sd | 
					all.data[[paste0("SD_",var)]][all.data$freq == freq] > max.4sd
                	idx_susp <- apply(do.call(rbind,shift(idx_susp,0:N,F)),2,function(x) {any(x)})

			idx_na <- is.na(all.data[[paste0("SD_",var)]][all.data$freq == freq])
                	idx_na <- apply(do.call(rbind,shift(idx_na,0:N,F)),2,function(x) {any(x)})

			all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_na] <- qc_values$not.eval
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_susp] <- qc_values$suspect
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][!idx_susp] <- qc_values$pass
		}
		
		#---------------------------------
                # Missing data
                #---------------------------------
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing

		#---------------------------------
        	# Remove computing columns
	        #---------------------------------
		all.data[[paste0("SD_",var)]] <- NULL
	}
	
	return(all.data)
}

#----------------------------------------
# Name: Systematic errors
# Aim:  Detect random errors in the experimental measurements. This tests computes 30-day moving averages over wind time series
# Flags:
#   Suspect: periods with standard deviation out of the range: mean(mean_30day)+-4*SD(mean_30day)
#	     Where:
# 	       mean(mean_30day) stands for the mean of all the 30-day moving means in the time series
#	       SD(mean_30day) stands for the standard deviation of all the 30-day moving means in the time series
# Note: this test will require more memory as the time stamp sampling decreases. It could take several minutes to run this routine depending on the amount of data.
#----------------------------------------
systematic_errors <- function(all.data) {
	
	flag_name <- "flagse_"
	
	#---------------------------------
        # Variables in the dataset and time stamp sampling
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
        freqs <- unique(all.data[["freq"]])

	#---------------------------------
        # Loop all variables
        #---------------------------------
        for (var in grep("^windagl",filtvars,value=TRUE)) {
		#---------------------------------
	        # Given a variable, loop all frequencies.
        	#---------------------------------
        	for (freq in freqs) {

                	freq.in.sec <- freq2sec(freq)
	
        	        #---------------------------------
                	# 30-day moving window width.
               		# See how many time stamps do we have within the 30 day period
                	#---------------------------------
                	N <- 30*24*3600/freq.in.sec

			all.data[[paste0(flag_name,var)]][all.data$freq == freq] <- qc_values$not.eval	
			
			#---------------------------------
	        	# Compute 30-day moving SD
			# if more than half of the data is missing, do not compute SD
        		#---------------------------------
			all.data[[paste0("MEAN_",var)]][all.data$freq == freq] <-
                                rollapply(all.data[[var]][all.data$freq == freq],N,
                                function(x) {ifelse(sum(is.na(x))/length(x)>0.5,NA,mean(x,na.rm=T))},
                                                fill=NA,align=c("left"))

        		#---------------------------------
			# Compute thresholds for strange MEAN values (i.e mean(mean)+-3*sd(mean)
        		#---------------------------------
			mean.mean <- mean(all.data[[paste0("MEAN_",var)]][all.data$freq == freq],na.rm=TRUE)
			sd.mean <- sd(all.data[[paste0("MEAN_",var)]][all.data$freq == freq],na.rm=TRUE)
			max.4sd <- mean.mean + 4*sd.mean
			min.4sd <- mean.mean - 4*sd.mean

        		#---------------------------------
			# Set values above/below threshold to suspect
        		#---------------------------------
			idx_susp <- all.data[[paste0("MEAN_",var)]][all.data$freq == freq] < min.4sd | 
				    all.data[[paste0("MEAN_",var)]][all.data$freq == freq] > max.4sd
                	idx_susp <- apply(do.call(rbind,shift(idx_susp,0:N,F)),2,function(x) {any(x)})

			idx_na <- is.na(all.data[[paste0("MEAN_",var)]][all.data$freq == freq])
                	idx_na <- apply(do.call(rbind,shift(idx_na,0:N,F)),2,function(x) {any(x)})

			all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_na] <- qc_values$not.eval
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_susp] <- qc_values$suspect
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][!idx_susp] <- qc_values$pass
		}
		
		#---------------------------------
                # Missing data
                #---------------------------------
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing

                #---------------------------------
                # Remove auxiliary columns
                #---------------------------------
                all.data[[paste0("MEAN_",var)]] <- NULL
	}

	return(all.data)
}

#----------------------------------------
# Name: Quartile occurrences.
# Aim:  Detect random errors in the experimental measurements. This tests looks for the frequency of appearance of the first, second and third quartiles.
# Flags:
#   Fail: 
#     30-day period without any observation within the first or third quartiles
#     10-day period with all observations within the first or third quartiles
#     20-day period with all observations above or below the second quartile (median)  
#   Suspect:
#     15-day period without any observation within the first or third quartiles
#     5-day period with all observations within the first or third quartiles
#     10-day period with all observations above or below the second quartile (median)  
#----------------------------------------
quartile_occurrences <- function(all.data) {
	
	flag_name <- "flagqo_"

        #---------------------------------
        # Variables in the dataset
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
	
	#---------------------------------
        # Preliminary flagging of values below (0) or above (1) quartiles
        #---------------------------------
	interval.flags <- c(0,1)

	#---------------------------------
        # Loop all wind speed variables
        #---------------------------------
        for (var in grep("^windagl",filtvars,value=TRUE)) {
		
		all.data[[paste0(flag_name,var)]] <- qc_values$pass
	
		if (any(grepl(paste0("flagfl_",var),names(all.data))) == FALSE | 
		    any(grepl(paste0("flagpv_",var),names(all.data))) == FALSE) {
			#---------------------------------
        		# To compute the quartiles of the time series, we need first to remove unrealistic measurements
        		#---------------------------------		
			stop("You must run first 'Flat line' and 'Plausible values' tests")
		}
		
		#---------------------------------
                # Computation of the first, second and third quartile
		# Compute these statistics removing those values that failed to pass both flat line 
		# and plausible values test. Also remove suspectv values flagged by the plausible values check
                #---------------------------------
		quartiles <- c(quantile(all.data[[var]][all.data[[paste0("flagfl_",var)]] != 4 &
							all.data[[paste0("flagpv_",var)]] != 4 &
							all.data[[paste0("flagpt_",var)]] != 5 &
							all.data[[paste0("flagpv_",var)]] != 2], 1/4, na.rm=T),
			       quantile(all.data[[var]][all.data[[paste0("flagfl_",var)]] != 4 &
                                                        all.data[[paste0("flagpv_",var)]] != 4 &
							all.data[[paste0("flagpt_",var)]] != 5 &
                                                        all.data[[paste0("flagpv_",var)]] != 2], 2/4, na.rm=T),
			       quantile(all.data[[var]][all.data[[paste0("flagfl_",var)]] != 4 &
                                                        all.data[[paste0("flagpv_",var)]] != 4 &
							all.data[[paste0("flagpt_",var)]] != 5 &
                                                        all.data[[paste0("flagpv_",var)]] != 2], 3/4, na.rm=T))
						     		
		for (quart in seq(quartiles)) {
			#---------------------------------
        		# Flag data depending on the interval they belong to
			# 0: (-inf,quantile)
			# 1: [quantile,inf)
			#---------------------------------
			all.data[["int"]] <- NULL
			all.data[["int"]] <- findInterval(round(all.data[[var]],1),round(quartiles[quart],1))
		
			for (int.flag in interval.flags) {
				#---------------------------------
        		        # Look for the rarest sequences
                		#---------------------------------
				if (int.flag == 0) {compl.value = 1}
				else if (int.flag == 1) {compl.value = 0}

				runs <- data.table(values=rle(all.data[["int"]])$values, 
						   lengths=rle(all.data[["int"]])$lengths,
						   freq=freq2sec(rle(all.data[["freq"]])$values))
	
				#---------------------------------
                                # Transform the periods from seconds to days
                                #---------------------------------
				runs[["len*freq"]] = (runs[["lengths"]]*runs[["freq"]])/86400
				
				#---------------------------------
                                # Threshold (in days) to flag data as Fail, Suspect or Pass
                                #---------------------------------
				if ((quart == 1 & int.flag == 0) | (quart == 3 & int.flag == 1)) {
					ndays_1 = 15
					ndays_2 = 30
				} else if ((quart == 3 & int.flag == 0) | (quart == 1 & int.flag == 1)) {
					ndays_1 = 5
					ndays_2 = 10
				} else if (quart == 2) {
					ndays_1 = 10
					ndays_2 = 20
				}
					
				#---------------------------------
                                # Detect and flag as SUSPECT the longest sequences without any appearance of values
                                # within the first/second/third quartile
                                #---------------------------------
                                myruns_s = which(runs[["len*freq"]] >= ndays_1)
                                myruns_s <- myruns_s[runs[myruns_s,]$value == compl.value]
				runs_s.lengths.cumsum = cumsum(runs$lengths)
                                ends_s = runs_s.lengths.cumsum[myruns_s]
                                newindex_s = ifelse(myruns_s>1, myruns_s-1, 0)
                                starts_s = runs_s.lengths.cumsum[newindex_s] + 1
                                if (0 %in% newindex_s) starts_s = c(1,starts_s)

				if (length(starts_s) != 0) {
                                        #---------------------------------
                                        # Flag all the period
                                        #---------------------------------
                                        for (p in seq(length(starts_s))) {
                                                
						all.data[[paste0(flag_name,var)]][starts_s[p]:ends_s[p]] <- qc_values$suspect
                                        }
                                }

				#---------------------------------
                                # Detect and flag as FAIL the longest sequences without any appearance of values
                                # within the first/second/third quartile
                                #---------------------------------
                                myruns_f = which(runs[["len*freq"]] >= ndays_2)
                                myruns_f <- myruns_f[runs[myruns_f,]$value == compl.value]
				runs_f.lengths.cumsum = cumsum(runs$lengths)
                                ends_f = runs_f.lengths.cumsum[myruns_f]
                                newindex_f = ifelse(myruns_f>1, myruns_f-1, 0)
                                starts_f = runs_f.lengths.cumsum[newindex_f] + 1
                                if (0 %in% newindex_f) starts_f = c(1,starts_f)

                                if (length(starts_f) != 0) {
                                        #---------------------------------
                                        # Flag all the period
                                        #---------------------------------
                                        for (k in seq(length(starts_f))) {
                                                
						all.data[[paste0(flag_name,var)]][starts_f[k]:ends_f[k]] <- qc_values$fail
                                        }
                                }
			}
		}
		
		all.data[["int"]] <- NULL
	
		#---------------------------------
                # Missing data
                #---------------------------------
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing
	}
	
	return(all.data)
}

#----------------------------------------
# Name: Vertical ratios
# Aim:  Detect and flag erroneous measurements by comparing simultaneous observations at different tower measurement levels.
#       This test computes the ratios between pairs of simultaneous wind speed measurements at different heights.
# Flags: 
#   Fail: The ratio between two observations at different levels is mean(ratio)+-30
#  	  (where mean(ratio) stands for the average of all the ratios between pairs of simultaneous measurements
#   Suspect: The ratio between two observations at different levels is mean(ratio)+-15
#  	  (where mean(ratio) stands for the average of all the ratios between pairs of simultaneous measurements
#----------------------------------------
vertical_ratios <- function(all.data) {	
	
	flag_name <- "flagvr_"

	#---------------------------------
        # Original attributes
        #---------------------------------
	filtvars <- attr(all.data,'filtvars')
	heights <- attr(all.data,'heights')
	freq <- attr(all.data,'freq')
	sensors <- attr(all.data,'sensors')

        #---------------------------------
        # Wind speed variables (ONLY FOR SENSOR 1) in the dataset
	# Please, ensure that if you have redundant sensors, data have been properly merged        
	# Anemometer heights in decreasing order
	#---------------------------------
        filtvars_ws <- grep("^windagl.*S1",filtvars,value=T)
	heights_ws <- sort(heights$windagl,decreasing=T)

	#---------------------------------
        # Loop all anemometer heights
        #---------------------------------
	for (height in heights_ws[-length(heights_ws)]) {
		#---------------------------------
	        # Given a tower level, compute the ratio between simultaneous values taken at other measurement levels 
        	# Select only for wind speeds higher than 1 m s-1 and those that did not fail to pass successfully the plausible values test
		#---------------------------------
		for (low.hgt in heights_ws[heights_ws < height]) {

			all.data[[paste0("ws",height,"S1&&ws",low.hgt,"S1")]] <- 
				ifelse(all.data[[paste0("windagl",height,"S1")]] >= 1 & 
			       	       all.data[[paste0("windagl",low.hgt,"S1")]] >= 1 &
			               all.data[[paste0("flagpv_windagl",low.hgt,"S1")]] != 4 &
				       all.data[[paste0("flagpv_windagl",height,"S1")]] != 4,
			          all.data[[paste0("windagl",height,"S1")]]/(
				  	all.data[[paste0("windagl",low.hgt,"S1")]]),
			          NA)		
		}
	}
	
	added_cols <- grep("&&",names(all.data),value=T)
	
	#---------------------------------
        # Initialize the flag columns and flag no data
        #---------------------------------
	for (var in filtvars_ws) {
		all.data[[paste0(flag_name,var)]] <- factor(qc_values$pass,levels=unlist(qc_values))
		all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing
	}
	
	#---------------------------------
        # See the highest and lowest values for each ratio
        #---------------------------------
	for (add.col in added_cols) {
		
		mean.ratio <- mean(all.data[[add.col]],na.rm=T)

		vnames <- list()
		vnames[[1]] <- paste0("windagl",sub("ws(\\d+)S1&&.*","\\1",add.col),"S1")
		vnames[[2]] <- paste0("windagl",sub(".*&&ws(\\d+)S1","\\1",add.col),"S1")
			
		for (vname in vnames) { 
			all.data[[paste0(flag_name,vname)]][(all.data[[add.col]] >= (mean.ratio+15)) | 
					 (all.data[[add.col]] <= (mean.ratio-15))] <- qc_values$suspect
	
			all.data[[paste0(flag_name,vname)]][(all.data[[add.col]] >= (mean.ratio+30)) | 
                                         (all.data[[add.col]] <= (mean.ratio-30))] <- qc_values$fail
		}
	}

	#-------------------------------------
	# Set original attributes to the data table
	#-------------------------------------
	attr(all.data,'filtvars') <- filtvars
	attr(all.data,'heights')  <- heights
	attr(all.data,'freq')     <- freq
	attr(all.data,'sensors')  <- sensors

	#---------------------------------
        # Finally, clean the data table
        #---------------------------------
        all.data[,grep(".*&&.*",names(all.data),value=T)]  <- NULL
	
	return(all.data)
}

#----------------------------------------
# Name: Icing
# Aim:  Detect and flag icing periods when an anemometer has been frosted and produces erroneous measurements
# Flags:
#   Fail: 4 or more days with 0 m s-1 as the maximum wind speed value and below zero temperatures 
#----------------------------------------
icing <- function(all.data) {
	
	flag_name <- "flagi_"

	#---------------------------------
        # Save original attributes (height, frequency, sensors...)
        #---------------------------------
	wind.heights <- attr(all.data,"heights")[c("windagl","wdiragl")]
	term.heights <- attr(all.data,"heights")$taagl
	filtvars <- attr(all.data,"filtvars")
	freqs <- unique(all.data[["freq"]])
	sensors <- attr(all.data,"sensors")

	#---------------------------------
        # Ensure temperature data is available. Stop if not
        #---------------------------------
	if (!any(grepl("taagl.*",names(all.data))==TRUE)) {
		stop("Cannot run Icing test. Temperature data is needed to continue.")	
	}

	#---------------------------------
        # Loop all wind speed variables
        #---------------------------------
        for (var in grep("^windagl",filtvars,value=TRUE)) {
		#---------------------------------
        	# Look for the closest termometer
	        #---------------------------------
		hgt <- as.numeric(sub("windagl(\\d+)S.*","\\1",var))
		term.hgt <- term.heights[which.min(abs(term.heights - hgt))]
			
		temp.var <- paste0("taagl",term.hgt,"S1")
				
		#---------------------------------
        	# Loop all time stamp samplings
	        #---------------------------------
		for (freq in freqs) {
			
			freq.in.sec <- freq2sec(freq)

                        #---------------------------------
                        # 4-day moving window width.
                        # Compute the time stamp number we have in the 4 day period
                        #---------------------------------
                        N <- 4*24*3600/freq.in.sec

                        all.data[[paste0(flag_name,var)]][all.data$freq == freq] <- qc_values$pass
		
			#---------------------------------
        	        # Maximum wind speed in each 4-day period. NA if we have less than 3 measurements
	                #---------------------------------
			all.data[[paste0("max_",var)]][all.data$freq == freq] <-
                                rollapply(all.data[[var]][all.data$freq == freq],N,
                                function(x) {ifelse(sum(!is.na(x))<3,NA,max(x,na.rm=T))},
                                                fill=NA,align=c("left"))
			
			#---------------------------------
                        # Maximum temperature in each 4-day period. NA if we have less than 3 measurements
                        #---------------------------------
			all.data[[paste0("max_",temp.var)]][all.data$freq == freq] <-
                                rollapply(all.data[[temp.var]][all.data$freq == freq],N,
                                function(x) {ifelse(sum(!is.na(x))<3,NA,max(x,na.rm=T))},
                                                fill=NA,align=c("left"))

			#---------------------------------
                        # Look for 4-day periods with 0 wind speeds and below 0ºC temperatures (273.15 K)
			# Flag as fail
                        #---------------------------------
			idx_fail <- all.data[[paste0("max_",var)]][all.data$freq == freq] == 0 &
                                    all.data[[paste0("max_",temp.var)]][all.data$freq == freq] < 273.15 #K 
                        
			idx_fail <- apply(do.call(rbind,shift(idx_fail,0:N,F)),2,function(x) {any(x)})
		
			all.data[[paste0(flag_name,var)]][all.data$freq == freq][idx_fail] <- qc_values$fail
		
			#---------------------------------
                	# Missing data
        	        #---------------------------------
	                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing
		
			#---------------------------------
                        # Clean up data table
                        #---------------------------------
			all.data[[paste0("max_",var)]] <- NULL
			all.data[[paste0("max_",temp.var)]] <- NULL
		}
	}

	#---------------------------------
        # Remove temperature columns
        #---------------------------------
	all.data <- all.data[,!grep("^taagl.*",names(all.data)),with=F]		

	#-------------------------------------
        # Set original attributes to the data table
        #-------------------------------------
        attr(all.data,'filtvars') <- filtvars
        attr(all.data,'heights')  <- wind.heights
        attr(all.data,'freq')     <- freq
        attr(all.data,'sensors')  <- sensors

	return(all.data)	
}	

#----------------------------------------
# Name: Tower shadow
# Aim:  Look for wind directions that are in the wake produced by the vertical structure of the mast. 
# Flags:
#   Suspect: wind speed values which direction is in the wake of the tower. These directions are identified by computing
#            the ratio between wind speeds measured by redundant sensors. Then, this wind speed ratios are grouped by
#            wind direction. Wind directions which ratios fall below the 5th percentile and above 95th percentile are
#            flagged as suspect.
#----------------------------------------
tower_shadow <- function(all.data) {
	
	flag_name <- "flagts_"

        #---------------------------------
        # Variables in the dataset
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
	heights <- attr(all.data,"heights")

	#---------------------------------
        # Check if we have, at least, one height with redundant anemos
        #---------------------------------
	nsens <- attr(all.data,"sensors")
	if (nsens$windagl < 2) {
		print("This test only works in towers with redundant anemos")	
		quit()	
	}
	
	redund.vars <- grep("^windagl.*S([2-9])",filtvars,value=TRUE) 
	redund.heights <- unique(as.numeric(sub("^windagl(\\d+)S.*","\\1",redund.vars)))
	
	#---------------------------------
        # Loop all levels with redundant anemos
        #---------------------------------
	wake.dir <- NULL
	sec.means.plot <- NULL
	for (height in sort(redund.heights,decreasing=TRUE)) {
	
		#---------------------------------
        	# Variable names, heights and number of sensors at a certain level
        	#---------------------------------
		ws_vars <- grep(paste0("windagl",height),filtvars,value=TRUE)
		nsens <- max(as.numeric(sub("windagl.*S(\\d+)","\\1",ws_vars)))
		
		#---------------------------------
                # Get the nearest height where we have wind direction observations
		# In case we have redundant vanes, average wind direction
                #---------------------------------
		wd_height <- heights$wdiragl[which.min(abs(heights$wdiragl - height))] 
		wd_var <- grep(paste0("wdiragl",wd_height,".*"),filtvars,value=TRUE)
		wd_mean <- circular.avg(all.data[,wd_var,with=F])

		#---------------------------------
	        # Compute the ratios between redundant measurements
        	#---------------------------------
		sector.means <- NULL
		for (i in seq(length(ws_vars)-1)) {
				
			all.data[[paste0(ws_vars[i],"&&",ws_vars[i+1])]] <- 
				ifelse(all.data[[ws_vars[i]]] >= 0.5 & all.data[[ws_vars[i+1]]] >= 0.5,
				       all.data[[ws_vars[i]]]/(all.data[[ws_vars[i+1]]]),
				       NA)
								
			#---------------------------------
                        # In case we do not have overlapping of data, we jump to the next measurement level
                        #---------------------------------
			if(all(is.na(all.data[[paste0(ws_vars[i],"&&",ws_vars[i+1])]]))) {
				print(paste0("Data from redundant sensors at ",height, " m do not overlap. Quitting.")) 
				quit()
			}

			#---------------------------------
        		# Average data in sectors of 1 deg
	        	#---------------------------------
			sector.means[[paste0(ws_vars[i],"&&",ws_vars[i+1])]] <- aggregate(
				  x = all.data[[paste0(ws_vars[i],"&&",ws_vars[i+1])]],
                                  by = list(cut(all.data[[paste0("wdiragl",wd_height,"S1")]],breaks=seq(0,360,1))),
				  FUN = mean, na.rm=TRUE)
		
			names(sector.means[[paste0(ws_vars[i],"&&",ws_vars[i+1])]]) <- 
									c("sect",paste0(ws_vars[i],"&&",ws_vars[i+1]))
			
			sec.means.plot[[paste0(ws_vars[i],"&&",ws_vars[i+1])]] <- sector.means[[paste0(ws_vars[i],"&&",ws_vars[i+1])]]
		}

		sec.means.hgt <- Reduce(function(a,b) base::merge(x=a, y=b, by = "sect",all = TRUE),sector.means)

		for (sens in seq(nsens-1)) {
			#---------------------------------
        	        # For each redundant height, indentify those directions falling in the wake
                	#---------------------------------
			quant <- quantile(sec.means.hgt[,sens+1], probs = c(0.05,0.95))
			wake.dir[[ws_vars[sens]]] <- sec.means.hgt[["sect"]][sec.means.hgt[,sens+1] <= quant[1]]
			wake.dir[[ws_vars[sens+1]]] <- sec.means.hgt[["sect"]][sec.means.hgt[,sens+1] >= quant[2]]
		}
	}
		
	#---------------------------------
        # Flag as suspect those wind speed values falling in the wake
        #---------------------------------
	redundant <- unlist(lapply(redund.heights,function(x) grep(paste0("^windagl",x,".*"),filtvars,value=TRUE)))
	
	for (red in redundant) {
		all.data[[paste0(flag_name,red)]] <- qc_values$pass
		all.data[[paste0(flag_name,red)]][round(wd_mean,0) %in% as.numeric(wake.dir[[red]])] <- qc_values$suspect
		all.data[[paste0(flag_name,red)]][is.na(all.data[[red]])] <- qc_values$missing
	}
	
	#---------------------------------
        # Finally, clean the dataset
        #---------------------------------
	all.data[,grep(".*&&.*",names(all.data),value=T)]  <- NULL
	
	return(all.data)
}

#----------------------------------------
# Name: Isolated pass.
# Aim:  Detect and re-flag values that have been previously flagged as 'Pass' by some of the tests, but appear bounded by suspicious or wrong observations
# Flags:
#   Fail:    c(rep("Fail",3),rep("Pass",1),rep("Fail",3))
#            c(rep("Fail",5),rep("Pass",2),rep("Fail",5))
#	     c(rep("Fail",10),rep("Pass",3),rep("Fail",10))
#	     c(rep("Fail",15),rep("Pass",4),rep("Fail",15))
#	     c(rep("Fail",25),rep("Pass",5),rep("Fail",25))
#	     c(rep("Missing",50),rep("Pass",1),rep("Missing",50))
#	     c(rep("Missing",50),rep("Calm",1),rep("Missing",50)) 
#   Suspect: c(rep("Suspect",3),rep("Pass",1),rep("Suspect",3))	
#	     c(rep("Suspect",5),rep("Pass",2),rep("Suspect",5))
#	     c(rep("Suspect",10),rep("Pass",3),rep("Suspect",10))
#	     c(rep("Suspect",15),rep("Pass",4),rep("Suspect",15))
#	     c(rep("Suspect",25),rep("Pass",5),rep("Suspect",25))
#----------------------------------------
isolated_pass <- function(all.data) {
	
	flag_name <- "flagip_"

	#---------------------------------
        # Variables to be quality controlled
        #---------------------------------
	filtflagvars <- unique(grep("flag.*",names(all.data),value=T))
	
	#---------------------------------
        # Checks if any QC test has been applied
        #---------------------------------
	if (length(filtflagvars) == 0){
		stop("This test must be applied after running other routines")
	}	
	
	#---------------------------------
        # Explicit definition of the suspicious sequences
        #---------------------------------
	isol.pass <- list()	
	isol.pass[[1]] <- c(rep(4,3),rep(1,1),rep(4,3)) # c(rep("Fail",3),rep("Pass",1),rep("Fail",3))
	isol.pass[[2]] <- c(rep(4,5),rep(1,2),rep(4,5)) # c(rep("Fail",5),rep("Pass",2),rep("Fail",5))
	isol.pass[[3]] <- c(rep(4,10),rep(1,3),rep(4,10)) # c(rep("Fail",10),rep("Pass",3),rep("Fail",10))
	isol.pass[[4]] <- c(rep(4,15),rep(1,4),rep(4,15)) # c(rep("Fail",15),rep("Pass",4),rep("Fail",15))
	isol.pass[[5]] <- c(rep(4,25),rep(1,5),rep(4,25)) # c(rep("Fail",25),rep("Pass",5),rep("Fail",25))
	isol.pass[[6]] <- c(rep(2,3),rep(1,1),rep(2,3)) # c(rep("Suspect",3),rep("Pass",1),rep("Suspect",3))	
	isol.pass[[7]] <- c(rep(2,5),rep(1,2),rep(2,5)) # c(rep("Suspect",5),rep("Pass",2),rep("Suspect",5))
        isol.pass[[8]] <- c(rep(2,10),rep(1,3),rep(2,10)) # c(rep("Suspect",10),rep("Pass",3),rep("Suspect",10))
        isol.pass[[9]] <- c(rep(2,15),rep(1,4),rep(2,15)) # c(rep("Suspect",15),rep("Pass",4),rep("Suspect",15))
        isol.pass[[10]] <- c(rep(2,25),rep(1,5),rep(1,25))  # c(rep("Suspect",25),rep("Pass",5),rep("Suspect",25))
	isol.pass[[11]] <- c(rep(9,50),rep(1,1),rep(9,50)) # c(rep("Missing",50),rep("Pass",1),rep("Missing",50))
	isol.pass[[12]] <- c(rep(9,50),rep(5,1),rep(9,50)) # c(rep("Missing",50),rep("Calm",1),rep("Missing",50)) 
	
	#---------------------------------
        # Look for sequences with 'Pass' flags surrounded by 'Fail' flags
        #---------------------------------
	for (flagvar in filtflagvars) {
			
		var <- sub("flag.*_(\\w+)","\\1",flagvar)
		
		#---------------------------------
	        # Define the column if it does not exist
        	#---------------------------------
		if (!(paste0(flag_name,var) %in% names(all.data))) {
			all.data[[paste0(flag_name,var)]] <- factor(qc_values$pass,levels=unlist(qc_values))
		} 		

		#---------------------------------
	        # Loop all suspicious sequences defined above
        	#---------------------------------
		for (j in seq(isol.pass)) {
	
			x <- isol.pass[[j]]
		
			beg.seq <- which(rowSums(mapply('==',shift(all.data[[flagvar]],
                                                type = 'lead', n = 0:(length(x) - 1)),x)) == length(x))

			#---------------------------------
        	        # Flag the isolated pass according to neighbor flags
	                #---------------------------------
			if (j %in% c(1,2,3,4,5,11,12)) {
				ctrl.flag <- qc_values$fail
			} else { 
				ctrl.flag <- qc_values$suspect
			}

                        #---------------------------------
                        # Flag all the period (only if any sequence has been found)
                        #---------------------------------
                        if (length(beg.seq) != 0) {
			        for (k in seq(length(beg.seq))) {
                                        all.data[[paste0(flag_name,var)]][beg.seq[k]:(beg.seq[k]+(length(x)-1))] <- ctrl.flag
                                }
			}
		}
	
		#---------------------------------
                # Missing data
                #---------------------------------
                all.data[[paste0(flag_name,var)]][is.na(all.data[[var]])] <- qc_values$missing
	}
	
	return(all.data)
}
	
#----------------------------------------
# Name: zeros and 360s.
# Aim:  compute the percentage of zeros and 360s values in the time series
# Flags: this test does not give any flag. It includes the attributes 
#	 attr(all.data,"perc.of.zeros")
#	 attr(all.data,"perc.of.360")
#	 to the data table.
# NOTE: The user must decide whether the time series should be removed or not, depending on the percentage of zeros and 360s
#----------------------------------------
zerosand360 <- function(all.data) {
	#---------------------------------
        # Variables in the dataset
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
	
	#---------------------------------
        # Check the frequency of zeros in wind speed and wind direction variables
	#---------------------------------
	perc.of.zeros <- list()	
	for (i in seq(filtvars)) {
		var <- filtvars[i]
		perc.of.zeros[[i]] <- 100*sum(all.data[[var]] == 0, na.rm=T)/sum(!is.na(all.data[[var]]))
		names(perc.of.zeros[[i]]) <- paste0("zeros_",var)
	}

	#---------------------------------
        # Check the frequency of 360 in wind direction variables
        #---------------------------------
	perc.of.360 <- list()
        for (i in seq(grep("^wdiragl",filtvars,value=T))) {
                var <- grep("^wdiragl",filtvars,value=T)[i]
                perc.of.360[[i]] <- 100*sum(all.data[[var]] == 360, na.rm=T)/sum(!is.na(all.data[[var]]))
                names(perc.of.360[[i]]) <- paste0("num360_",var)
        }

	#---------------------------------
        # Add these parameters as attributes to all.data
        #---------------------------------
	attr(all.data,"perc.of.zeros") <- perc.of.zeros
	attr(all.data,"perc.of.360") <- perc.of.360	

	return(all.data)
}

#----------------------------------------
# Name: Internal consistency check
# Aim:  Ensure that a specific convention -previously defined- has been properly applied to wind speed and wind direction time series. 
# Flags:
#  Fail: pairs of observations at a same measurement height that do not match the following condition:
#        wind speed == ws and wind direction == wd 	
#        By default: ws == 0 and wd == NA
# NOTE:  This check must be applied after defining a convention and introducing it in the wind time series.	
#----------------------------------------
internal_consistency <- function(all.data,ws=00,wd=NA) {
	
	flag_name <- "flagic_"
	
	#---------------------------------
        # Variables to be quality controlled
        #---------------------------------
        filtvars <- attr(all.data,"filtvars")
	heights  <- attr(all.data,"heights")

	#---------------------------------
        # We will apply this test if we have wind speed and wind direction values at the same height
        #---------------------------------
	filt.heights <- intersect(heights$windagl,heights$wdiragl)
		
	if (length(filt.heights) == 0) {
		print("This tower does not have any level with both wind speed and wind direction measurements")
	}

	#---------------------------------
        # filt.heights: heights in which we have both anemo and vane sensors
        #---------------------------------
	else {
		for (height in filt.heights) {
			#---------------------------------
	        	# The flag is named using wind speed variable name (windagl), though this test is applied in both
			# wind speed and wind direction
        		#---------------------------------
			var_ws <- grep(paste0("^windagl",height,"S1"),names(all.data),value=TRUE)	
			var_wd <- grep(paste0("^wdiragl",height,"S1"),names(all.data),value=TRUE)		

			all.data[[paste0(flag_name,var_ws)]] <- factor(qc_values$pass,levels=unlist(qc_values))
			all.data[[paste0(flag_name,var_ws)]][!(all.data[[var_ws]] == ws & all.data[[var_wd]] == wd)] <- qc_values$fail
			all.data[[paste0(flag_name,var_ws)]][is.na(all.data[[var_ws]]) & is.na(all.data[[var_wd]])] <- qc_values$missing 
		}
	}

	return(all.data)
}


