library(ncdf4)
library(ggplot2)


load("NorthAfrica_em.RData")
load("NorthAfrica_od.RData")

load("NorthAfrica_lat.RData")
load("NorthAfrica_lon.RData")


load("NorthAfrica_ustar.RData")
load("NorthAfrica_ustt3.RData")
load("NorthAfrica_ustarsd.RData")
load("intercept_wind.RData")
load("windthrreal.RData")
load("windthr2.RData")
load("ustt.RData")

windthr_robust_matlab = read.csv("wthreshold_robust.csv", header=FALSE) 
c_robust_matlab = read.csv("c_robust.csv", header=FALSE) 

windthr_robust_matlab <- as.numeric(windthr_robust_matlab)
c_robust_matlab <- as.numeric(c_robust_matlab)


wind_steps <- seq(from = 0, to = 1.5, by = 0.025)

vectorem <-(NorthAfrica_em[seq(2684,2924,by=8),])
vectorod <- (NorthAfrica_od[seq(2684,2924,by=8),])
vectorwind <- (NorthAfrica_ustar[seq(2684,2924,by=8),])

vectorod <- rbind(vectorod,((NorthAfrica_od[seq(4,476,by=8),])))
vectorem <- rbind(vectorem,((NorthAfrica_em[seq(4,476,by=8),])))
vectorwind <- rbind(vectorwind,((NorthAfrica_ustar[seq(4,476,by=8),])))

vectorwind2 <- ((NorthAfrica_ustt3[seq(4,2924,by=8),]))

vectorod <- vectorod*0.31
vectorem <- vectorem*0.31

NorthAfrica_od0<- vectorod[(vectorwind>=0.5)]
NorthAfrica_em0<- vectorem[(vectorwind>=0.5)]
linearMod <- lm(as.vector(NorthAfrica_od0) ~ as.vector(NorthAfrica_em0))
summary(linearMod)
intercept <- summary(linearMod)$coefficients[1,1]
print("intercept")
print(intercept)
cor(NorthAfrica_od0,NorthAfrica_em0)


vectorod <- vectorod-intercept
enes <- c(0,0,0)
npoints <- vector(length=length(vectorod[1,]))
head(npoints)
for (d in 1:91)
{
  for (n in 1:length(npoints))
  {
    if ((vectorwind[d,n]>0.5)&(vectorod[d,n]>=0))
    {
      npoints[n] <- npoints[n] + 1 
    }
    else
    {
      vectorod[d,n] <- NA
      vectorem[d,n] <- NA
      vectorwind[d,n] <- NA
    }
  }
}
head(npoints)

counter <-0
lol <-0
lol2 <- 0
roots <- replicate(length(npoints), NaN)

for (n in 1:length(npoints))
{
  print("n")
  print(n)
  # print(npoints[n])
  y=vectorod[,n]
  x=vectorwind[,n]
  z=vectorem[,n]
  y = y[!is.na(y)]
  x = x[!is.na(x)]
  z = z[!is.na(z)]
  # print(length(y))
  dat <- as.data.frame(cbind(x,y))
  names(dat) <- c("x", "y")
  if (npoints[n]>3)
  {
    min.OLS <- function(data, par) {
      with(data, sum((   -par[1]*(par[2]^(3))     -
                           par[1]*(par[2]^(2)) * x + 
                           par[1]*par[2] * (x^2) +
                           par[1] * (x^3) +
                           - y )^2))}
    
    # set upper & lower bound for your regression
    result.opt <- optim(par = c(5,5),
                        min.OLS, 
                        data = dat, 
                        lower = c(0.00000000001,0.00000000001),
                        upper=c(10,Inf),
                        method="L-BFGS-B"
    )
    
    predict.yy <- function(data, par) {
      print(with(data, ((
        -par[1]*(par[2]^(3))     -
          par[1]*(par[2]^(2)) * x + 
          par[1]*par[2] * (x^2) +
          par[1] * (x^3)
      ))))
    }
    
    yy<-predict.yy(dat, result.opt$par)
    
    # print(result.opt$par[1])
    # print(result.opt)
    
    roots[n] <- result.opt$par[2]
    
    t <- (-result.opt$par[1]*(result.opt$par[1])^(2))/(sum((yy-y)^2)/(sum((x-mean(x))^2)*(npoints[n]-2)))^(1/2)
    #  print("t")
    # print(t)
    # print(npoints[n])
    # print(qt(p = 0.95 + 0.05/2, df = (npoints[n]-2), lower.tail = TRUE))
    # print(qt(p =  0.05/2, df = (npoints[n]-2), lower.tail = TRUE))
    tdist <- qt(p = 0.95 + 0.05/2, df = (npoints[n]-2), lower.tail = TRUE)
    
    # if (abs(t)<tdist)
    # {
    #   roots[n] <- NaN
    #   lol <- lol +1
    #  
    # }
    # else
    {
      
      lol2 <- lol2 +1
      plot(x,y,xlim=c(0,1.5), ylim=c(0, 5) )
      lines(x,yy,col="red",lwd=3)
      
      
      if (n==961)
      {
        xwind <- seq(0,4,by=0.05)
        ydod <- vector(length=length(xwind))
        for (s in 1:length(ydod))
        {
          ydod[s] <- -c_robust_matlab[n]*(windthr_robust_matlab[n]^(3))-c_robust_matlab[n]*(windthr_robust_matlab[n]^(2)) * xwind[s] +c_robust_matlab[n]*windthr_robust_matlab[n] * (xwind[s]^2) +c_robust_matlab[n] * (xwind[s]^3) 
        }
        
        enes[1] <- n
        print("holasfhkeskjdfnsd")
        print(npoints[n])
        datos2 <- data.frame(x,y,z)
        x2  <- c(roots[n],x)
        yy2 <- c(0,yy)
        datos3 <- data.frame(xwind,ydod)
        xem <- x[z>0]
        yem <- y[z>0]
        dataa <- data.frame(xem,yem)
        # png("scatterplot_fitthreshold27.png")
        # myplot1 <- ggplot(datos2) +
        #   geom_point(aes((x), (y)), size = 1) +
        #   geom_line(data=datos3, aes(x2, yy2),fullrange = TRUE, colour = "red", lty=2, lwd=1) +
        #   #   #scale_x_continuous(limits=c(0,0.01))+ scale_y_continuous(limits=c(0,3))+
        #   #   # scale_y_log10(limits = c(0.0001,10)) +  scale_x_log10(limits = c(5*10^(-4),0.1)) +
        #   #   #scale_y_log10(limits = c(0.0003,30)) +  scale_x_log10(limits = c(10^(-11),0.2)) +
        #   #   #scale_x_log10() + scale_y_log10() +
        #   #   theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
        #   #         panel.background = element_blank(), axis.line = element_line(colour = "black"), text = element_text(size=14)) +
        #   labs(x = "u* [m/s]", y= "Dust Optical Depth-Intercept") + theme(legend.position="top") +
        #   ggtitle("")  
        # print(myplot1)
        # dev.off()
      }
      
      if (n==241)
      {
        xwind <- seq(0,4,by=0.05)
        ydod <- vector(length=length(xwind))
        for (s in 1:length(ydod))
        {
          ydod[s] <- -c_robust_matlab[n]*(windthr_robust_matlab[n]^(3))-c_robust_matlab[n]*(windthr_robust_matlab[n]^(2)) * xwind[s] +c_robust_matlab[n]*windthr_robust_matlab[n] * (xwind[s]^2) +c_robust_matlab[n] * (xwind[s]^3) 
        }
        
        enes[2] <- n
        print("holasfhkeskjdfnsd")
        print(npoints[n])
        datos4 <- data.frame(x,y,z)
        x2  <- c(roots[n],x)
        yy2 <- c(0,yy)
        datos5 <- data.frame(xwind,ydod)
        xem <- x[z>0]
        yem <- y[z>0]
        datab <- data.frame(xem,yem)
        # png("scatterplot_fitthreshold155.png")
        # myplot1 <- ggplot(datos4) +
        #   geom_point(aes((x), (y)), size = 1) +
        #   geom_line(data=datos5, aes(x2, yy2),fullrange = TRUE, colour = "red", lty=2, lwd=1) +
        #   #   #scale_x_continuous(limits=c(0,0.01))+ scale_y_continuous(limits=c(0,3))+
        #   #   # scale_y_log10(limits = c(0.0001,10)) +  scale_x_log10(limits = c(5*10^(-4),0.1)) +
        #   #   #scale_y_log10(limits = c(0.0003,30)) +  scale_x_log10(limits = c(10^(-11),0.2)) +
        #   #   #scale_x_log10() + scale_y_log10() +
        #   #   theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
        #   #         panel.background = element_blank(), axis.line = element_line(colour = "black"), text = element_text(size=14)) +
        #   labs(x = "u* [m/s]", y= "Dust Optical Depth-Intercept") + theme(legend.position="top") +
        #   ggtitle("")  
        # print(myplot1)
        # dev.off()
      }
      
      if (n==296)
      {
        
        xwind <- seq(0,4,by=0.05)
        ydod <- vector(length=length(xwind))
        for (s in 1:length(ydod))
        {
          ydod[s] <- -c_robust_matlab[n]*(windthr_robust_matlab[n]^(3))-c_robust_matlab[n]*(windthr_robust_matlab[n]^(2)) * xwind[s] +c_robust_matlab[n]*windthr_robust_matlab[n] * (xwind[s]^2) +c_robust_matlab[n] * (xwind[s]^3) 
        }
        
        
        enes[3] <- n
        print("holasfhkeskjdfnsd")
        print(npoints[n])
        datos6 <- data.frame(x,y,z)
        x2  <- c(roots[n],x)
        yy2 <- c(0,yy)
        datos7 <- data.frame(xwind,ydod)
        xem <- x[z>0]
        yem <- y[z>0]
        datac <- data.frame(xem,yem)
        # png("scatterplot_fitthreshold108.png")
        # myplot1 <- ggplot(datos6) +
        #   geom_point(aes((x), (y)), size = 1) +
        #   geom_line(data=datos7, aes(x2, yy2),fullrange = TRUE, colour = "red", lty=2, lwd=1) +
        #   #   #scale_x_continuous(limits=c(0,0.01))+ scale_y_continuous(limits=c(0,3))+
        #   #   # scale_y_log10(limits = c(0.0001,10)) +  scale_x_log10(limits = c(5*10^(-4),0.1)) +
        #   #   #scale_y_log10(limits = c(0.0003,30)) +  scale_x_log10(limits = c(10^(-11),0.2)) +
        #   #   #scale_x_log10() + scale_y_log10() +
        #   #   theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
        #   #         panel.background = element_blank(), axis.line = element_line(colour = "black"), text = element_text(size=14)) +
        #   labs(x = "u* [m/s]", y= "Dust Optical Depth-Intercept") + theme(legend.position="top") +
        #   ggtitle("")  
        # print(myplot1)
        # dev.off()
      }
      
      
      
    }
    
    
  }
  else
  {
    counter <- counter +1 
  }
  #print(summary(fit3b))
}
counter
lol
lol2

vectorwindthr <- replicate(length(npoints), NaN)
for (s in 1:length(npoints))
{
  if (!is.na(roots[s]))
  {
    vectorwindthr[s] <- mean(vectorwind2[,s])
  }
  else
  {
    windthrreal[s] <- NaN
    windthr2[s] <- NaN
  }
}

#-----------------------------------------------------------------------


xwind <- seq(0,4,by=0.05)
ydod <- vector(length=length(xwind))
for (s in 1:length(ydod))
{
  ydod[s] <- -c_robust_matlab[s]*(windthr_robust_matlab[s]^(3))-c_robust_matlab[s]*(windthr_robust_matlab[s]^(2)) * xwind[s] +c_robust_matlab[s]*windthr_robust_matlab[s] * (xwind[s]^2) +c_robust_matlab[s] * (xwind[s]^3) 
}

#--------------------------------------------------------------------------
print("enes")
lats <- c(NorthAfrica_lat[1,enes[1]],NorthAfrica_lat[1,enes[2]],NorthAfrica_lat[1,enes[3]])
lons <- c(NorthAfrica_lon[1,enes[1]],NorthAfrica_lon[1,enes[2]],NorthAfrica_lon[1,enes[3]])
print(lats)
print(lons)
NorthAfrica_lat[1,]
NorthAfrica_lon[1,]
dim(NorthAfrica_lat)
ys <- c(0,0,0)
xs <- c( vectorwindthr[enes[1]], vectorwindthr[enes[2]], vectorwindthr[enes[3]])
xs
datos8 <- data.frame(xs, ys )
datac
print("hola elisa")
png("scatterplot_fitts_05_sd.png")
myplot1 <- ggplot(datos2) +
  geom_point(aes((x), (y),colour = "[20N, 38.1E]"), size = 3, shape=22) +
  #geom_point(data=datos8, aes(xs[1], ys[1],colour = "[20N, 38.1E]"),fullrange = TRUE, size = 3, shape =15) +
  geom_point(data=dataa, aes(xem, yem, colour = "[20N, 38.1E]"),fullrange = TRUE,size = 3, shape =15) +
  geom_point(data=datos4, aes(x, y, colour = "[24N, 2.8W]"),fullrange = TRUE,size = 3, shape =23) +
  geom_point(data=datab, aes(xem, yem, colour = "[24N, 2.8W]"),fullrange = TRUE,size = 4, shape =18) +
#  geom_point(data=datos8, aes(xs[2], ys[2], colour = "[24N, 2.8W]"),fullrange = TRUE, size = 3, shape =18) +
  geom_point(data=datos6, aes(x, y, colour = "[15N, 0W]"),fullrange = TRUE, size = 3, shape=24) +
 # geom_point(data=datac, aes(xem, yem, colour = "[15N, 0W]"),fullrange = TRUE,size = 3, shape =17) +
  #geom_point(data=datos8, aes(xs[3], ys[3], colour = "[15N, 0W]"),fullrange = TRUE,size = 3, shape =17) +
  geom_line(data=datos3, aes(xwind, ydod, colour = "[20N, 38.1E]"),fullrange = TRUE, lty=2, lwd=1) +
    geom_line(data=datos5, aes(xwind, ydod, colour = "[24N, 2.8W]"),fullrange = TRUE, lty=2, lwd=1) +
   geom_line(data=datos7, aes(xwind, ydod, colour = "[15N, 0W]"),fullrange = TRUE, lty=2, lwd=1) +
   geom_hline(yintercept=0, size=0.2) +
  scale_x_continuous(limits=c(0.2,1))+
  scale_y_continuous(limits=c(-0.1,1))+
  #   # scale_y_log10(limits = c(0.0001,10)) +  scale_x_log10(limits = c(5*10^(-4),0.1)) +
  #   #scale_y_log10(limits = c(0.0003,30)) +  scale_x_log10(limits = c(10^(-11),0.2)) +
  #   #scale_x_log10() + scale_y_log10() +
  scale_colour_manual("", 
                      breaks = c("[20N, 38.1E]", "[24N, 2.8W]", "[15N, 0W]"),
                      values = c("#56B4E9", "#E69F00", "#009E73", "#56B4E9")) +
  theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
        panel.background = element_blank(), axis.line = element_line(colour = "black"), text = element_text(size=14)) +
  labs(x = "Wind Friction Velocity [m/s]", y= "Dust Optical Depth-Intercept") + theme(legend.position="bottom") +

  ggtitle("")  +
  theme_bw() + theme(legend.position="bottom") +theme(axis.text.x=element_text(size=14, angle=0, vjust=0.3),
                     axis.text.y=element_text(size=14),
                     axis.title=element_text(size=15),
                     legend.text=element_text(size=15))
print(myplot1)
dev.off()



#-------------------------------------------------------------------------
bias <- sum(vectorwindthr[!is.na(vectorwindthr)])/sum(roots[!is.na(roots)])
print("BIAS")
print(bias)

print("RMSE")
RMSE <- sqrt(sum((vectorwindthr[!is.na(vectorwindthr)]-roots[!is.na(roots)])^(2))/lol2)
print(RMSE/mean(vectorwindthr[!is.na(vectorwindthr)]))

print("cor")
print(cor(vectorwindthr[!is.na(vectorwindthr)],roots[!is.na(roots)]))


bias <- sum(windthrreal[!is.na(windthrreal)])/sum(windthr2[!is.na(windthr2)])
print("BIAS")
print(bias)

print("RMSE")
RMSE <- sqrt(sum((windthrreal[!is.na(windthrreal)]-windthr2[!is.na(windthr2)])^(2))/lol2)
print(RMSE/(mean(windthrreal[!is.na(windthrreal)])))

print("cor")
print(cor(windthrreal[!is.na(windthrreal)],windthr2[!is.na(windthr2)]))
#___________________________________________________________________________________

relative_error <- vector(length=length(ustt))
for (s in 1:length(ustt))
{
  if (is.na(ustt[s]))
  {
    relative_error[s] <- NaN
  }
  else if (ustt[s]==0)
  {
    relative_error[s] <- NaN
  }
  else
  {
    relative_error[s] <- 100*(windthr_robust_matlab[s]-ustt[s])/ustt[s]
  }
  
}

#_________________________________________________________________________________

#file for mask the sea
ncpathsm <- "/esarchive/exp/nmmb-bsc-ctm/b051/"
ncnamesm <- "seamask"  
ncseamask <- paste(ncpathsm, ncnamesm, ".nc", sep="")
nsm <- nc_open(ncseamask)
# get seamask
seamask_array <- ncvar_get(nsm,"seamask")
fillvalue <- ncatt_get(nsm,"seamask","_FillValue")
dim(seamask_array)
lat <- length(seamask_array[1,])
lon <- length(seamask_array[,1])


seamask_array_NorthAfrica <- matrix(0,47,41)
lon <- 47
lat <- 41
for (l in 1:lat)
{
  for (n in 1:lon)
  {
    seamask_array_NorthAfrica[n,l] <- seamask_array[107+n,89+l]
  }
}







npoints_matrix <- matrix(0,lon,lat)

roots_matrix <- matrix(0,lon,lat)

windthr <- matrix(0,lon,lat)

wthr_matlab <- matrix(0,lon,lat)
wthr_robust_matlab <- matrix(0,lon,lat)

relative_error_matrix <- matrix(0,lon,lat)


k <- 1
for (n in 1:lon)
{
  for (l in 1:lat)
  {
    if(seamask_array_NorthAfrica[n,l]==1)
    {
      
      npoints_matrix[n,l] <- NaN
      roots_matrix[n,l] <- NaN
      windthr[n,l] <- NaN
      wthr_matlab[n,l] <- NaN
      wthr_robust_matlab[n,l] <- NaN
      relative_error_matrix[n,l] <- NaN
    }
    else
    {
      
      npoints_matrix[n,l] <- npoints[k]
      roots_matrix[n,l] <- roots[k]
      windthr[n,l] <- vectorwindthr[k]
      wthr_matlab[n,l] <- windthr_matlab[k]
      wthr_robust_matlab[n,l] <- windthr_robust_matlab[k]
      relative_error_matrix[n,l] <- relative_error[k]
      if (k==enes[1])
      {
        print("1,iejhfiodjgjigwre")
        print(enes[1])
        print(n)
        print(l)
      }
      if (k==enes[2])
      {
        print("2,iejhfiodjgjigwre")
        print(enes[1])
        print(n)
        print(l)
      }
      if (k==enes[3])
      {
        print("3,iejhfiodjgjigwre")
        print(enes[1])
        print(n)
        print(l)
      }
      if(n==42 & l==22)
      {
        print("k1")
        print(k)
      }
      if(n==20 & l==26)
      {
        print("k2")
        print(k)
      }
      if(n==22 & l==18)
      {
        print("k3")
        print(k)
      }
      k <- k +1
    }
  }
}


#save(FoO_matrix, file = "FoO_matrix.RData")
#----------------------------------------------------------------------------





#-------------------------------------------------------------------------

library(ncdf4)

# path and file name, set dname
ncpath <- "/Documents/sct_plots/"
ncname <- "2012means"  
ncfname <- paste(ncpath, ncname, ".nc", sep="")
dname <- "correlation"  # note: tmp means temperature (not temporary)


lon3 <- seq(-29.53, 35.16, by=(360/256))
lat3 <- seq(-1, 39, by=(1))
print("lat")
lat3[18]
lat3[(26)]
lat3[(22)]
print("lon")
lon3[(22)]
lon3[(20)]
lon3[(42)]
lon3
length(lon3)
head(lat3)
length(lat3)
time3 <- c(2012)
length(time3)
# create and write the netCDF file -- ncdf4 version
# define dimensions
londim <- ncdim_def("lon","degrees_east",as.double(lon3)) 
latdim <- ncdim_def("lat","degrees_north",as.double(lat3)) 
timedim <- ncdim_def("time","year",as.double(time3))

# define variables
fillvalue <- 1e32
dlname <- "npoints"
mod_def <- ncvar_def("npoints","",list(londim,latdim,timedim),fillvalue,dlname,prec="single")
dlname2 <- "roots"
mod_def2 <- ncvar_def("roots","",list(londim,latdim,timedim),fillvalue,dlname2,prec="single")
dlname3 <- "windthr"
mod_def3 <- ncvar_def("windthr","",list(londim,latdim,timedim),fillvalue,dlname3,prec="single")
dlname4 <- "windthrmatlab"
mod_def4 <- ncvar_def("windthrmatlab","",list(londim,latdim,timedim),fillvalue,dlname4,prec="single")
dlname5 <- "windthrmatlab_robust"
mod_def5 <- ncvar_def("windthrmatlab_robust","",list(londim,latdim,timedim),fillvalue,dlname5,prec="single")
dlname6 <- "relative_error"
mod_def6 <- ncvar_def("relative_error","",list(londim,latdim,timedim),fillvalue,dlname6,prec="single")
# # 


# create netCDF file and put arrays
ncfname <- "npoints_05_sd.nc"
ncout <- nc_create(ncfname,list(mod_def,mod_def2,mod_def3,mod_def4,mod_def5,mod_def6), force_v4=TRUE)
#ncout <- nc_create(ncfname,list(mod_def,mod_def2,mod_def3), force_v4=TRUE)

#load("seamask_array_NorthAfrica.RData")


# load("biasod_wind.RData")
# load("meanem_wind.RData")
dim(seamask_array_NorthAfrica)
x <- array(0, dim=c(2,47,41))
x[2,,] <- seamask_array_NorthAfrica
# put variables
ncvar_put(ncout,mod_def,npoints_matrix)
ncvar_put(ncout,mod_def2,roots_matrix)
ncvar_put(ncout,mod_def3,windthr)
ncvar_put(ncout,mod_def4,wthr_matlab)
ncvar_put(ncout,mod_def5,wthr_robust_matlab)
ncvar_put(ncout,mod_def6,relative_error_matrix)
# ncvar_put(ncout,mod_def3,FoO_em_matrix)
# ncvar_put(ncout,mod_def4,windthr2_matrix)
# y <- array(0, dim=c(2,46,42))
# y[2,,] <- biasod_wind
# # put variables
# ncvar_put(ncout,mod_def2,biasod_wind)
# 
# z <- array(0, dim=c(2,46,42))
# z[2,,] <- meanem_wind
# # put variables
# ncvar_put(ncout,mod_def3,meanem_wind)


# put additional attributes into dimension and data variables
ncatt_put(ncout,"lon","axis","X") #,verbose=FALSE) #,definemode=FALSE)
ncatt_put(ncout,"lat","axis","Y")
ncatt_put(ncout,"time","axis","T")

# add global attributes
ncatt_put(ncout,0,"title","2012-Mean od550ducoarse and emidu")
ncatt_put(ncout,0,"institution","Barcelona Supercomputing Center")
ncatt_put(ncout,0,"source","nmmb")

# Get a summary of the created file:
ncout

# close the file, writing data to disk
nc_close(ncout)
