####################################################################################
#                                                                                  #   
# Open and crop Traffic data index to Catalonia and AMB domain                     #
#                                                                                  #
####################################################################################


library(data.table)
library(sf)
library(purrr)

wdir <- "/esarchive/scratch/ajulian/datasets_MPCS/carto/unica_traffic/2024/" 
wout <- "/esarchive/scratch/ajulian/scripts/Open_Data/Traffic_buffers/Results/"
wout2 <- "/esarchive/scratch/ajulian/scripts/Open_Data/Traffic_buffers/Results/temporal/"


# Load domain shapefiles for cropping
def_domini_AMB <- readRDS(file.path(wout, "def_domini_AMB.RDS"))
def_domini_CAT <- readRDS(file.path(wout, "def_domini_CAT.RDS"))

def_domini_AMB <- st_transform(def_domini_AMB, crs = 4326)
def_domini_CAT <- st_transform(def_domini_CAT, crs = 4326)

# Function to process and crop the file by domain
process_file <- function(file_path, wout, def_domini_AMB, def_domini_CAT) {
  # Ensure the output directory exists
  if (!dir.exists(wout)) {
    dir.create(wout, recursive = TRUE)  
  }
  
  # Use fread to load large CSVs in a memory-efficient way
  df <- fread(file_path)
  
  # Convert the 'geom' column to an sf object directly
  sf_df <- st_as_sf(df, wkt = "geom", crs = 4326) 
  
  # Check if the sf_df has valid geometries
  if (nrow(sf_df) > 0) {
    # Ensure the CRS of the file matches the CRS of the domain boundaries
    sf_df <- st_transform(sf_df, crs = 4326)
    
    # Crop by AMB domain
    sf_AMB <- st_intersection(sf_df, def_domini_AMB)
    saveRDS(sf_AMB, file.path(wout, paste0(basename(file_path), "_AMB_processed.rds")))
    
    # Crop by Catalonia domain
    sf_CAT <- st_intersection(sf_df, def_domini_CAT)
    saveRDS(sf_CAT, file.path(wout, paste0(basename(file_path), "_CAT_processed.rds")))
  } else {
    message("No valid geometries found in file:", file_path)
  }

  
  rm(df, sf_df, sf_AMB, sf_CAT)
  gc()  
}

# Loop through the files and process them one by one
for (i in 0:99) {
  file_suffix <- sprintf("%02d.csv", i)
  file_path <- file.path(wdir, paste0("Road Traffic Index - Spain (Road Segment)-0000000000", file_suffix))
  
  if (file.exists(file_path)) {
    process_file(file_path, wout2, def_domini_AMB, def_domini_CAT)
  } else {
    message(paste("File not found:", file_path))
  }
}

processed_files_AMB <- list.files(wout2, pattern = "_AMB_processed.rds$", full.names = TRUE)
combined_sf_df_AMB <- map_dfr(processed_files_AMB, readRDS)
saveRDS(combined_sf_df_AMB, file.path(wout, "traffic_ind_AMB_2024.RDS"))

processed_files_CAT <- list.files(wout2, pattern = "_CAT_processed.rds$", full.names = TRUE)
combined_sf_df_CAT <- map_dfr(processed_files_CAT, readRDS)
saveRDS(combined_sf_df_CAT, file.path(wout, "traffic_ind_CAT_2024.RDS"))