## Generating fitting and predicting datasets

# Load packages 
library(readr)
library(tidyr)
library(dplyr)
library(sf)

## HARMONISING DATA TO ONE SET

# Load my climate data to act as my grid to join too 
clim <- read_rds("clim_combined.rds")

# Load my other variables 
pop <- read_rds("population.rds")
land <- read_rds("copernicus_global_land_use_2019_0.25.rds")
elev <- read_rds("nasa_srtm_elev.rds")

# Ensure the grid size is compatable 
land <- land %>% # change to dataset you want 
  mutate(x_agg = floor(x * 2) / 2 + 0.25,  # Aggregate to 0.5° at steps 0.25 & 0.75
         y_agg = floor(y * 2) / 2 + 0.25) %>%
  group_by(x_agg, y_agg) %>%
  # Change to variable of the dataset 
  summarise(across(3:13, sum, na.rm = TRUE), .groups = "drop") %>% 
  # summarise(elevation = sum(elevation, na.rm = TRUE), .groups = "drop") %>%
  rename("x" = x_agg, "y" = y_agg)

# Join to my cliamte data 
all_dat <- left_join(all_dat, land)

# Check for missing data 
# If you are worried you have lost data via merging 
# You can select a year and plot to see 
all_dat %>% filter(year == 2000) %>% 
  ggplot(aes(x = x, y = y, fill = Grass)) + geom_tile()

# Transform NAs, NaNs and 0s accordingly 
all_dat <- all_dat %>% 
  mutate(across(everything(), ~ifelse(is.nan(.x) | is.na(.x), 0, .x)))

# Climate, land, population and elevation can all be merged at the grid cell 
# However, the outcomes (vectors and disease), AROPE and endemicity need to be joined at the location ID

# Load the NUTS shapefiles 
nuts <- read_sf("NUTS_RG_20M_2024_4326.shp", stringsAsFactors = F)

# Set up my data before I try to match 
nuts <- nuts %>% st_cast("MULTIPOLYGON") %>% st_cast("POLYGON")
nuts2 <- nuts %>% filter(LEVL_CODE == 2)
nuts2 <- st_transform(nuts2, crs = 4326)
length(unique(nuts2$NUTS_ID))

# Generate a grid - going for 0.25 to match to more POLYGONS
x_vals <- seq(-31.50, 62, by = 0.05)
y_vals <- seq(35, 81, by = 0.05)
grid_df <- expand.grid(x = x_vals, y = y_vals)

# Identify points which fall within the NUTS regions 
coords_sf <- st_as_sf(grid_df, coords = c("x", "y"), crs = st_crs(nuts2))
coords_within <- st_join(coords_sf, nuts2, join = st_within)
length(unique(coords_within$NUTS_ID))

# Extract the coordinates back out 
coords_within <- coords_within %>%
  st_transform(crs = 4326) %>%  
  mutate(
    x = st_coordinates(.)[, 1], 
    y = st_coordinates(.)[, 2]  
  )

# Transform back to 0.5
coords_within <- coords_within %>% 
  mutate(x = floor(x * 2) / 2 + 0.25,
         y = floor(y * 2) / 2 + 0.25) 

# Remove unwanted columns 
coords_within <- coords_within %>% 
  dplyr::select(x, y, NUTS_ID, CNTR_CODE) %>% st_drop_geometry() %>%
  distinct()

# Save for later use 
write_csv(coords_within, "latlonnuts2.csv")

# Merge to my climate data 
all_dat <- left_join(all_dat, coords_within)
all_dat <- all_dat %>% distinct()

# Load the inequity, vector and disease data 
vec <- read_rds("sandfly_occurance.rds")
dis <- read_csv("piggott_leish.csv")
arope <- read_delim("arope.csv", delim = ";", escape_double = FALSE, trim_ws = TRUE)

# The AROPE data are quite patchy, so I think it makes more sense to just select the highest for each NUTS2
arope <- arope %>% group_by(NUTS_ID) %>% filter(arope == max(arope)) %>% 
  dplyr::select(-year) %>% ungroup() 
all_dat <- left_join(all_dat, arope)

# Save the full dataset before splitting 
all_dat <- all_dat %>% distinct()
write_rds(all_dat, "harmonised_dataset.rds")

## CREATING FITTING DATASET 

# Identify the NUTS2 regions for the vector and leish data 
vec <- vec %>% 
  mutate(x = round(x / 0.25) * 0.25) %>% 
  mutate(y = round(y / 0.25) * 0.25) %>% distinct()
vec <- left_join(vec, coords_within)

dis <- dis %>% 
  mutate(x = round(x / 0.25) * 0.25) %>% 
  mutate(y = round(y / 0.25) * 0.25) %>% distinct()
dis <- left_join(dis, coords_within)
dis$adm0 <- NULL

# Select the vector species of interest 
vec <- vec %>% filter(species == "Phlebotomus perniciosus" |
                        species ==  "Phlebotomus ariasi" |
                        species == "Phlebotomus perfiliewi" |
                        species == "Phlebotomus neglectus" |
                        species == "Phlebotomus tobbi")

# Filter repeated values for each year and NUTS2 
vec <- vec %>% dplyr::select(NUTS_ID, CNTR_CODE, year, species, occurance) %>% distinct()
dis <- dis %>% dplyr::select(NUTS_ID, CNTR_CODE, year, cl, vl) %>% distinct() 
dis <- na.omit(dis)
vec <- na.omit(vec)
vec$year <- as.numeric(vec$year)

## FOR THE FLIES 

# Identify the years to subset for 
unique(vec$year)

# For the species of interest, only 2023 has data 
# Many of these species are suggested to have been in Europe for a long time though
# So I am going to take the most recent 10 years, assign the same value for presence 
# And use that for prediction 

# Have a column for each species and one joint, so that I can have the option to fit a model per species and one for 
# Phlebotomus carrying leishmania species 
vec <- vec %>% 
  mutate(species = stringr::str_replace(species, "Phlebotomus ", "")) %>% 
  pivot_wider(names_from = species, values_from = occurance, values_fill = list(occurance = 0)) %>% 
  dplyr::select(-year)

# Filter to last 10 years of data 
fit_flies <- all_dat %>% filter(year > 2014)

# Merge to fly data 
fit_flies <- left_join(fit_flies, vec)
fit_flies <- fit_flies %>%
  filter(!is.na(NUTS_ID))

# Assign zeros to locations without fly occurrence 
fit_flies <- fit_flies %>% 
  mutate(across(everything(), ~ifelse(is.na(.x), 0, .x)))

# Save in the model fitting folder 
write_rds(fit_flies, "fitting_dataset_flies.rds")

## FOR THE DISEASE 

# Identify the years to subset for 
unique(dis$year)
# There is data for 1972-2012

# Filter to years with disease data 
fit_disease <- all_dat %>% filter(year > 1971 & year < 2013)

# Merge to disease data 
fit_disease <- left_join(fit_disease, dis)
fit_disease <- fit_disease %>%
  filter(!is.na(NUTS_ID))

# Assign zeros to locations without disease occurrence 
fit_disease <- fit_disease %>% 
  mutate(across(everything(), ~ifelse(is.na(.x), 0, .x))) %>% distinct()

# Save in the model fitting folder 
write_rds(fit_disease, "fitting_dataset_disease.rds")

## CREATING PREDICTING DATASET 

# Remove the outcomes from the harmonised dataset 
all_dat <- all_dat %>% select(-cl, -vl, -neglectus, -ariasi, -perniciosus, -perfiliewi, -tobbi)

# Save in the model predicting folder 
write_rds(all_dat, "predicting_dataset.rds")



