## Process the sandfly data 

# https://geoportal.ecdc.europa.eu/geocatalogue/?session=4uoj7pdbovo6&searchTerm=&isFullPhrase=false&pageIndex=1&pageSize=10&geoItemType=All&sortType=published_desc&filterOptions%5BHealth%5D=&filterOptions%5BEsri%5D=fc%7CVector&isUrlSearch=true

# Load packages 
library(purrr)
library(dplyr)
library(sf)
library(ggplot2)
library(readr)

## VECTORNET DATA 

# List file names 
paths <- list.files()

# Check the layer names
layers <- st_layers(paths[3])
print(layers)

# List all shapefiles in the folder
shp23 <- st_read(dsn = paths[1], layer = "Output")
shp24 <- st_read(dsn = paths[2], layer = "VectorNet_Sandflies")
shp22 <- st_read(dsn = paths[3], layer = "Output")

# Filter to sandflies for the 2022 and 23 data  
shp23 <- shp23 %>% filter(grepl("Phlebotomus", VectorSpeciesName))
shp22 <- shp22[-c(3:23)]

# The VectorNet distributions are coded
# NAT001A = Present
# NAT002A = Antic.Absent
# NAT003A = Obs.Absent
# NAT004A = No data 
# Filter to the areas where the vector is present 
shp23 <- shp23 %>% filter(AssessedDistributionStatus == "NAT001A")

shp22 <- shp22 %>% tidyr::gather("species", "code", 3:12)
shp24 <- shp24 %>% tidyr::gather("species", "code", 4:13)

shp22 <- shp22 %>% filter(code == "NAT001A")
shp24 <- shp24 %>% filter(code == "NAT001A")

# Extract the coordinates 
flies <- list(shp22, shp23, shp24)

flies <- flies %>% map(~ {
  points <- st_cast(.x, "POINT") 
  points$x <- st_coordinates(points)[,1]  
  points$y <- st_coordinates(points)[,2]  
  species_col <- intersect(c("species", "VectorSpeciesName"), names(points))
  points %>% select(x, y, all_of(species_col))
})

# Make column names consistent for merging
colnames(flies[[2]])[3] <- "species"

# Name elements after the year in order to later use as the ID
names(flies) <- c(2022, 2023, 2024)

# Bind and tidy 
flies <- bind_rows(flies, .id = "year")
rownames(flies)<-NULL
flies <- flies %>% st_drop_geometry()  
flies$occurance <- 1
flies <- flies %>% distinct()
flies <- flies %>%
  tidyr::pivot_wider(names_from = species, values_from = occurance, values_fill = list(occurance = 0))

readr::write_rds(flies, "sandfly_occurance.rds")

## GBIF DATA 
# https://www.gbif.org/dataset/193638ea-0d3c-4c23-865c-df53941174b8

# Read EXCEL
gbif <- readxl::read_excel("gbif.xlsx")

# Format correctly 
colnames(gbif) <- gbif[1, ]
gbif <- gbif[-c(1),]

gbif <- tibble(x = gbif$decimalLongitude,
               y = gbif$decimalLatitude,
               year = gbif$year,
               species = gbif$species)

gbif <- gbif[apply(gbif, 1, function(row) any(grepl("Phlebotomus", row))), ]

gbif <- gbif %>% mutate(across(1:3, as.numeric))

# The GBIF data does not have enough Europe specific species to include 










