#!/bin/bash
#SBATCH -n 1
#SBATCH -t 01:00:00
#SBATCH -J gfasftp
#SBATCH -e /esarchive/recon/ecmwf/gfas/scripts/i535-marc-hourly/out-logs/format-ftp-%j.err
#SBATCH -o /esarchive/recon/ecmwf/gfas/scripts/i535-marc-hourly/out-logs/format-ftp-%j.out

#module load xarray/0.19.0-foss-2019b-Python-3.7.4
#module load netcdf4-python/1.5.8-foss-2019b-Python-3.7.4
#module load dask/2022.2.0-foss-2019b-Python-3.7.4
#module load numpy/1.21.6-foss-2019b-Python-3.7.4

module load requests
module load pandas
module load beautifulsoup4 # dealing with html
module load selenium  # scraping from java script
module load chromedriver-py/107.0.5304.62-foss-2021b-Python-3.9.6


time python3 /esarchive/scratch/rgrodofz/software/ghost/GHOST/download_scripts/download_networks.py --mode=all