From eac99eb9d94491b95e748378c2067f072e5ea56e Mon Sep 17 00:00:00 2001 From: Elliott Rose Date: Fri, 19 May 2023 17:36:16 +0200 Subject: [PATCH] Fix issue with gifs not generating. Update pngs to remove the bottom navbar from showing in png. Unify js loop scripts and shell loop scripts. --- assets/custom-functions.js | 2 +- bin/gen_country_loop.sh | 76 +++--- bin/gen_models_maps.sh | 91 ------- conf/coords.json | 22 +- js/create_country_loop.js | 221 ----------------- js/create_model_loop.js | 207 ---------------- js/create_model_loop_zoom_fit.js | 405 ++++++++++++++++--------------- 7 files changed, 263 insertions(+), 761 deletions(-) delete mode 100755 bin/gen_models_maps.sh delete mode 100644 js/create_country_loop.js delete mode 100644 js/create_model_loop.js diff --git a/assets/custom-functions.js b/assets/custom-functions.js index 76bec1e..f7fa9f3 100644 --- a/assets/custom-functions.js +++ b/assets/custom-functions.js @@ -77,7 +77,7 @@ $(document).ready(function () { // Add logos for the animated gifs $(document).ready(function () { - $(document).on('click', "#slider-graph", function () { + $(document).on('click', "#model-slider-graph", function () { const logos = document.getElementById('logos'); if(!logos) { var img = new Image(); diff --git a/bin/gen_country_loop.sh b/bin/gen_country_loop.sh index 1bf94a7..a711c62 100755 --- a/bin/gen_country_loop.sh +++ b/bin/gen_country_loop.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash ########################################## -# country should be written with first letter as capital, or how appears in coords.conf +# country should be written lowercase, or how appears in coords.conf # date currently does not function and will return always today's date (or most recent available) # only monarch accepts all variables, the rest get od550_dust and sconc_dust # anim should only be called with true here, as the pngs are automatically deleted # ./bin/gen_country_loop.sh monarch true sconc_dust 20220808 spain +# for development change repodir and currdir # generate daily maps for specified couuntry # usage: script.sh [model] [anim] [variable] [date] [country] @@ -37,13 +38,38 @@ fi if [ "$5" != "" ]; then country="$5" else - country="spain" + country="default" fi curyear=`date '+%Y' -d ${curdate}` curmon=`date '+%m' -d ${curdate}` repodir='/data/daily_dashboard/comparison/' +runProcess () { + tmpdir=${HOME}/tmp/${var}/${country} + mkdir -p $tmpdir + rm -rf $tmpdir/* + # SET REPO AND FILENAME FOR NON-COUNTRY SELECTIONS + currepo=${repodir}/${model}/${variable}/${curyear}/${curmon} + filename=${curdate}_${model} + # SET FOLDER STRUCTURE AND FILENAME FOR SPECIFIC COUNTRIES + if [ "$country" != "default" ] && [ "$country" != "monarch_fit" ]; then + tmpdir=${HOME}/tmp/${var}/${country} + mkdir -p $tmpdir + rm -rf $tmpdir/* + currepo=${repodir}/${model}/${variable}/${curyear}/${curmon}/${country} + filename=${curdate}_${model}_${country} + fi + node js/create_model_loop_zoom_fit.js $anim $model $curdate $variable $country + wait + sleep 1 + convert -loop 0 -delay 25 ${tmpdir}/${filename}_??.png ${tmpdir}/${filename}_loop.gif + mkdir -p $currepo + wait + mv ${tmpdir}/${filename}_* $currepo + rm $currepo/${filename}_??.png +} + for var in od550_dust sconc_dust dust_depd dust_depw dust_load dust_ext_sfc do variable=$var @@ -51,46 +77,26 @@ do if [ "$curvar" != "$variable" ] && [ "$curvar" != "all" ]; then continue fi - tmpdir=${HOME}/tmp/${var}/${country} - mkdir -p $tmpdir - rm -rf $tmpdir/* + # IF VARIABLE ISN'T AOD OR CONCENTRATION, FORCE MONARCH if [ "$variable" != "od550_dust" ] && [ "$variable" != "sconc_dust" ]; then model="monarch" - node js/create_country_loop.js $anim $model $curdate $variable $country - wait - sleep 1 - convert -loop 0 -delay 25 ${tmpdir}/${curdate}_${model}_${country}_??.png ${tmpdir}/${curdate}_${model}_${country}_loop.gif - currepo=${repodir}/${model}/${variable}/${curyear}/${curmon}/${country} - mkdir -p $currepo - wait - mv ${tmpdir}/${curdate}_${model}_${country}_* $currepo - rm $currepo/${curdate}_${model}_${country}_??.png - + runProcess + # IF MODEL INPUT IS ALL, RUN THROUGH ALL MODELS elif [ "$model" == "all" ]; then - for mod in `cat interactive-forecast-viewer/conf/models.json | grep '": {' | sed 's/^.*"\(.*\)".*$/\1/g'` + selectedCountry=$country + for model in `cat ./conf/models.json | grep '": {' | sed 's/^.*"\(.*\)".*$/\1/g'` do - node js/create_country_loop.js $anim $mod $curdate $variable $country - wait - sleep 1 - convert -loop 0 -delay 25 ${tmpdir}/${curdate}_${mod}_${country}_??.png ${tmpdir}/${curdate}_${mod}_${country}_loop.gif - currepo=${repodir}/${mod}/${variable}/${curyear}/${curmon}/${country} - mkdir -p $currepo - wait - mv ${tmpdir}/${curdate}_${mod}_${country}_* $currepo - rm $currepo/${curdate}_${mod}_${country}_??.png + if [ "$model" == "monarch" ] && [ "$country" == "default" ]; then + country="monarch_fit" + else + country=$selectedCountry + fi + runProcess done + # PRODUCE GIF FOR DECLARED MODEL AND VARIABLE else - node js/create_country_loop.js $anim $model $curdate $variable $country - wait - sleep 1 - convert -loop 0 -delay 25 ${tmpdir}/${curdate}_${model}_${country}_??.png ${tmpdir}/${curdate}_${model}_${country}_loop.gif - currepo=${repodir}/${model}/${variable}/${curyear}/${curmon}/${country} - mkdir -p $currepo - wait - mv ${tmpdir}/${curdate}_${model}_${country}_* $currepo - # rm $currepo/${curdate}_${model}_${country}_??.png + runProcess fi - wait sleep 1 done diff --git a/bin/gen_models_maps.sh b/bin/gen_models_maps.sh deleted file mode 100755 index 700f860..0000000 --- a/bin/gen_models_maps.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -# generate daily maps for each model and for the whole ensamble -# usage: script.sh [MODEL] [ANIM] [VARIABLE] [DATE] - -if [ "$1" != "" ]; then - model="$1" -else - model="all" -fi - -if [ "$2" != "" ]; then - anim="$2" -else - anim="true" -fi - -if [ "$3" != "" ]; then - curvar="$3" -else - curvar="all" -fi - -if [ "$4" != "" ]; then - curdate="$4" -else - curdate=`date '+%Y%m%d' -d '-1day'` -fi - -curyear=`date '+%Y' -d ${curdate}` -curmon=`date '+%m' -d ${curdate}` -repodir='/data/daily_dashboard/comparison/' - -for var in od550_dust sconc_dust dust_depd dust_depw dust_load dust_ext_sfc -do - variable=$var - echo "************ $model $anim $curvar $variable ****************" - if [ "$curvar" != "$variable" ] && [ "$curvar" != "all" ]; then - continue - fi - tmpdir=${HOME}/tmp/${var} - mkdir -p $tmpdir - rm -Rf $tmpdir/* - if [ "$variable" != "od550_dust" ] && [ "$variable" != "sconc_dust" ]; then - model="monarch" - node js/create_model_loop.js $anim $model $curdate ${variable} - wait - sleep 1 - convert -loop 0 -delay 25 ${tmpdir}/${curdate}_${model}_??.png ${tmpdir}/${curdate}_${model}_loop.gif - currepo=${repodir}/${model}/${variable}/${curyear}/${curmon}/ - mkdir -p $currepo - wait - mv ${tmpdir}/${curdate}_${model}_* $currepo - - elif [ "$model" == "all" ]; then - for mod in `cat interactive-forecast-viewer/conf/models.json | grep '": {' | sed 's/^.*"\(.*\)".*$/\1/g'` - do - node js/create_model_loop.js $anim $mod $curdate ${variable} - wait - sleep 1 - convert -loop 0 -delay 25 ${tmpdir}/${curdate}_${mod}_??.png ${tmpdir}/${curdate}_${mod}_loop.gif - currepo=${repodir}/${mod}/${variable}/${curyear}/${curmon}/ - mkdir -p $currepo - wait - mv ${tmpdir}/${curdate}_${mod}_* $currepo - done - else - node js/create_model_loop.js $anim $model $curdate ${variable} - wait - sleep 1 - convert -loop 0 -delay 25 ${tmpdir}/${curdate}_${model}_??.png ${tmpdir}/${curdate}_${model}_loop.gif - currepo=${repodir}/${model}/${variable}/${curyear}/${curmon}/ - mkdir -p $currepo - wait - mv ${tmpdir}/${curdate}_${model}_* $currepo - fi - -# if [ "$ensemble" == "true" ]; then -# model="all" -# node js/create_model_loop.js $anim $model $curdate ${variable} -# wait -# sleep 1 -# convert -loop 0 -delay 25 ${tmpdir}/${curdate}_${model}_??.png ${tmpdir}/${curdate}_${model}_loop.gif -# currepo=${repodir}/${model}/${variable}/${curyear}/${curmon}/ -# mkdir -p $currepo -# wait -# mv ${tmpdir}/${curdate}_${model}_* $currepo -# fi - wait - sleep 1 -done diff --git a/conf/coords.json b/conf/coords.json index 82910f3..36ab8f3 100644 --- a/conf/coords.json +++ b/conf/coords.json @@ -1,5 +1,5 @@ { - "default":{ + "old_default":{ "zoom":"2", "lat":"34", "lon": "-7", @@ -9,6 +9,16 @@ "marginTop": "0px", "logos": true }, + "default":{ + "zoom":"4", + "lat":"39.25", + "lon": "16.5", + "width": "1250", + "height": "1100", + "paddingRight": "5px", + "marginTop": "0px", + "logos": true + }, "spain":{ "zoom":"5", "lat":"34", @@ -19,16 +29,6 @@ "marginTop": "0px", "logos": true }, - "fit":{ - "zoom":"4", - "lat":"39.25", - "lon": "16.5", - "width": "1250", - "height": "1100", - "paddingRight": "5px", - "marginTop": "0px", - "logos": true - }, "monarch_fit":{ "zoom":"3", "lat":"41.25", diff --git a/js/create_country_loop.js b/js/create_country_loop.js deleted file mode 100644 index a895355..0000000 --- a/js/create_country_loop.js +++ /dev/null @@ -1,221 +0,0 @@ -// CREATE A GIF FOCUSED GIF OR PNG OVER SELECTED COUNTRY -// anim variable can be true(for gif) or a number for timestep, for png -// call from interactive-forecast-viewer folder -// date currently does not work, and defaults to current day, or most recent data -// country should have first letter capitalized, or coincide to coords.conf -// node create_country_loop true monarch 20220808 od550_dust Spain - -const { Cluster } = require('puppeteer-cluster'); -const util = require('util'); -const path = require('path'); -const url = 'http://127.0.0.1:9000/dashboard/' -const modelDict = {'od550_dust':'AOD', - 'sconc_dust':'Concentration', - 'dust_depd':'Dry deposition', - 'dust_depw':'Wet deposition', - 'dust_load':'Load', - 'dust_ext_sfc':'Extinction'}; - -//READ COORDINATES AND CENTER FROM CONF FILE -const relpath = path.relative('js/create_country_loop.js', 'interactive-forecast-viewer/conf/coords.json'); -const coords = require(relpath); - -function delay(time) { - return new Promise(function(resolve) { - setTimeout(resolve, time) - }); -} - -const RunCluster = async (anim, curmodel, seldate, variable, country) => { - const cluster = await Cluster.launch({ - concurrency: Cluster.CONCURRENCY_CONTEXT, - puppeteerOptions: { - headless: true, - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - ] - }, - maxConcurrency: 4, - }); - - await cluster.task(async ({ page, data: args }) => { - process.stdout.write("ARGS: " + args + "\n"); - var tstep = args[0]; - var curmodel = args[1]; - var seldate = args[2]; - var variable = args[3]; - var country = args[4]; - process.stdout.write("TSTEP: " + tstep + " -- MOD: " + curmodel + " -- DATE: " + seldate + " -- VAR: " + variable +' country: ' + country + "\n"); - await page.setViewport({ width: 1280, height: 768}); - await page.goto(url, { - waitUntil: 'networkidle0', - }); - await page.waitForSelector("#graph-collection"); - // await page.waitForSelector(".graph-with-slider"); - // select variable - try { - const sel = await page.$('#variable-dropdown-forecast'); - await sel.click(); - await page.waitForSelector(".Select-menu-outer"); - const dropdown_variable = modelDict[variable]; - process.stdout.write("Selected dropdown variable: " + dropdown_variable + '\n'); - await page.evaluate((dropdown_variable) => { - const options = [...document.querySelectorAll('.VirtualizedSelectOption')]; - selected_variable = options.find(element => element.textContent === dropdown_variable); - selected_variable.click(); - }, dropdown_variable); - } catch (err) { - process.stdout.write("ERR0: " + err + "\n"); - } - // select all models - if (curmodel === "all") { - try { - process.stdout.write("SELECT ALL MODELS\n"); - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED BEFORE: " + checked + "\n"); - if (!checked) { - await model.click(); - } - } - } catch (err) { - process.stdout.write("ERR1: " + err + "\n"); - } - } - // select only one model - else { - try { - process.stdout.write("SELECT MODEL: " + curmodel + "\n"); - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED BEFORE: " + checked + "\n"); - const value = await model.evaluate(elem => elem.value); - process.stdout.write("VALUE: " + value + "\n"); - if (!checked && value === curmodel) { - await model.click(); - } - if (checked && value !== curmodel) { - await model.click(); - await delay(500); - } - } - } catch (err) { - process.stdout.write("ERR1: " + err + "\n"); - } - } - // apply button - try { - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED AFTER: " + checked + "\n"); - } - process.stdout.write("CLICK APPLY BUTTON\n"); - const btn = await page.$('#models-apply'); - await btn.click(); // "button#models-apply"); - await delay(500); - } catch (err) { - process.stdout.write("ERR2: " + err + "\n"); - } - // apply none country button - try { - var zoom = coords[country].zoom; - var lat = coords[country].lat; - var lon = coords[country].lon; - process.stdout.write("Zoom: " + zoom + " Lat " + lat + " Lon " + lon +"\n"); - - // Change hidden inputs and button to visible - let zoomInput = await page.$('#country-zoom'); - await zoomInput.evaluate((el) => el.style.display = 'block'); - let latInput = await page.$('#country-lat'); - await latInput.evaluate((el) => el.style.display = 'block'); - let lonInput = await page.$('#country-lon'); - await lonInput.evaluate((el) => el.style.display = 'block'); - let zoomButton = await page.$('#country-focus'); - await zoomButton.evaluate((el) => el.style.display = 'block'); - - // Add data and click - await page.type('#country-zoom', zoom); - await page.type('#country-lat', lat); - await page.type('#country-lon', lon); - process.stdout.write("CLICK HIDDEN COUNTRY BUTTON\n"); - await page.click('#country-focus'); - - // Make elements invisible again - await zoomInput.evaluate((el) => el.style.display = 'none'); - await latInput.evaluate((el) => el.style.display = 'none'); - await lonInput.evaluate((el) => el.style.display = 'none'); - await zoomButton.evaluate((el) => el.style.display = 'none'); - - } catch (err) { - process.stdout.write("ERR3: " + err + "\n"); - } - var num = "00"; - if (tstep === "false") { - process.stdout.write("CURRENT SELECTION: " + tstep + "\n"); - num = "_curr"; - } else { - process.stdout.write("CURRENT TSTEP: " + tstep + "\n"); - const steps = await page.$$("span.rc-slider-dot"); - await steps[tstep].click(); - if (tstep<10) { - num = "0"+tstep; - } else { - num = tstep; - } - } - // wait for alert div to disappear to start grabbing elements - await page.waitForSelector("#alert-forecast", {hidden: true}); - await page.waitForSelector("#graph-collection"); - const graph = await page.$('#graph-collection'); - // remove timeslider - await page.waitForSelector(".layout-dropdown"); - process.stdout.write("REMOVING LAYOUT DROPDOWN" + "\n"); - await page.evaluate((sel) => { - let toRemove = document.querySelector(sel); - toRemove.parentNode.removeChild(toRemove); - }, '.layout-dropdown'); - // remove zoom panel - process.stdout.write("REMOVING ZOOM PANEL(S)" + "\n"); - await page.waitForSelector(".leaflet-bar"); - await page.evaluate((sel) => { - let toRemove = document.querySelectorAll(sel); - for (let j=0; j " + outputFile + "\n"); - await graph.screenshot({ path: outputFile }); - - }); - - if (anim === "true") { - for (let i=0; i<25; i++) { - try { - process.stdout.write("QUEUEING step:" + i + "\n"); - await cluster.queue([i, curmodel, seldate, variable, country]); - } catch (err) { - console.log(err); - process.stdout.write("ERROR:" + err + "\n"); - } - } - } else { - process.stdout.write("QUEUEING current:" + anim + "\n"); - cluster.queue([anim, curmodel, seldate, variable, country]); - } - - await cluster.idle(); - await cluster.close(); -} - -var anim = process.argv[2]; // default: false; -var curmodel = process.argv[3]; // default: "none"; -var seldate = process.argv[4]; // default: "none"; -var variable = process.argv[5]; // default: OD550_DUST -var country = process.argv[6]; // default: Spain - -process.stdout.write("START -> ANIM: " + anim + " CURMODEL: " + curmodel + " DATE: " + seldate + " VAR: " + variable + " country: " + country + "\n"); -RunCluster(anim, curmodel, seldate, variable, country); - diff --git a/js/create_model_loop.js b/js/create_model_loop.js deleted file mode 100644 index c70102a..0000000 --- a/js/create_model_loop.js +++ /dev/null @@ -1,207 +0,0 @@ -const { Cluster } = require('puppeteer-cluster'); -const util = require('util'); -const url = 'http://127.0.0.1:9000/daily_dashboard/' -//const uniqueId = Date.now().toString(36) + Math.random().toString(36).substring(2); -// const imageTemplate = '/data/daily_dashboard/comparison/'; // + uniqueId + '_image'; -const modelDict = {'od550_dust':'AOD', - 'sconc_dust':'Concentration', - 'dust_depd':'Dry deposition', - 'dust_depw':'Wet deposition', - 'dust_load':'Load', - 'dust_ext_sfc':'Extinction'}; - -function delay(time) { - return new Promise(function(resolve) { - setTimeout(resolve, time) - }); -} - -const RunCluster = async (anim, curmodel, seldate, variable) => { - const cluster = await Cluster.launch({ - concurrency: Cluster.CONCURRENCY_CONTEXT, - puppeteerOptions: { - headless: true, - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - // '--start-maximized', - ] - }, - maxConcurrency: 4, - }); - - await cluster.task(async ({ page, data: args }) => { - process.stdout.write("ARGS: " + args + "\n"); - var tstep = args[0]; - var curmodel = args[1]; - var seldate = args[2]; - var variable = args[3]; - process.stdout.write("TSTEP: " + tstep + " -- MOD: " + curmodel + " -- DATE: " + seldate + " -- VAR: " + variable + "\n"); - await page.setViewport({ width: 1280, height: 768}); - await page.goto(url, { - waitUntil: 'networkidle0', - }); - await page.waitForSelector("#graph-collection"); - await page.waitForSelector(".graph-with-slider"); - // select variable - try { - const sel = await page.$('#variable-dropdown-forecast'); - await sel.click(); - await page.waitForSelector(".Select-menu-outer"); - const dropdown_variable = modelDict[variable]; - process.stdout.write("Selected dropdown variable: " + dropdown_variable + '\n'); - await page.evaluate((dropdown_variable) => { - const options = [...document.querySelectorAll('.VirtualizedSelectOption')]; - selected_variable = options.find(element => element.textContent === dropdown_variable); - selected_variable.click(); - }, dropdown_variable); - } catch (err) { - process.stdout.write("ERR0: " + err + "\n"); - } - // select all models - if (curmodel === "all") { - try { - process.stdout.write("SELECT ALL MODELS\n"); - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED BEFORE: " + checked + "\n"); - if (!checked) { - await model.click(); - } - } - } catch (err) { - process.stdout.write("ERR1: " + err + "\n"); - } - } - // select only one model - else { - try { - process.stdout.write("SELECT MODEL: " + curmodel + "\n"); - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED BEFORE: " + checked + "\n"); - const value = await model.evaluate(elem => elem.value); - process.stdout.write("VALUE: " + value + "\n"); - if (!checked && value === curmodel) { - await model.click(); - } - if (checked && value !== curmodel) { - await model.click(); - await delay(500); - } - } - } catch (err) { - process.stdout.write("ERR1: " + err + "\n"); - } - } - // apply button - try { - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED AFTER: " + checked + "\n"); - } - process.stdout.write("CLICK APPLY BUTTON\n"); - const btn = await page.$('#models-apply'); - await btn.click(); // "button#models-apply"); - await delay(500); - } catch (err) { - process.stdout.write("ERR2: " + err + "\n"); - } - // select date -// if (seldate !== "none") { -// try { -// await page.waitForSelector('input#date'); -// const curdate = await page.evaluate(() => document.querySelector('input[id=date]').value); -// process.stdout.write("CURDATE: " + curdate + "\n"); -// // const input = await page.$('input[id=date]'); -// await page.focus('input[id=date]'); // input.click(); -// await page.keyboard.type(""); -// await page.keyboard.type(seldate); -// // await page.$eval('input[id=date]', (el, seldate) => el.setAttribute("value", seldate), seldate); -// process.stdout.write("DATE: " + seldate + "\n"); -// // await page.evaluate(() => document.querySelector('input[id=date]').value = seldate); -// // await page.$eval('input[id=date]', (e, seldate) => { -// // e.setAttribute("value", seldate), -// // seldate -// // }); -// await delay(500); -// const curdate2 = await page.evaluate(() => document.querySelector('input[id=date]').value); -// process.stdout.write("CURDATE2: " + curdate2 + "\n"); -// } catch (err) { -// process.stdout.write("ERR3: " + err + "\n"); -// } -// } - // select timestep - var num = "00"; - if (tstep === "false") { - process.stdout.write("CURRENT SELECTION: " + tstep + "\n"); - num = "_curr"; - } else { - process.stdout.write("CURRENT TSTEP: " + tstep + "\n"); - const steps = await page.$$("span.rc-slider-dot"); - await steps[tstep].click(); - if (tstep<10) { - num = "0"+tstep; - } else { - num = tstep; - } - } - // wait for alert div to disappear to start grabbing elements - await page.waitForSelector("#alert-forecast", {hidden: true}); - await page.waitForSelector("#graph-collection"); - const graph = await page.$('#graph-collection'); - // remove timeslider - await page.waitForSelector(".layout-dropdown"); - process.stdout.write("REMOVING LAYOUT DROPDOWN" + "\n"); - await page.evaluate((sel) => { - let toRemove = document.querySelector(sel); - toRemove.parentNode.removeChild(toRemove); - }, '.layout-dropdown'); - // remove zoom panel - process.stdout.write("REMOVING ZOOM PANEL(S)" + "\n"); - await page.waitForSelector(".leaflet-bar"); - await page.evaluate((sel) => { - let toRemove = document.querySelectorAll(sel); - for (let j=0; j el.style.display = 'inline'); - - const outputFile = './tmp/' + variable.toLowerCase() + '/' + seldate + '_' + curmodel + '_' + num + '.png'; - process.stdout.write("TAKE SCREENSHOT => " + outputFile + "\n"); - await graph.screenshot({ path: outputFile }); - - }); - - if (anim === "true") { - for (let i=0; i<25; i++) { - try { - process.stdout.write("QUEUEING step:" + i + "\n"); - await cluster.queue([i, curmodel, seldate, variable]); - } catch (err) { - console.log(err); - process.stdout.write("ERROR:" + err + "\n"); - } - } - } else { - process.stdout.write("QUEUEING current:" + anim + "\n"); - cluster.queue([anim, curmodel, seldate, variable]); - } - - await cluster.idle(); - await cluster.close(); -} - -var anim = process.argv[2]; // default: false; -var curmodel = process.argv[3]; // default: "none"; -var seldate = process.argv[4]; // default: "none"; -var variable = process.argv[5]; // default: OD550_DUST - -process.stdout.write("START -> ANIM: " + anim + " CURMODEL: " + curmodel + " DATE: " + seldate + " VAR: " + variable + "\n"); -RunCluster(anim, curmodel, seldate, variable); - diff --git a/js/create_model_loop_zoom_fit.js b/js/create_model_loop_zoom_fit.js index 77d3b4c..d42d300 100644 --- a/js/create_model_loop_zoom_fit.js +++ b/js/create_model_loop_zoom_fit.js @@ -2,222 +2,237 @@ // anim variable can be true(for gif) or a number for timestep, for png // call from interactive-forecast-viewer folder // date currently does not work, and defaults to current day, or most recent data -// node create_fit_loop true monarch 20220808 od550_dust spain +// node create_model_loop_zoom_fit true monarch 20220808 od550_dust spain const { Cluster } = require('puppeteer-cluster'); const util = require('util'); const path = require('path'); -// const url = 'http://127.0.0.1:9000/dashboard/' -const url = 'http://0.0.0.0:7778/dashboard/' +const url = 'http://127.0.0.1:9000/dashboard/' +// const url = 'http://0.0.0.0:7778/dashboard/' const modelDict = {'od550_dust':'AOD', - 'sconc_dust':'Concentration', - 'dust_depd':'Dry deposition', - 'dust_depw':'Wet deposition', - 'dust_load':'Load', - 'dust_ext_sfc':'Extinction'}; + 'sconc_dust':'Concentration', + 'dust_depd':'Dry deposition', + 'dust_depw':'Wet deposition', + 'dust_load':'Load', + 'dust_ext_sfc':'Extinction'}; //READ COORDINATES AND CENTER FROM CONF FILE const relpath = path.relative('js/create_model_loop_zoom_fit.js', 'interactive-forecast-viewer/conf/coords.json'); const coords = require(relpath); function delay(time) { - return new Promise(function(resolve) { - setTimeout(resolve, time) - }); + return new Promise(function(resolve) { + setTimeout(resolve, time) + }); } const RunCluster = async (anim, curmodel, seldate, variable, fit) => { - const cluster = await Cluster.launch({ - concurrency: Cluster.CONCURRENCY_CONTEXT, - puppeteerOptions: { - headless: true, - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - ] - }, - maxConcurrency: 4, - }); - - await cluster.task(async ({ page, data: args }) => { - process.stdout.write("ARGS: " + args + "\n"); - var tstep = args[0]; - var curmodel = args[1]; - var seldate = args[2]; - var variable = args[3]; - var fit = args[4].toLowerCase(); - process.stdout.write("TSTEP: " + tstep + " -- MOD: " + curmodel + " -- DATE: " + seldate + " -- VAR: " + variable +' fit: ' + fit + "\n"); - var width_aspect = parseInt(coords[fit].width); - var height_aspect = parseInt(coords[fit].height); - await page.setViewport({ width: width_aspect, height: height_aspect}); - await page.goto(url, { - waitUntil: 'networkidle0', - }); - await page.waitForSelector("#graph-collection"); - // await page.waitForSelector(".graph-with-slider"); - // select variable - try { - const sel = await page.$('#variable-dropdown-forecast'); - await sel.click(); - await page.waitForSelector(".Select-menu-outer"); - const dropdown_variable = modelDict[variable]; - process.stdout.write("Selected dropdown variable: " + dropdown_variable + '\n'); - await page.evaluate((dropdown_variable) => { - const options = [...document.querySelectorAll('.VirtualizedSelectOption')]; - selected_variable = options.find(element => element.textContent === dropdown_variable); - selected_variable.click(); - }, dropdown_variable); + const cluster = await Cluster.launch({ + concurrency: Cluster.CONCURRENCY_CONTEXT, + puppeteerOptions: { + headless: true, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + ] + }, + maxConcurrency: 4, + }); + + await cluster.task(async ({ page, data: args }) => { + process.stdout.write("ARGS: " + args + "\n"); + var tstep = args[0]; + var curmodel = args[1]; + var seldate = args[2]; + var variable = args[3]; + var fit = args[4].toLowerCase(); + process.stdout.write("TSTEP: " + tstep + " -- MOD: " + curmodel + " -- DATE: " + seldate + " -- VAR: " + variable +' fit: ' + fit + "\n"); + var width_aspect = parseInt(coords[fit].width); + var height_aspect = parseInt(coords[fit].height); + await page.setViewport({ width: width_aspect, height: height_aspect}); + await page.goto(url, { + waitUntil: 'networkidle0', + }); + await page.waitForSelector("#graph-collection"); + // select variable + try { + const sel = await page.$('#variable-dropdown-forecast'); + await sel.click(); + await page.waitForSelector(".Select-menu-outer"); + const dropdown_variable = modelDict[variable]; + process.stdout.write("Selected dropdown variable: " + dropdown_variable + '\n'); + await page.evaluate((dropdown_variable) => { + const options = [...document.querySelectorAll('.VirtualizedSelectOption')]; + selected_variable = options.find(element => element.textContent === dropdown_variable); + selected_variable.click(); + }, dropdown_variable); + } catch (err) { + process.stdout.write("ERR0: " + err + "\n"); + } + // select all models + if (curmodel === "all") { + try { + process.stdout.write("SELECT ALL MODELS\n"); + for (const model of await page.$$('.custom-control-input')) { + const checked = await model.evaluate(elem => elem.checked); + process.stdout.write("CHECKED BEFORE: " + checked + "\n"); + if (!checked) { + await model.click(); + } + } + } catch (err) { + process.stdout.write("ERR1: " + err + "\n"); + } + } + // select only one model + else { + try { + process.stdout.write("SELECT MODEL: " + curmodel + "\n"); + for (const model of await page.$$('.custom-control-input')) { + const checked = await model.evaluate(elem => elem.checked); + process.stdout.write("CHECKED BEFORE: " + checked + "\n"); + const value = await model.evaluate(elem => elem.value); + process.stdout.write("VALUE: " + value + "\n"); + if (!checked && value === curmodel) { + await model.click(); + } + if (checked && value !== curmodel) { + await model.click(); + await delay(500); + } + } + } catch (err) { + process.stdout.write("ERR2: " + err + "\n"); + } + } + // apply button + try { + for (const model of await page.$$('.custom-control-input')) { + const checked = await model.evaluate(elem => elem.checked); + process.stdout.write("CHECKED AFTER: " + checked + "\n"); + } + process.stdout.write("CLICK APPLY BUTTON\n"); + const btn = await page.$('#models-apply'); + await btn.click(); // "button#models-apply"); + await delay(500); } catch (err) { - process.stdout.write("ERR0: " + err + "\n"); + process.stdout.write("ERR3: " + err + "\n"); } - // select all models - if (curmodel === "all") { - try { - process.stdout.write("SELECT ALL MODELS\n"); - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED BEFORE: " + checked + "\n"); - if (!checked) { - await model.click(); - } - } - } catch (err) { - process.stdout.write("ERR1: " + err + "\n"); - } - } - // select only one model - else { - try { - process.stdout.write("SELECT MODEL: " + curmodel + "\n"); - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED BEFORE: " + checked + "\n"); - const value = await model.evaluate(elem => elem.value); - process.stdout.write("VALUE: " + value + "\n"); - if (!checked && value === curmodel) { - await model.click(); - } - if (checked && value !== curmodel) { - await model.click(); - await delay(500); - } - } - } catch (err) { - process.stdout.write("ERR1: " + err + "\n"); - } - } -// CHANGE fit TO FIT AND COMBINE ALL FILES INTO ONE - // apply button - try { - for (const model of await page.$$('.custom-control-input')) { - const checked = await model.evaluate(elem => elem.checked); - process.stdout.write("CHECKED AFTER: " + checked + "\n"); - } - process.stdout.write("CLICK APPLY BUTTON\n"); - const btn = await page.$('#models-apply'); - await btn.click(); // "button#models-apply"); - await delay(500); - } catch (err) { - process.stdout.write("ERR2: " + err + "\n"); - } - // apply none fit button - try { - var zoom = coords[fit].zoom; - var lat = coords[fit].lat; - var lon = coords[fit].lon; - process.stdout.write("Zoom: " + zoom + " Lat " + lat + " Lon " + lon +"\n"); - - // Change hidden inputs and button to visible - let zoomInput = await page.$('#country-zoom'); - await zoomInput.evaluate((el) => el.style.display = 'block'); - let latInput = await page.$('#country-lat'); - await latInput.evaluate((el) => el.style.display = 'block'); - let lonInput = await page.$('#country-lon'); - await lonInput.evaluate((el) => el.style.display = 'block'); - let zoomButton = await page.$('#country-focus'); - await zoomButton.evaluate((el) => el.style.display = 'block'); - - // Add data and click - await page.type('#country-zoom', zoom); - await page.type('#country-lat', lat); - await page.type('#country-lon', lon); - process.stdout.write("CLICK HIDDEN country BUTTON\n"); - await page.click('#country-focus'); - - // Make elements invisible again - await zoomInput.evaluate((el) => el.style.display = 'none'); - await latInput.evaluate((el) => el.style.display = 'none'); - await lonInput.evaluate((el) => el.style.display = 'none'); - await zoomButton.evaluate((el) => el.style.display = 'none'); - - } catch (err) { - process.stdout.write("ERR3: " + err + "\n"); - } - var num = "00"; - if (tstep === "false") { - process.stdout.write("CURRENT SELECTION: " + tstep + "\n"); - num = "_curr"; - } else { - process.stdout.write("CURRENT TSTEP: " + tstep + "\n"); - const steps = await page.$$("span.rc-slider-dot"); - await steps[tstep].click(); - if (tstep<10) { - num = "0"+tstep; - } else { - num = tstep; - } - } - // wait for alert div to disappear to start grabbing elements - await page.waitForSelector("#alert-forecast", {hidden: true}); - await page.waitForSelector("#graph-collection"); - const graph = await page.$('#graph-collection'); - // remove timeslider - await page.waitForSelector(".layout-dropdown"); - process.stdout.write("REMOVING LAYOUT DROPDOWN" + "\n"); - await page.evaluate((sel) => { - let toRemove = document.querySelector(sel); - toRemove.parentNode.removeChild(toRemove); - }, '.layout-dropdown'); - // remove zoom panel - process.stdout.write("REMOVING ZOOM PANEL(S)" + "\n"); - await page.waitForSelector(".leaflet-bar"); - await page.evaluate((sel) => { - let toRemove = document.querySelectorAll(sel); - for (let j=0; j el.style.display = 'inline'); - await logos.evaluate((el, margin) => {el.style.marginTop = margin}, coords[fit].marginTop); - await logos.evaluate((el, padding) => {el.style.paddingRight = padding}, coords[fit].paddingRight); - }; - // Handle output and take screenshot - const outputFile = './tmp/' + variable.toLowerCase() + '/' + fit + '/' + seldate + '_' + curmodel + '_' + fit + '_' + num + '.png'; - process.stdout.write("TAKE SCREENSHOT => " + outputFile + "\n"); - await graph.screenshot({ path: outputFile }); + // apply none fit button + try { + var zoom = coords[fit].zoom; + var lat = coords[fit].lat; + var lon = coords[fit].lon; + process.stdout.write("Zoom: " + zoom + " Lat " + lat + " Lon " + lon +"\n"); + + // Change hidden inputs and button to visible + let zoomInput = await page.$('#country-zoom'); + await zoomInput.evaluate((el) => el.style.display = 'block'); + let latInput = await page.$('#country-lat'); + await latInput.evaluate((el) => el.style.display = 'block'); + let lonInput = await page.$('#country-lon'); + await lonInput.evaluate((el) => el.style.display = 'block'); + let zoomButton = await page.$('#country-focus'); + await zoomButton.evaluate((el) => el.style.display = 'block'); + + // Add data and click + await page.type('#country-zoom', zoom); + await page.type('#country-lat', lat); + await page.type('#country-lon', lon); + process.stdout.write("CLICK HIDDEN country BUTTON\n"); + await page.click('#country-focus'); + + // Make elements invisible again + await zoomInput.evaluate((el) => el.style.display = 'none'); + await latInput.evaluate((el) => el.style.display = 'none'); + await lonInput.evaluate((el) => el.style.display = 'none'); + await zoomButton.evaluate((el) => el.style.display = 'none'); + + } catch (err) { + process.stdout.write("ERR4: " + err + "\n"); + } + var num = "00"; + if (tstep === "false") { + process.stdout.write("CURRENT SELECTION: " + tstep + "\n"); + num = "_curr"; + } else { + process.stdout.write("CURRENT TSTEP: " + tstep + "\n"); + const steps = await page.$$("span.rc-slider-dot"); + await steps[tstep].click(); + if (tstep<10) { + num = "0"+tstep; + } else { + num = tstep; + } + } + // wait for alert div to disappear to start grabbing elements + await page.waitForSelector("#alert-forecast", {hidden: true}); + await page.waitForSelector("#graph-collection"); + const graph = await page.$('#graph-collection'); + // change graph height to fill output gif + let graphHeight = await page.$('.leaflet-container'); + await graphHeight.evaluate((el) => el.style.height = '93vh'); + // remove timeslider + await page.waitForSelector(".navbar-timebar"); + process.stdout.write("REMOVING NAVBAR" + "\n"); + await page.evaluate((sel) => { + let toRemove = document.querySelector(sel); + toRemove.parentNode.removeChild(toRemove); + }, '.navbar-timebar'); + // remove zoom panel + process.stdout.write("REMOVING ZOOM PANEL(S)" + "\n"); + await page.waitForSelector(".leaflet-bar"); + await page.evaluate((sel) => { + let toRemove = document.querySelectorAll(sel); + for (let j=0; j el.style.display = 'inline'); + await logos.evaluate((el, margin) => {el.style.marginTop = margin}, coords[fit].marginTop); + await logos.evaluate((el, padding) => {el.style.paddingRight = padding}, coords[fit].paddingRight); + } catch (err) { + console.log(err); + process.stdout.write("ERR5:" + err + "\n"); + } + }; + // Handle output and take screenshot + //Filename shouldn't include fit unless it is a specified country + if (fit == 'monarch_fit' || fit == 'default') { + // Change the below to './js/tmp' ... during developement, run script from interactiv... + var outputFile = './tmp/' + variable.toLowerCase() + '/' + seldate + '_' + curmodel + '_' + num + '.png'; + } else { + // Create a filename that includes the specified country fit + // Change the below to './js/tmp' ... during developement, run script from interactiv... + var outputFile = './tmp/' + variable.toLowerCase() + '/' + fit + '/' + seldate + '_' + curmodel + '_' + fit + '_' + num + '.png'; + } + + process.stdout.write("TAKE SCREENSHOT => " + outputFile + "\n"); + await graph.screenshot({ path: outputFile }); }); if (anim === "true") { - for (let i=0; i<25; i++) { - try { - process.stdout.write("QUEUEING step:" + i + "\n"); - await cluster.queue([i, curmodel, seldate, variable, fit]); - } catch (err) { - console.log(err); - process.stdout.write("ERROR:" + err + "\n"); - } - } + for (let i=0; i<25; i++) { + try { + process.stdout.write("QUEUEING step:" + i + "\n"); + await cluster.queue([i, curmodel, seldate, variable, fit]); + } catch (err) { + console.log(err); + process.stdout.write("ERR6:" + err + "\n"); + } + } } else { - process.stdout.write("QUEUEING current:" + anim + "\n"); - cluster.queue([anim, curmodel, seldate, variable, fit]); + process.stdout.write("QUEUEING current:" + anim + "\n"); + cluster.queue([anim, curmodel, seldate, variable, fit]); } await cluster.idle(); -- GitLab