diff --git a/.codacy.yml b/.codacy.yml index bf12fc2d77140a852ff1b9197370766a0ecec7c3..0bc0d39b0f2175fbc21f61847dace53b65460c48 100755 --- a/.codacy.yml +++ b/.codacy.yml @@ -13,7 +13,7 @@ engines: enabled: true pylint: enabled: true - python_version: 2 + python_version: 3 exclude_paths: [ 'doc/**', diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08c1355a02d158efce8b3c8438df86fdc3275b6e..d4676c915a2d3b2e503a4a7e293c64e664d1875b 100755 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,11 +20,11 @@ prepare: test_python2: stage: test script: - - conda env update -f environment.yml -n hermesv3_gr python=2.7 + - conda env update -f environment.yml -n hermesv3_gr python=3.7 - source activate hermesv3_gr - python run_test.py - pip install codacy-coverage --upgrade - - python-codacy-coverage -r tests/report/python2/coverage.xml + - python-codacy-coverage -r tests/report/python3/coverage.xml #test_python3: # stage: test diff --git a/CHANGELOG b/CHANGELOG index 7d4bbafa65b02f3ce2e21662713287f5f90fbc05..fdfe3ac3dd15ac1d59e474d20963d2a386fd2c32 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +2.0.0 + 2019/11/20 + - Python 3 + - first_time option + - erase_auxiliary_files option + - Option to return emissions in memory + 1.0.4 2019/07/19 - Specified version on timezonefinder to be less than 4.0.0 (not further support to python 2.7.X) diff --git a/conf/hermes.conf b/conf/hermes.conf index a9d139395600ea7e6bdb7c016f6b4d649469b192..7a125976b8999b572c72dcc012ebf68037b17ee4 100755 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -11,6 +11,8 @@ start_date = 2017/01/01 00:00:00 output_timestep_type = hourly output_timestep_num = 24 output_timestep_freq = 1 +first_time = 0 +erase_auxiliary_files = 1 [DOMAIN] @@ -27,7 +29,7 @@ domain_type = global # domain_type = rotated # domain_type = mercator vertical_description = /data/profiles/vertical/Benchmark_15layers_vertical_description.csv -auxiliar_files_path = /data/auxiliar_files/_ +auxiliary_files_path = /data/auxiliar_files/_ # if domain_type == global: inc_lat = 1. diff --git a/data/profiles/speciation/MolecularWeights.csv b/data/profiles/speciation/MolecularWeights.csv index f005210cc7e5ade22ae2dc05df1b85cfe274fb0a..262b5917458ec2ea0c89eba806f74e7597d1801e 100755 --- a/data/profiles/speciation/MolecularWeights.csv +++ b/data/profiles/speciation/MolecularWeights.csv @@ -10,6 +10,9 @@ pm25_fossil;1.0 pm25_bio;1.0 oc;1.0 bc;1.0 +ec;1.0 +so4;1.0 +ash;1.0 c2h6s;62.13 hcl;36.46 c2h2;26.04 @@ -63,3 +66,4 @@ voc22;68.8 voc23;75.3 voc24;59.1 voc25;86.9 +nmvoc;1.0 \ No newline at end of file diff --git a/data/profiles/speciation/Speciation_profile_cb05_aero6_CMAQ.csv b/data/profiles/speciation/Speciation_profile_cb05_aero6_CMAQ.csv index 15467354fd0fb6a99c84a680da24971aa5f0b692..e546468550329be7eca55a483ba1c66a20b8a4ce 100755 --- a/data/profiles/speciation/Speciation_profile_cb05_aero6_CMAQ.csv +++ b/data/profiles/speciation/Speciation_profile_cb05_aero6_CMAQ.csv @@ -26,3 +26,49 @@ E022;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0; E023;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25;pm10-pm25 E024;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-bc;pm10-pm25 E086;0;0;0;0;so2;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E074;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.000008*nmvoc;0;0;0.000858*nmvoc;0.004177*nmvoc;0;0.018548*nmvoc;0;0;0;0.00104*nmvoc;0.011594*nmvoc;0;0;0;0.000893*nmvoc;0;0;0;0.02*pm25;0.01*pm25;0;0.15*pm25;0;pm25*0.00379;pm25*0.01152;pm25*0.03437;pm25*0.0844;pm25*0.00227;pm25*0.00021;pm25*0.00064;pm25*0.01573;pm25*0.05032;pm25*0.02394;pm25*0.00311;pm25*0.00438;pm25*0.58524;pm10-pm25 +E075;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.000013*nmvoc;0;0;0.003224*nmvoc;0.001662*nmvoc;0.000659*nmvoc;0.008038*nmvoc;0;0;0.000175*nmvoc;0.000496*nmvoc;0.025751*nmvoc;0;0;0;0.001726*nmvoc;0;0;0;0.03*pm25;0.01*pm25;0;0.1*pm25;pm25*0.02498;0;pm25*0.03098;0;0;0;0;0;0;0;0;0;0;pm25*0.80402;pm10-pm25 +E076;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.000047*nmvoc;0;0;0.002914*nmvoc;0.004187*nmvoc;0.000849*nmvoc;0.00186*nmvoc;0;0;0;0.002559*nmvoc;0.029992*nmvoc;0;0;0;0.001755*nmvoc;0;0;0;0.35*pm25;0.18*pm25;0;0.02*pm25;0;pm25*0.00254;pm25*0.3602;pm25*0.00767;pm25*0.00472;pm25*0.00197;pm25*0.00003;pm25*0.00074;pm25*0.01023;pm25*0.00446;pm25*0.00569;pm25*0.00026;pm25*0.00984;pm25*0.04158;pm10-pm25 +E077;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0.000006*nmvoc;0.005934*nmvoc;0;0.000026*nmvoc;0;0.000001*nmvoc;0;0.000373*nmvoc;0.055357*nmvoc;0;0;0;0.000048*nmvoc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25;pm10-pm25 +E078;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0.000087*nmvoc;0.000148*nmvoc;0.003563*nmvoc;0.000004*nmvoc;0;0.000001*nmvoc;0.001904*nmvoc;0.000161*nmvoc;0.036006*nmvoc;0;0;0;0.001686*nmvoc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E079;0.7*nox_no2;0.283*nox_no2;0.017*nox_no2;co;so2;nh3;0.000592*nmvoc;0;0;0.002423*nmvoc;0.001607*nmvoc;0;0.000899*nmvoc;0;0;0;0.002589*nmvoc;0.028079*nmvoc;0;0;0;0.003302*nmvoc;0;0;0;0.32*pm25;0.49*pm25;0;0.01*pm25;0;pm25*0.00006;pm25*0.02119;pm25*0.00013;pm25*0.00047;pm25*0.00002;0;pm25*0.00006;pm25*0.00116;pm25*0.00003;pm25*0.00018;0;0;pm25*0.00364;pm10-pm25 +E080;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0.002496*nmvoc;0.004456*nmvoc;0;0;0.001693*nmvoc;0;0;0;0.001111*nmvoc;0.030672*nmvoc;0;0;0.001132*nmvoc;0.000762*nmvoc;0;0;0;0.12*pm25;0.005*pm25;0;0.40*pm25;0;0;pm25*0.07404;pm25*0.00018;pm25*0.00074;0;0;0;0;0;pm25*0.00006;0;0;pm25*0.17496;pm10-pm25 +E081;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.001084*nmvoc;0.000756*nmvoc;0.000248*nmvoc;0.00622*nmvoc;0.000293*nmvoc;0;0.005437*nmvoc;0.000086*nmvoc;0;0;0.003296*nmvoc;0.011816*nmvoc;0;0;0.000192*nmvoc;0.000124*nmvoc;0;0;0;0.62*pm25;0.16*pm25;0;0.15*pm25;0;0;pm25*0.07404;pm25*0.00018;pm25*0.00074;0;0;0;0;0;pm25*0.00006;0;0;pm25*0.17496;pm10-pm25 +E082;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.000227*nmvoc;0.000311*nmvoc;0.000333*nmvoc;0.003921*nmvoc;0.000166*nmvoc;0;0.00206*nmvoc;0.00079*nmvoc;0;0;0.00111*nmvoc;0.039123*nmvoc;0;0;0.000245*nmvoc;0.000476*nmvoc;0;0;0;0.31*pm25;0.41*pm25;0;0.03*pm25;0;0;pm25*0.07404;pm25*0.00018;pm25*0.00074;0;0;0;0;0;pm25*0.00006;0;0;pm25*0.17496;pm10-pm25 +E083;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0;0;0.001867*nmvoc;0.008553*nmvoc;0.000529*nmvoc;0.005349*nmvoc;0;0.000169*nmvoc;0.000133*nmvoc;0.000435*nmvoc;0.025797*nmvoc;0;0;0;0.000742*nmvoc;0;0;0;0.31*pm25;0.2*pm25;0;0;0;pm25*0.08261;pm25*0.02011;pm25*0.01311;pm25*0.04112;0;pm25*0.00035;pm25*0.00484;pm25*0.0492;pm25*0.01191;pm25*0.01424;pm25*0.00143;pm25*0.01293;pm25*0.23809;pm10-pm25 +E084;nox_no2;0;0;co;0;nh3;0.001591*nmvoc;0.000103*nmvoc;0.000038*nmvoc;0;0;0.000022*nmvoc;0;0;0;0;0.000023*nmvoc;0.03941*nmvoc;0;0.000007*nmvoc;0.000237*nmvoc;0;0;0;0;0.318*pm25;0.0516*pm25;0;0.0446*pm25;pm25*0.00427;pm25*0.04911;pm25*0.17393;pm25*0.00506;pm25*0.00919;pm25*0.00042;pm25*0.00009;pm25*0.00463;pm25*0.01719;pm25*0.0031;pm25*0.00277;pm25*0.00011;pm25*0.04219;pm25*0.05788;pm10-pm25 +E085;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.002093*nmvoc;0.000525*nmvoc;0.000105*nmvoc;0.002929*nmvoc;0.001894*nmvoc;0.000023*nmvoc;0.002826*nmvoc;0.000116*nmvoc;0.000043*nmvoc;0.003664*nmvoc;0.003078*nmvoc;0.010261*nmvoc;0;0.000006*nmvoc;0.00013*nmvoc;0.000006*nmvoc;0;0;0;0.48*pm25;0.15*pm25;0;0;pm25*0.00427;pm25*0.04911;pm25*0.17393;pm25*0.00506;pm25*0.00919;pm25*0.00042;pm25*0.00009;pm25*0.00463;pm25*0.01719;pm25*0.0031;pm25*0.00277;pm25*0.00011;pm25*0.04219;pm25*0.05788;pm10-pm25 +E096;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E097;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E098;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E099;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E100;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0;0.01*voc18+0.3*voc19;0;0;0;0.5*voc01;0;0;0;0.5*voc01;0;7.5*voc06+2.2*voc17+4.11*voc18+4*voc19+4*voc23;0;0;voc14+0.2*voc17;voc15+voc17;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25;pm10-pm25 +E101;0.95*nox_no2;0.042*nox_no2;0.008*nox_no2;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+1.875*voc22+4*voc23;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E102;0.7*nox_no2;0.283*nox_no2;0.017*nox_no2;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+2.2*voc17+1.875*voc22+4*voc23;0;0;voc14+0.2*voc17;voc15+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E103;0.95*nox_no2;0.042*nox_no2;0.008*nox_no2;co;so2;nh3;0.625*voc22;0.375*voc22;0;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+voc08+2.2*voc17+1.875*voc22;0;0;0.2*voc17;voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E104;0;0;0;0;0;0;0;0;voc13;0;0;0;0;0.666*voc12;0;0;0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc13;0;0;voc14;voc15;0;0;0;oc;ec;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec;pm10-pm25 +E105;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+1.875*voc22+4*voc23;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E106;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+5*voc05+7.5*voc06+voc08+voc09+voc13+2.2*voc17+1.875*voc22+4*voc23;0;0;voc14+0.2*voc17;voc15+voc17;0;0;0;oc;0;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-so4;pm10-pm25 +E107;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.375*voc22;voc13;voc07;voc02;0;voc21;0.666*voc12;0;0;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;so4;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec-so4;pm10-pm25 +E108;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec;pm10-pm25 +E109;0;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;oc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc;pm10-pm25 +E110;0.9*nox_no2;0.1*nox_no2;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec;pm10-pm25 +E902;0;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E903;0;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E906;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;oc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc;pm10-pm25 +E907;0.9*nox_no2;0.1*nox_no2;0;co;so2;0;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;ec;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;pm25-oc-ec;pm10-pm25 +E927;nox_no2;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E928;0;0;0;co;0;0;0.001591*nmvoc;0.000103*nmvoc;0.000038*nmvoc;0;0;0.000022*nmvoc;0;0;0;0;0.000023*nmvoc;0.03941*nmvoc;0;0.000007*nmvoc;0.000237*nmvoc;0;0;0;0;0.318*pm25;0.0516*pm25;0;0.0446*pm25;pm25*0.00427;pm25*0.04911;pm25*0.17393;pm25*0.00506;pm25*0.00919;pm25*0.00042;pm25*0.00009;pm25*0.00463;pm25*0.01719;pm25*0.0031;pm25*0.00277;pm25*0.00011;pm25*0.04219;pm25*0.05788;pm10-pm25 +E929;0;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E930;0.9*nox_no2;0.1*nox_no2;0;co;so2;0;0.002093*nmvoc;0.000525*nmvoc;0.000105*nmvoc;0.002929*nmvoc;0.001894*nmvoc;0.000023*nmvoc;0.002826*nmvoc;0.000116*nmvoc;0.000043*nmvoc;0.003664*nmvoc;0.003078*nmvoc;0.010261*nmvoc;0;0.000006*nmvoc;0.00013*nmvoc;0.000006*nmvoc;0;0;0;0.48*pm25;0.15*pm25;0;0;pm25*0.00427;pm25*0.04911;pm25*0.17393;pm25*0.00506;pm25*0.00919;pm25*0.00042;pm25*0.00009;pm25*0.00463;pm25*0.01719;pm25*0.0031;pm25*0.00277;pm25*0.00011;pm25*0.04219;pm25*0.05788;pm10-pm25 +E114;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E115;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;5.6*(oc+bc)-oc-bc;3.4*(oc+bc) +E116;0.9*nox_no;0.1*nox_no;0;co;so2;0;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E117;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;3.1*(oc+bc)-oc-bc;0.6*(oc+bc) +E118;nox_no;0;0;0;0;nh3;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E119;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;1.7*(oc+bc)-oc-bc;1.5*(oc+bc) +E120;0;0;0;0;0;0;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E121;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0 +E122;0.9*nox_no;0.1*nox_no;0;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;3.4*(oc+bc)-oc-bc;0 +E123;0.7*nox_no;0.283*nox_no;0.017*nox_no;co;so2;nh3;0.625*voc22;0.01*voc18+0.3*voc19+0.375*voc22;voc13;voc07;voc02;0.5*voc01;voc21;0.666*voc12;0;0.5*voc01;voc08+0.333*voc12;1.5*voc03+4*voc04+5*voc05+7.5*voc06+voc08+voc09+voc13+voc16+2.2*voc17+4.11*voc18+4*voc19+1.875*voc22+4*voc23+voc24;0;0;voc14+0.2*voc17;voc15+voc16+voc17;0;0;0;oc;bc;0;0;0;0;0;0;0;0;0;0;0;0;0;0;0;1.2*(oc+bc)-oc-bc;0 + diff --git a/data/profiles/temporal/TemporalProfile_Hourly.csv b/data/profiles/temporal/TemporalProfile_Hourly.csv index 0434817b5b90770d2c964b8d321e71660824546f..6a65ae5f3dbb03f6498404954381803ce39ca27f 100755 --- a/data/profiles/temporal/TemporalProfile_Hourly.csv +++ b/data/profiles/temporal/TemporalProfile_Hourly.csv @@ -9,4 +9,7 @@ H007,0.6,0.6,0.6,0.6,0.6,0.65,0.75,0.9,1.1,1.35,1.45,1.6,1.65,1.75,1.7,1.55,1.35 H008,0.06,0.168,0.192,0.18,0.24,0.192,0.204,0.216,0.336,0.6,1.08,1.8,2.52,3.12,3.12,3.12,2.64,1.68,0.96,0.72,0.42,0.36,0.036,0.036 H009,0.597235,0.552995,0.497696,0.508756,0.575115,0.741014,0.917972,1.15023,1.32719,1.42673,1.45991,1.43779,1.39355,1.37143,1.36037,1.29401,1.24977,1.19447,1.09493,0.973272,0.829493,0.729954,0.685714,0.630415 H010,0.613383,0.557621,0.490706,0.468401,0.479554,0.568773,0.669145,0.814126,0.97026,1.12639,1.22677,1.27138,1.29368,1.3829,1.46097,1.47212,1.46097,1.46097,1.3829,1.23792,1.08178,0.936803,0.847584,0.724907 +H011,0.06,0.06,0.06,0.07,0.07,0.07,0.2,0.2,0.2,1.82,1.82,1.82,3.39,3.39,3.39,1.68,1.68,1.68,0.56,0.56,0.56,0.22,0.22,0.22 +H012,0.61,0.57,0.48,0.36,0.29,0.25,0.21,0.25,0.4,0.74,1.01,1.38,1.57,1.71,1.8,1.86,1.86,1.79,1.6,1.43,1.24,1.03,0.86,0.7 +H013,0.28,0.13,0.1,0.08,0.09,0.09,0.44,0.91,1.18,1.43,1.6,1.61,1.66,1.58,1.47,1.43,1.42,1.46,1.46,1.4,1.39,1.25,0.96,0.58 diff --git a/data/profiles/temporal/TemporalProfile_Monthly.csv b/data/profiles/temporal/TemporalProfile_Monthly.csv index 1c2108f814218ba132bda8db04d9021c217c4b3d..347292fe33854e95944c1d64feeaae8999983c15 100755 --- a/data/profiles/temporal/TemporalProfile_Monthly.csv +++ b/data/profiles/temporal/TemporalProfile_Monthly.csv @@ -8,5 +8,8 @@ M006,0.95,0.96,1.02,1,1.01,1.03,1.03,1.01,1.04,1.03,1.01,0.91 M007,0.88,0.92,0.98,1.03,1.05,1.06,1.01,1.02,1.06,1.05,1.01,0.93 M008,0.88,0.92,0.98,1.03,1.05,1.06,1.01,1.02,1.06,1.05,1.01,0.93 M009,0.45,1.3,2.35,1.7,0.85,0.85,0.85,1,1.1,0.65,0.45,0.45 -M999,0,0,0,0,0,0,0,0,1,0,0,0 +M010,0.7,0.75,0.85,0.9,1,1.1,1.2,1.25,1.3,1.1,1,0.85 +M011,0,2,4.75,2.9,0.5,0.4,0.2,0.5,0.75,0,0,0 +M012,0.740123177,0.685722463,0.741070559,0.840611444,1.003188502,1.206099053,1.535051489,1.500726051,1.233704586,1.026921016,0.771651666,0.715129995 + diff --git a/data/profiles/temporal/tz_world_country_iso3166.csv b/data/profiles/temporal/tz_world_country_iso3166.csv index 4c8218028d6d827280913bdebf0816a4dcd908d1..b6ddfd777ff6fcec6b2e23c380f8dbba02438a2f 100755 --- a/data/profiles/temporal/tz_world_country_iso3166.csv +++ b/data/profiles/temporal/tz_world_country_iso3166.csv @@ -420,4 +420,17 @@ "Samoa";882;"Pacific/Apia";378;"WSM" "Yemen";887;"Asia/Aden";203;"YEM" "Zambia";894;"Africa/Lusaka";36;"ZMB" -"Namibia";"264";"Africa/Windhoek";52;"NAM" \ No newline at end of file +"Namibia";"264";"Africa/Windhoek";52;"NAM" +"Antarctica";10;"Antarctica/McMurdo";424;"ATA" +"Antarctica";10;"Antarctica/Mawson";425;"ATA" +"Antarctica";10;"Antarctica/Davis";426;"ATA" +"Antarctica";10;"Antarctica/Rothera";427;"ATA" +"Antarctica";10;"Antarctica/Troll";428;"ATA" +"Antarctica";10;"Antarctica/Syowa";429;"ATA" +"Antarctica";10;"Antarctica/Vostok";430;"ATA" +"Antarctica";10;"Antarctica/Casey";431;"ATA" +"Antarctica";10;"Antarctica/DumontDUrville";432;"ATA" +"Antarctica";10;"Antarctica/Palmer";433;"ATA" +"Myanmar";104;"Asia/Yangon";434;"MMR" +"Kazakhstan";398;"Asia/Qostanay";435;"KAZ" +"UTC";0;"Etc/UTC";999;"UTC" \ No newline at end of file diff --git a/data/profiles/vertical/CMAQ_37layers_vertical_description.csv b/data/profiles/vertical/CMAQ_37layers_vertical_description.csv new file mode 100644 index 0000000000000000000000000000000000000000..a920af5ebd59f292223b40abdd9d7659d33ec634 --- /dev/null +++ b/data/profiles/vertical/CMAQ_37layers_vertical_description.csv @@ -0,0 +1,38 @@ +Ilayer;height_magl +1;40 +2;88 +3;145 +4;226 +5;324 +6;456 +7;623 +8;819 +9;1044 +10;1319 +11;1657 +12;2074 +13;2582 +14;3215 +15;3848 +16;4489 +17;5122 +18;5742 +19;6374 +20;6987 +21;7609 +22;8223 +23;8843 +24;9469 +25;10078 +26;10689 +27;11300 +28;11939 +29;12580 +30;13218 +31;13971 +32;14779 +33;15599 +34;16616 +35;17604 +36;18685 +37;18870 diff --git a/data/profiles/vertical/CMAQ_37layers_vertical_description_9layers.csv b/data/profiles/vertical/CMAQ_37layers_vertical_description_9layers.csv new file mode 100644 index 0000000000000000000000000000000000000000..7f83db510fd77b9be1a03e62fe695ba44bc22f4a --- /dev/null +++ b/data/profiles/vertical/CMAQ_37layers_vertical_description_9layers.csv @@ -0,0 +1,10 @@ +Ilayer;height_magl +1;40 +2;88 +3;145 +4;226 +5;324 +6;456 +7;623 +8;819 +9;18870 diff --git a/data/profiles/vertical/Vertical_profile.csv b/data/profiles/vertical/Vertical_profile.csv index 2518a5a9b804a8306341bb14c8f95a1b99462299..3add62a81a5815c0a324ddab31f662b70c75d6f8 100755 --- a/data/profiles/vertical/Vertical_profile.csv +++ b/data/profiles/vertical/Vertical_profile.csv @@ -4,4 +4,5 @@ V002;36,72,108,144,218,292,366,440,514,588,724,896,1050,1204,1358;0,0.18,0.47,0. V003;10,1000;0.6,0.4 V004;1000,9000;0,1 V005;9000,12000;0,1 -V006;36,72,108,144,218,292,366,440,514,588,724,896,1050,1204,1358;0,0,0.02,0.26,0.56,0.13,0.03,0,0,0,0,0,0,0,0 \ No newline at end of file +V006;36,72,108,144,218,292,366,440,514,588,724,896,1050,1204,1358;0,0,0.02,0.26,0.56,0.13,0.03,0,0,0,0,0,0,0,0 +V007;20,92,184,324,522,781,1106;0,0,0.41,0.57,0.02,0,0 \ No newline at end of file diff --git a/environment.yml b/environment.yml index bebcce37b2e46490fde22e27f8e5d1a8de3ebaf5..dc46ab7a228c1d91821ffda6d4e2e27ca5f5abc3 100755 --- a/environment.yml +++ b/environment.yml @@ -6,7 +6,7 @@ channels: - conda-forge dependencies: - - python = 2 + - python = 3 - numpy - netcdf4 >= 1.3.1 - python-cdo >= 1.3.6 diff --git a/hermesv3_gr/__init__.py b/hermesv3_gr/__init__.py index 92192eed4fc88eaee9fdbafbc8e2139c43ba00e2..8c0d5d5bb20beaaae2d684e1726eac58c9fec0ad 100755 --- a/hermesv3_gr/__init__.py +++ b/hermesv3_gr/__init__.py @@ -1 +1 @@ -__version__ = "1.0.4" +__version__ = "2.0.0" diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index d174e04ab52e2125c975864146702e3b3704c886..7775770f7c6802c6534e0a658724e0f76b17b193 100755 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -20,19 +20,26 @@ from configargparse import ArgParser import sys +import os +from shutil import rmtree +from mpi4py import MPI class Config(ArgParser): """ Initialization of the arguments that the parser can handle. """ - def __init__(self): + def __init__(self, comm=None): + if comm is None: + comm = MPI.COMM_WORLD + self.comm = comm + super(Config, self).__init__() self.options = self.read_options() def read_options(self): """ - Reads all the options from command line or from the configuration file. + Reads all the arguments from command line or from the configuration file. The value of an argument given by command line has high priority that the one that appear in the configuration file. @@ -61,6 +68,10 @@ class Config(ArgParser): type=str, choices=['hourly', 'daily', 'monthly', 'yearly']) p.add_argument('--output_timestep_num', required=True, help='Number of timesteps to simulate.', type=int) p.add_argument('--output_timestep_freq', required=True, help='Frequency between timesteps.', type=int) + p.add_argument('--first_time', required=False, default='False', type=str, + help='Indicates if you want to run it for first time (only create auxiliary files).') + p.add_argument('--erase_auxiliary_files', required=False, default='False', type=str, + help='Indicates if you want to start from scratch removing the auxiliary files already created.') p.add_argument('--output_model', required=True, help='Name of the output model.', choices=['MONARCH', 'CMAQ', 'WRF_CHEM']) @@ -69,24 +80,24 @@ class Config(ArgParser): p.add_argument('--domain_type', required=True, help='Type of domain to simulate.', choices=['global', 'lcc', 'rotated', 'mercator', 'regular']) - p.add_argument('--auxiliar_files_path', required=True, + p.add_argument('--auxiliary_files_path', required=True, help='Path to the directory where the necessary auxiliary files will be created if them are ' + 'not created yet.') p.add_argument('--vertical_description', required=True, help='Path to the file that contains the vertical description of the desired output.') - # Global options + # Global arguments p.add_argument('--inc_lat', required=False, help='Latitude resolution for a global domain.', type=float) p.add_argument('--inc_lon', required=False, help='Longitude resolution for a global domain.', type=float) - # Regular lat-lon options: + # Regular lat-lon arguments: p.add_argument('--lat_orig', required=False, help='Latitude of the corner of the first cell.', type=float) p.add_argument('--lon_orig', required=False, help='Longitude of the corner of the first cell.', type=float) p.add_argument('--n_lat', required=False, help='Number of latitude elements.', type=float) p.add_argument('--n_lon', required=False, help='Number of longitude elements.', type=float) - # Rotated options + # Rotated arguments p.add_argument('--centre_lat', required=False, help='Central geographic latitude of grid (non-rotated degrees). Corresponds to the TPH0D ' + 'parameter in NMMB-MONARCH.', type=float) @@ -106,7 +117,7 @@ class Config(ArgParser): help='Longitudinal grid resolution (rotated degrees). Corresponds to the DLMD parameter ' + 'in NMMB-MONARCH.', type=float) - # Lambert conformal conic options + # Lambert conformal conic arguments p.add_argument('--lat_1', required=False, help='Standard parallel 1 (in deg). Corresponds to the P_ALP parameter of the GRIDDESC file.', type=float) @@ -157,30 +168,38 @@ class Config(ArgParser): p.add_argument('--world_info', required=True, help='Path to the file that contains the world information like timezones, ISO codes, ...') - options = p.parse_args() - for item in vars(options): - is_str = False - exec ("is_str = str == type(options.{0})".format(item)) + arguments = p.parse_args() + for item in vars(arguments): + is_str = isinstance(arguments.__dict__[item], str) if is_str: - exec("options.{0} = options.{0}.replace('', options.input_dir)".format(item)) - exec("options.{0} = options.{0}.replace('', options.domain_type)".format(item)) - if options.domain_type == 'global' or options.domain_type == 'regular': - exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( - item, options.inc_lat, options.inc_lon)) - elif options.domain_type == 'rotated': - exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( - item, options.inc_rlat, options.inc_rlon)) - elif options.domain_type == 'lcc' or options.domain_type == 'mercator': - exec("options.{0} = options.{0}.replace('', '{1}_{2}')".format( - item, options.inc_x, options.inc_y)) - - options.start_date = self._parse_start_date(options.start_date) - options.end_date = self._parse_end_date(options.end_date, options.start_date) - - self.create_dir(options.output_dir) - self.create_dir(options.auxiliar_files_path) - - return options + arguments.__dict__[item] = arguments.__dict__[item].replace('', arguments.data_path) + arguments.__dict__[item] = arguments.__dict__[item].replace('', arguments.input_dir) + arguments.__dict__[item] = arguments.__dict__[item].replace('', arguments.domain_type) + if arguments.domain_type in ['global', 'regular']: + arguments.__dict__[item] = arguments.__dict__[item].replace('', '{1}_{2}'.format( + item, arguments.inc_lat, arguments.inc_lon)) + elif arguments.domain_type == 'rotated': + arguments.__dict__[item] = arguments.__dict__[item].replace('', '{1}_{2}'.format( + item, arguments.inc_rlat, arguments.inc_rlon)) + elif arguments.domain_type == 'lcc' or arguments.domain_type == 'mercator': + arguments.__dict__[item] = arguments.__dict__[item].replace('', '{1}_{2}'.format( + item, arguments.inc_x, arguments.inc_y)) + + arguments.start_date = self._parse_start_date(arguments.start_date) + arguments.end_date = self._parse_end_date(arguments.end_date, arguments.start_date) + + arguments.first_time = self._parse_bool(arguments.first_time) + arguments.erase_auxiliary_files = self._parse_bool(arguments.erase_auxiliary_files) + + self.create_dir(arguments.output_dir) + if arguments.erase_auxiliary_files: + if os.path.exists(arguments.auxiliary_files_path): + if self.comm.Get_rank() == 0: + rmtree(arguments.auxiliary_files_path) + self.comm.Barrier() + self.create_dir(arguments.auxiliary_files_path) + + return arguments def get_output_name(self, date): """ @@ -207,8 +226,7 @@ class Config(ArgParser): full_path = os.path.join(self.options.output_dir, file_name) return full_path - @staticmethod - def create_dir(path): + def create_dir(self, path): """ Create the given folder if it is not created yet. @@ -217,8 +235,7 @@ class Config(ArgParser): """ import os from mpi4py import MPI - icomm = MPI.COMM_WORLD - comm = icomm.Split(color=0, key=0) + comm = self.comm.Split(color=0, key=0) rank = comm.Get_rank() if rank == 0: @@ -247,10 +264,10 @@ class Config(ArgParser): elif str_bool in false_options: return False else: - print 'WARNING: Boolean value not contemplated use {0} for True values and {1} for the False ones'.format( + print('WARNING: Boolean value not contemplated use {0} for True values and {1} for the False ones'.format( true_options, false_options - ) - print '/t Using False as default' + )) + print('/t Using False as default') return False @staticmethod @@ -277,7 +294,7 @@ class Config(ArgParser): date = datetime.strptime(str_date, date_format) break except ValueError as e: - if e.message == 'day is out of range for month': + if str(e) == 'day is out of range for month': raise ValueError(e) if date is None: @@ -304,14 +321,14 @@ class Config(ArgParser): else: return self._parse_start_date(end_date) - def set_log_level(self): + def set_log_level(self, comm): """ Defines the log_level using the common script settings. """ - import settings - settings.define_global_vars(self.options.log_level) + from . import settings + settings.define_global_vars(self.options.log_level, comm) if __name__ == '__main__': config = Config() - print config.options + print(config.options) diff --git a/hermesv3_gr/config/settings.py b/hermesv3_gr/config/settings.py index 1b93cfa1182ba517531a30737ae119b8c81f5942..8a6966b3743a8dadd6fbe295d1852f74149ed8ac 100755 --- a/hermesv3_gr/config/settings.py +++ b/hermesv3_gr/config/settings.py @@ -20,6 +20,7 @@ import os import numpy as np +from functools import reduce global refresh_log @@ -27,7 +28,7 @@ global precision precision = np.float64 global writing_serial -writing_serial = False +writing_serial = True global compressed_netcdf compressed_netcdf = True @@ -45,7 +46,7 @@ global log_file global df_times -def define_global_vars(in_log_level): +def define_global_vars(in_log_level, mycomm): # TODO Documentation from mpi4py import MPI @@ -54,8 +55,12 @@ def define_global_vars(in_log_level): global rank global size - icomm = MPI.COMM_WORLD - comm = icomm.Split(color=0, key=0) + if mycomm is None: + icomm = MPI.COMM_WORLD + else: + icomm = mycomm + comm = icomm + # comm = icomm.Split(color=0, key=0) rank = comm.Get_rank() size = comm.Get_size() @@ -118,9 +123,9 @@ def finish_logs(output_dir, date): os.remove(times_path) df_merged = reduce(lambda left, right: pd.merge(left, right, on=['Class', 'Function'], how='outer'), data_frames) - df_merged['min'] = df_merged.loc[:, range(size)].min(axis=1) - df_merged['max'] = df_merged.loc[:, range(size)].max(axis=1) - df_merged['mean'] = df_merged.loc[:, range(size)].mean(axis=1) + df_merged['min'] = df_merged.loc[:, list(range(size))].min(axis=1) + df_merged['max'] = df_merged.loc[:, list(range(size))].max(axis=1) + df_merged['mean'] = df_merged.loc[:, list(range(size))].mean(axis=1) df_merged.to_csv(times_path) comm.Barrier() diff --git a/hermesv3_gr/hermes.py b/hermesv3_gr/hermes.py index 52d46cba127a6672cd2439c69a37ee53b23d0e22..1df2970d2e8cb67a377177a046b381952e077637 100755 --- a/hermesv3_gr/hermes.py +++ b/hermesv3_gr/hermes.py @@ -25,22 +25,24 @@ from hermesv3_gr.modules.emision_inventories.emission_inventory import EmissionI from hermesv3_gr.modules.vertical.vertical import VerticalDistribution from hermesv3_gr.tools.netcdf_tools import * -# import pyextrae.sequential as pyextrae global full_time -class Hermes(object): +class HermesGr(object): """ - Interface class for HERMESv3. + Interface class for HERMESv3_GR. """ - def __init__(self, config, new_date=None): + def __init__(self, config, new_date=None, comm=None): from hermesv3_gr.modules.grids.grid import Grid from hermesv3_gr.modules.temporal.temporal import TemporalDistribution from hermesv3_gr.modules.writing.writer import Writer global full_time st_time = full_time = timeit.default_timer() + if comm is None: + comm = MPI.COMM_WORLD + self.comm = comm self.config = config self.options = config.options @@ -48,7 +50,7 @@ class Hermes(object): if new_date is not None: self.options.start_date = new_date - config.set_log_level() + config.set_log_level(self.comm) settings.define_log_file(self.options.output_dir, self.options.start_date) settings.define_times_file() @@ -64,8 +66,9 @@ class Hermes(object): self.levels = VerticalDistribution.get_vertical_output_profile(self.options.vertical_description) self.grid = Grid.select_grid( + self.comm, self.options.domain_type, self.options.vertical_description, self.options.output_timestep_num, - self.options.auxiliar_files_path, self.options.inc_lat, self.options.inc_lon, self.options.lat_orig, + self.options.auxiliary_files_path, self.options.inc_lat, self.options.inc_lon, self.options.lat_orig, self.options.lon_orig, self.options.n_lat, self.options.n_lon, self.options.centre_lat, self.options.centre_lon, self.options.west_boundary, self.options.south_boundary, self.options.inc_rlat, self.options.inc_rlon, self.options.lat_1, self.options.lat_2, self.options.lon_0, self.options.lat_0, @@ -89,47 +92,53 @@ class Hermes(object): settings.write_time('HERMES', 'Init', timeit.default_timer() - st_time, level=1) # @profile - def main(self): + def main(self, return_emis=False): """ Main functionality of the model. """ from datetime import timedelta - st_time = timeit.default_timer() - settings.write_log('') - settings.write_log('***** Starting HERMESv3 *****') - num = 1 - for ei in self.emission_list: - settings.write_log('Processing emission inventory {0} for the sector {1} ({2}/{3}):'.format( - ei.inventory_name, ei.sector, num, len(self.emission_list))) - num += 1 - - ei.do_regrid() - - if ei.vertical is not None: - settings.write_log("\tCalculating vertical distribution.", level=2) - if ei.source_type == 'area': - ei.vertical_factors = ei.vertical.calculate_weights() - elif ei.source_type == 'point': - ei.calculate_altitudes(self.options.vertical_description) - ei.point_source_by_cell() - # To avoid use point source as area source when is going to apply vertical factors while writing. - ei.vertical = None - else: - settings.write_log('ERROR: Check the .err file to get more info.') - if settings.rank == 0: - raise AttributeError('Unrecognized emission source type {0}'.format(ei.source_type)) - sys.exit(1) - - if ei.temporal is not None: - ei.temporal_factors = ei.temporal.calculate_3d_temporal_factors() - if ei.speciation is not None: - ei.emissions = ei.speciation.do_speciation(ei.emissions) - - self.writer.write(self.emission_list) - - settings.write_log("***** HERMESv3 simulation finished successfully *****") - settings.write_time('HERMES', 'main', timeit.default_timer() - st_time) + if self.options.first_time: + # Stop run + settings.write_log('***** HERMESv3_BU First Time finished successfully *****') + else: + st_time = timeit.default_timer() + settings.write_log('') + settings.write_log('***** Starting HERMESv3_GR *****') + num = 1 + for ei in self.emission_list: + settings.write_log('Processing emission inventory {0} for the sector {1} ({2}/{3}):'.format( + ei.inventory_name, ei.sector, num, len(self.emission_list))) + num += 1 + + ei.do_regrid() + + if ei.vertical is not None: + settings.write_log("\tCalculating vertical distribution.", level=2) + if ei.source_type == 'area': + ei.vertical_factors = ei.vertical.calculate_weights() + elif ei.source_type == 'point': + ei.calculate_altitudes(self.options.vertical_description) + ei.point_source_by_cell() + # To avoid use point source as area source when is going to apply vertical factors while writing + ei.vertical = None + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('Unrecognized emission source type {0}'.format(ei.source_type)) + sys.exit(1) + + if ei.temporal is not None: + ei.temporal_factors = ei.temporal.calculate_3d_temporal_factors() + if ei.speciation is not None: + ei.emissions = ei.speciation.do_speciation(ei.emissions) + if return_emis: + return self.writer.get_emis(self.emission_list) + else: + self.writer.write(self.emission_list) + + settings.write_log("***** HERMESv3_GR simulation finished successfully *****") + settings.write_time('HERMES', 'main', timeit.default_timer() - st_time) settings.write_time('HERMES', 'TOTAL', timeit.default_timer() - full_time) settings.finish_logs(self.options.output_dir, self.options.start_date) @@ -140,9 +149,9 @@ class Hermes(object): def run(): - date = Hermes(Config()).main() + date = HermesGr(Config()).main() while date is not None: - date = Hermes(Config(), new_date=date).main() + date = HermesGr(Config(), new_date=date).main() sys.exit(0) diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index ed0a8377db316da63c710b91b92c96c6f2d83557..167042479a4a6f79db5b1c727b89eec59dfd8e35 100755 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -103,7 +103,7 @@ class EmissionInventory(object): # It will also create the WoldMasks necessaries self.masking = Masking( options.world_info, factors, regrid_mask, grid, - world_mask_file=os.path.join(os.path.dirname(options.auxiliar_files_path), + world_mask_file=os.path.join(os.path.dirname(options.auxiliary_files_path), '{0}_WorldMask.nc'.format(inventory_name))) self.input_pollutants = pollutants @@ -116,19 +116,19 @@ class EmissionInventory(object): if self.source_type == 'area': self.regrid = ConservativeRegrid( self.pollutant_dicts, - os.path.join(options.auxiliar_files_path, + os.path.join(options.auxiliary_files_path, "Weight_Matrix_{0}_{1}.nc".format(self.inventory_name, settings.size)), grid, masking=self.masking) - - # Creating Vertical Object - if p_vertical is not None: - self.vertical = VerticalDistribution( - self.get_profile(p_vertical), vertical_profile_path=options.p_vertical, - vertical_output_profile=vertical_output_profile) - else: - self.vertical = None - settings.write_log('\t\tNone vertical profile set.', level=2) - self.vertical_factors = None + if not options.first_time: + # Creating Vertical Object + if p_vertical is not None: + self.vertical = VerticalDistribution( + self.get_profile(p_vertical), vertical_profile_path=options.p_vertical, + vertical_output_profile=vertical_output_profile) + else: + self.vertical = None + settings.write_log('\t\tNone vertical profile set.', level=2) + self.vertical_factors = None # Creating Temporal Object # It will also create the necessaries timezone files @@ -136,18 +136,18 @@ class EmissionInventory(object): self.temporal = TemporalDistribution( current_date, options.output_timestep_type, options.output_timestep_num, options.output_timestep_freq, options.p_month, p_month, options.p_week, p_week, options.p_day, p_day, options.p_hour, p_hour, - options.world_info, options.auxiliar_files_path, grid) + options.world_info, options.auxiliary_files_path, grid, options.first_time) else: self.temporal = None settings.write_log('\t\tNone temporal profile set.', level=2) self.temporal_factors = None - - # Creating Speciation Object - if p_speciation is not None: - self.speciation = Speciation(p_speciation, options.p_speciation, options.molecular_weights) - else: - self.speciation = None - settings.write_log('\t\tNone speciation profile set.', level=2) + if not options.first_time: + # Creating Speciation Object + if p_speciation is not None: + self.speciation = Speciation(p_speciation, options.p_speciation, options.molecular_weights) + else: + self.speciation = None + settings.write_log('\t\tNone speciation profile set.', level=2) self.vertical_weights = None @@ -286,9 +286,9 @@ class EmissionInventory(object): """ import pandas as pd import re - from point_gfas_emission_inventory import PointGfasEmissionInventory - from gfas_emission_inventory import GfasEmissionInventory - from point_source_emission_inventory import PointSourceEmissionInventory + from .point_gfas_emission_inventory import PointGfasEmissionInventory + from .gfas_emission_inventory import GfasEmissionInventory + from .point_source_emission_inventory import PointSourceEmissionInventory st_time = timeit.default_timer() settings.write_log('Loading emissions') diff --git a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py index b237bc50fb6dcc1a102d00e3de45e81e5c3056d3..c06261561fa526b69de98473bb325f2aaec0914d 100755 --- a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py @@ -21,7 +21,7 @@ import os import timeit import hermesv3_gr.config.settings as settings -from emission_inventory import EmissionInventory +from .emission_inventory import EmissionInventory class GfasEmissionInventory(EmissionInventory): @@ -82,13 +82,13 @@ class GfasEmissionInventory(EmissionInventory): vertical_output_profile, reference_year=reference_year, factors=factors, regrid_mask=regrid_mask, p_vertical=None, p_month=p_month, p_week=p_week, p_day=p_day, p_hour=p_hour, p_speciation=p_speciation) + if not options.first_time: + self.approach = self.get_approach(p_vertical) + self.method = self.get_method(p_vertical) - self.approach = self.get_approach(p_vertical) - self.method = self.get_method(p_vertical) + self.altitude = self.get_altitude() - self.altitude = self.get_altitude() - - self.vertical = GfasVerticalDistribution(vertical_output_profile, self.approach, self.get_altitude()) + self.vertical = GfasVerticalDistribution(vertical_output_profile, self.approach, self.get_altitude()) settings.write_time('GFAS_EmissionInventory', 'Init', timeit.default_timer() - st_time, level=3) @@ -132,7 +132,7 @@ class GfasEmissionInventory(EmissionInventory): else: alt_var = None - print "ERROR: Only 'sovief' and 'prm' methods are accepted." + print("ERROR: Only 'sovief' and 'prm' methods are accepted.") [alt] = extract_vars(self.get_input_path(), [alt_var]) @@ -211,7 +211,7 @@ class GfasEmissionInventory(EmissionInventory): st_time = timeit.default_timer() settings.write_log("\tRegridding", level=2) - for i in xrange(len(self.emissions)): + for i in range(len(self.emissions)): self.emissions[i]["data"] = self.do_vertical_allocation(self.emissions[i]["data"]) regridded_emissions = self.regrid.start_regridding(gfas=True, vertical=self.vertical) diff --git a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py index 0eb27d300a8f419d2e22cf9153ccb1f528b966db..78167bba64179b2432ccf1ea76004e749854d5c4 100755 --- a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py @@ -23,7 +23,7 @@ import timeit import warnings import hermesv3_gr.config.settings as settings -from emission_inventory import EmissionInventory +from .emission_inventory import EmissionInventory class PointGfasEmissionInventory(EmissionInventory): @@ -148,7 +148,7 @@ class PointGfasEmissionInventory(EmissionInventory): else: alt_var = None - print "ERROR: Only 'sovief' and 'prm' methods are accepted." + print("ERROR: Only 'sovief' and 'prm' methods are accepted.") [alt] = extract_vars(self.get_input_path(), [alt_var]) @@ -213,7 +213,7 @@ class PointGfasEmissionInventory(EmissionInventory): :return: Emissions already allocated on the top altitude of each fire. :rtype: numpy.array """ - print 'do_vertical_allocation' + print('do_vertical_allocation') sys.exit() st_time = timeit.default_timer() @@ -336,7 +336,7 @@ class PointGfasEmissionInventory(EmissionInventory): # sys.exit() gdf = gpd.sjoin(gdf.to_crs(grid_shp.crs), grid_shp, how='inner') - print gdf + print(gdf) gdf = np.array_split(gdf, settings.size) else: gdf = None @@ -346,7 +346,7 @@ class PointGfasEmissionInventory(EmissionInventory): for num, pollutant in enumerate(self.pollutant_dicts): settings.write_log('\t\tPollutant {0} ({1}/{2})'.format( pollutant['name'], num + 1, len(self.pollutant_dicts)), level=3) - print ('\t\tPollutant {0} ({1}/{2})'.format(pollutant['name'], num + 1, len(self.pollutant_dicts))) + print(('\t\tPollutant {0} ({1}/{2})'.format(pollutant['name'], num + 1, len(self.pollutant_dicts)))) aux = netcdf.variables[pollutant['name']][:].flatten()[gdf['src_index']] gdf[pollutant['name']] = aux * gdf['src_area'] # print 'masa {0}: {1} '.format(pollutant['name'], gdf[pollutant['name']].sum()) @@ -357,7 +357,7 @@ class PointGfasEmissionInventory(EmissionInventory): netcdf.close() settings.write_time('PointGfasEmissionInventory', 'do_regrid', timeit.default_timer() - st_time, level=2) - print 'regrid done' + print('regrid done') del gdf['src_index'], gdf['index_right'] self.emissions = gdf @@ -393,7 +393,7 @@ class PointGfasEmissionInventory(EmissionInventory): warnings.warn('WARNING: One or more fires have an altitude of fire emission injection higher than the top' + ' layer of the model defined in the {0} file'.format(vertical_description_path)) - print self.emissions.loc[~self.emissions['altitude'].isna()] + print(self.emissions.loc[~self.emissions['altitude'].isna()]) del self.emissions['altitude'] self.emissions = self.emissions.groupby(['FID', 'layer']).sum() diff --git a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py index 60e32b82c19136c630a701e7fed1c6c99ab76541..791f60fa09dc5b0bbf5e2a8442223860d8be5c23 100755 --- a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py @@ -20,7 +20,7 @@ import timeit import hermesv3_gr.config.settings as settings -from emission_inventory import EmissionInventory +from .emission_inventory import EmissionInventory class PointSourceEmissionInventory(EmissionInventory): diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index ce1133bcb4df82e9b8b18004adcc6ef0e07d7ccc..a66f21f1ab7660272a2136cdf422a8775ec72c1f 100755 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -40,10 +40,10 @@ class Grid(object): :type temporal_path: str """ - def __init__(self, grid_type, vertical_description_path, temporal_path): + def __init__(self, grid_type, vertical_description_path, temporal_path, comm): st_time = timeit.default_timer() # settings.write_log('Creating Grid...', level=1) - + self.comm = comm # Defining class atributes self.procs_array = None self.nrows = 0 @@ -57,16 +57,12 @@ class Grid(object): self.boundary_longitudes = None self.cell_area = None - if settings.rank == 0: - if not os.path.exists(os.path.join(temporal_path)): - os.makedirs(os.path.join(temporal_path)) - settings.comm.Barrier() self.coords_netcdf_file = os.path.join(temporal_path, 'temporal_coords.nc') self.temporal_path = temporal_path self.shapefile_path = None - self.esmf_grid = None + # self.esmf_grid = None self.x_lower_bound = None self.x_upper_bound = None self.y_lower_bound = None @@ -78,6 +74,27 @@ class Grid(object): settings.write_time('Grid', 'Init', timeit.default_timer() - st_time, level=1) + def calculate_bounds(self): + x_min = 0 + y_min = 0 + if len(self.center_latitudes.shape) == 1: + x_max = self.center_latitudes.shape[0] + y_max = self.center_longitudes.shape[0] + else: + x_max = self.center_latitudes.shape[0] + y_max = self.center_longitudes.shape[1] + y_len = y_max // self.comm.Get_size() + exceeded_ranks = y_max % self.comm.Get_size() + self.x_lower_bound = x_min + self.x_upper_bound = x_max + if self.comm.Get_rank() < exceeded_ranks: + y_len = y_len + 1 + self.y_lower_bound = self.comm.Get_rank() * y_len + self.y_upper_bound = (self.comm.Get_rank() + 1) * y_len + else: + self.y_lower_bound = (self.comm.Get_rank() * y_len) + exceeded_ranks + self.y_upper_bound = ((self.comm.Get_rank() + 1) * y_len) + exceeded_ranks + @staticmethod def create_esmf_grid_from_file(file_name, sphere=True): import ESMF @@ -94,7 +111,7 @@ class Grid(object): return grid @staticmethod - def select_grid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, + def select_grid(comm, grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, lon_orig, n_lat, n_lon, centre_lat, centre_lon, west_boundary, south_boundary, inc_rlat, inc_rlon, lat_1, lat_2, lon_0, lat_0, nx, ny, inc_x, inc_y, x_0, y_0, lat_ts): # TODO describe better the rotated parameters @@ -180,26 +197,27 @@ class Grid(object): # Creating a different object depending on the grid type if grid_type == 'global': from hermesv3_gr.modules.grids.grid_global import GlobalGrid - grid = GlobalGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon) + grid = GlobalGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, + comm=comm) elif grid_type == 'regular': from hermesv3_gr.modules.grids.grid_latlon import LatLonGrid grid = LatLonGrid(grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, - lat_orig, lon_orig, n_lat, n_lon) + lat_orig, lon_orig, n_lat, n_lon, comm=comm) elif grid_type == 'rotated': from hermesv3_gr.modules.grids.grid_rotated import RotatedGrid grid = RotatedGrid(grid_type, vertical_description_path, timestep_num, temporal_path, - centre_lat, centre_lon, west_boundary, south_boundary, inc_rlat, inc_rlon) + centre_lat, centre_lon, west_boundary, south_boundary, inc_rlat, inc_rlon, comm=comm) elif grid_type == 'lcc': from hermesv3_gr.modules.grids.grid_lcc import LccGrid grid = LccGrid(grid_type, vertical_description_path, timestep_num, temporal_path, lat_1, lat_2, lon_0, - lat_0, nx, ny, inc_x, inc_y, x_0, y_0) + lat_0, nx, ny, inc_x, inc_y, x_0, y_0, comm=comm) elif grid_type == 'mercator': from hermesv3_gr.modules.grids.grid_mercator import MercatorGrid grid = MercatorGrid(grid_type, vertical_description_path, timestep_num, temporal_path, lat_ts, lon_0, - nx, ny, inc_x, inc_y, x_0, y_0) + nx, ny, inc_x, inc_y, x_0, y_0, comm=comm) else: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -275,19 +293,11 @@ class Grid(object): :rtype: numpy.array """ from cdo import Cdo - from netCDF4 import Dataset - st_time = timeit.default_timer() settings.write_log('\t\tGetting cell area from {0}'.format(self.coords_netcdf_file), level=3) - # Initialises the CDO cdo = Cdo() - # Create a temporal file 's' with the cell area - s = cdo.gridarea(input=self.coords_netcdf_file) - # Get the cell area of the temporal file - nc_aux = Dataset(s, mode='r') - cell_area = nc_aux.variables['cell_area'][:] - nc_aux.close() + cell_area = cdo.gridarea(input=self.coords_netcdf_file, returnArray='cell_area') settings.write_time('Grid', 'get_cell_area', timeit.default_timer() - st_time, level=3) @@ -389,8 +399,9 @@ class Grid(object): st_time = timeit.default_timer() settings.write_log('\t\tGetting 2D coordinates from ESMPy Grid', level=3) - lat = self.esmf_grid.get_coords(1, ESMF.StaggerLoc.CENTER).T - lon = self.esmf_grid.get_coords(0, ESMF.StaggerLoc.CENTER).T + esmf_grid = self.create_esmf_grid_from_file(self.coords_netcdf_file) + lat = esmf_grid.get_coords(1, ESMF.StaggerLoc.CENTER).T + lon = esmf_grid.get_coords(0, ESMF.StaggerLoc.CENTER).T settings.write_time('Grid', 'get_coordinates_2d', timeit.default_timer() - st_time, level=3) @@ -427,21 +438,13 @@ class Grid(object): settings.write_log('\t\tGrid shapefile not done. Lets try to create it.', level=3) # Create Shapefile - # Use the meters coordiantes to create the shapefile - y = self.boundary_latitudes x = self.boundary_longitudes - # sys.exit() if self.grid_type in ['global', 'regular']: x = x.reshape((x.shape[1], x.shape[2])) y = y.reshape((y.shape[1], y.shape[2])) - # x_aux = np.empty((x.shape[0], y.shape[0], 4)) - # x_aux[:, :, 0] = x[:, np.newaxis, 0] - # x_aux[:, :, 1] = x[:, np.newaxis, 1] - # x_aux[:, :, 2] = x[:, np.newaxis, 1] - # x_aux[:, :, 3] = x[:, np.newaxis, 0] aux_shape = (y.shape[0], x.shape[0], 4) x_aux = np.empty(aux_shape) x_aux[:, :, 0] = x[np.newaxis, :, 0] @@ -452,12 +455,6 @@ class Grid(object): x = x_aux del x_aux - # y_aux = np.empty((x.shape[0], y.shape[0], 4)) - # y_aux[:, :, 0] = y[np.newaxis, :, 0] - # y_aux[:, :, 1] = y[np.newaxis, :, 0] - # y_aux[:, :, 2] = y[np.newaxis, :, 1] - # y_aux[:, :, 3] = y[np.newaxis, :, 1] - y_aux = np.empty(aux_shape) y_aux[:, :, 0] = y[:, np.newaxis, 0] y_aux[:, :, 1] = y[:, np.newaxis, 0] @@ -467,8 +464,6 @@ class Grid(object): y = y_aux del y_aux - # exit() - if not full_grid: y = y[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound, :] x = x[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound, :] @@ -476,53 +471,29 @@ class Grid(object): aux_b_lats = y.reshape((y.shape[0] * y.shape[1], y.shape[2])) aux_b_lons = x.reshape((x.shape[0] * x.shape[1], x.shape[2])) - # The regular lat-lon projection has only 2 (laterals) points for each cell instead of 4 (corners) - # if aux_b_lats.shape[1] == 2: - # aux_b = np.empty((aux_b_lats.shape[0], 4)) - # aux_b[:, 0] = aux_b_lats[:, 0] - # aux_b[:, 1] = aux_b_lats[:, 0] - # aux_b[:, 2] = aux_b_lats[:, 1] - # aux_b[:, 3] = aux_b_lats[:, 1] - # aux_b_lats = aux_b - # - # if aux_b_lons.shape[1] == 2: - # aux_b = np.empty((aux_b_lons.shape[0], 4)) - # aux_b[:, 0] = aux_b_lons[:, 0] - # aux_b[:, 1] = aux_b_lons[:, 1] - # aux_b[:, 2] = aux_b_lons[:, 1] - # aux_b[:, 3] = aux_b_lons[:, 0] - # aux_b_lons = aux_b - # Create one dataframe with 8 columns, 4 points with two coordinates each one df_lats = pd.DataFrame(aux_b_lats, columns=['b_lat_1', 'b_lat_2', 'b_lat_3', 'b_lat_4']) df_lons = pd.DataFrame(aux_b_lons, columns=['b_lon_1', 'b_lon_2', 'b_lon_3', 'b_lon_4']) df = pd.concat([df_lats, df_lons], axis=1) - # Substituate 8 columns by 4 with the two coordinates - df['p1'] = zip(df.b_lon_1, df.b_lat_1) + # Substitute 8 columns by 4 with the two coordinates + df['p1'] = list(zip(df.b_lon_1, df.b_lat_1)) del df['b_lat_1'], df['b_lon_1'] - df['p2'] = zip(df.b_lon_2, df.b_lat_2) + df['p2'] = list(zip(df.b_lon_2, df.b_lat_2)) del df['b_lat_2'], df['b_lon_2'] - df['p3'] = zip(df.b_lon_3, df.b_lat_3) + df['p3'] = list(zip(df.b_lon_3, df.b_lat_3)) del df['b_lat_3'], df['b_lon_3'] - df['p4'] = zip(df.b_lon_4, df.b_lat_4) + df['p4'] = list(zip(df.b_lon_4, df.b_lat_4)) del df['b_lat_4'], df['b_lon_4'] # Make a list of list of tuples # [[(point_1.1), (point_1.2), (point_1.3), (point_1.4)], # [(point_2.1), (point_2.2), (point_2.3), (point_2.4)], ...] - list_points = df.as_matrix() + list_points = df.values del df['p1'], df['p2'], df['p3'], df['p4'] # List of polygons from the list of points geometry = [Polygon(list(points)) for points in list_points] - # geometry = [] - # for point in list_points: - # print point - # geometry.append(Polygon(list(point))) - # print geometry[0] - # sys.exit() - # print len(geometry), len(df), gdf = gpd.GeoDataFrame(df, crs={'init': 'epsg:4326'}, geometry=geometry) gdf = gdf.to_crs(self.crs) diff --git a/hermesv3_gr/modules/grids/grid_global.py b/hermesv3_gr/modules/grids/grid_global.py index ee0b1d17464f9cb4dc2b9e3744bc04935e7e0878..2f63c3cb213105ac43e778f5f1c04c6162e7af26 100755 --- a/hermesv3_gr/modules/grids/grid_global.py +++ b/hermesv3_gr/modules/grids/grid_global.py @@ -23,7 +23,7 @@ import sys import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class GlobalGrid(Grid): @@ -54,14 +54,13 @@ class GlobalGrid(Grid): """ def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, - center_longitude=float(0)): - import ESMF + center_longitude=float(0), comm=None): st_time = timeit.default_timer() settings.write_log('\tCreating Global grid.', level=2) # Initialize the class using parent - super(GlobalGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + super(GlobalGrid, self).__init__(grid_type, vertical_description_path, temporal_path, comm) self.center_lat = float(0) self.center_lon = center_longitude @@ -76,12 +75,7 @@ class GlobalGrid(Grid): super(GlobalGrid, self).write_coords_netcdf() settings.comm.Barrier() - self.esmf_grid = super(GlobalGrid, self).create_esmf_grid_from_file(self.coords_netcdf_file) - - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/grids/grid_latlon.py b/hermesv3_gr/modules/grids/grid_latlon.py index f66f80e1f963c8a2522b0cc4219a114365514f4c..df0a676794c2367748905522dd2f817e0334cfa1 100755 --- a/hermesv3_gr/modules/grids/grid_latlon.py +++ b/hermesv3_gr/modules/grids/grid_latlon.py @@ -23,7 +23,7 @@ import sys import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class LatLonGrid(Grid): @@ -63,14 +63,13 @@ class LatLonGrid(Grid): """ def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, inc_lat, inc_lon, lat_orig, - lon_orig, n_lat, n_lon): - import ESMF + lon_orig, n_lat, n_lon, comm=None): st_time = timeit.default_timer() settings.write_log('\tCreating Regional regular lat-lon grid.', level=2) # Initialize the class using parent - super(LatLonGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + super(LatLonGrid, self).__init__(grid_type, vertical_description_path, temporal_path, comm) self.inc_lat = inc_lat self.inc_lon = inc_lon @@ -87,12 +86,7 @@ class LatLonGrid(Grid): self.write_coords_netcdf() settings.comm.Barrier() - self.esmf_grid = self.create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) - - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/grids/grid_lcc.py b/hermesv3_gr/modules/grids/grid_lcc.py index c0b4d4e3266d647d98df229ab480c4f931022dbb..42aaebcc32e3829f60ad72d693232624e0cf7e99 100755 --- a/hermesv3_gr/modules/grids/grid_lcc.py +++ b/hermesv3_gr/modules/grids/grid_lcc.py @@ -22,7 +22,7 @@ import os import sys import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class LccGrid(Grid): @@ -77,13 +77,12 @@ class LccGrid(Grid): """ def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, lat_1, lat_2, lon_0, lat_0, - nx, ny, inc_x, inc_y, x_0, y_0, earth_radius=6370000.000): - import ESMF + nx, ny, inc_x, inc_y, x_0, y_0, earth_radius=6370000.000, comm=None): st_time = timeit.default_timer() settings.write_log('\tCreating Lambert Conformal Conic (LCC) grid.', level=2) # Initialises with parent class - super(LccGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + super(LccGrid, self).__init__(grid_type, vertical_description_path, temporal_path, comm) # Setting parameters self.lat_1 = lat_1 @@ -113,12 +112,7 @@ class LccGrid(Grid): self.write_coords_netcdf() settings.comm.Barrier() - self.esmf_grid = super(LccGrid, self).create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) - # - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/grids/grid_mercator.py b/hermesv3_gr/modules/grids/grid_mercator.py index c715b8e4ba8d02b2362c6a7c09ff2bba03f4f453..d18c32de25fe21f36737c24a6285149216e185ac 100755 --- a/hermesv3_gr/modules/grids/grid_mercator.py +++ b/hermesv3_gr/modules/grids/grid_mercator.py @@ -22,7 +22,7 @@ import os import sys import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class MercatorGrid(Grid): @@ -69,7 +69,6 @@ class MercatorGrid(Grid): def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, lat_ts, lon_0, nx, ny, inc_x, inc_y, x_0, y_0, earth_radius=6370000.000): - import ESMF st_time = timeit.default_timer() settings.write_log('\tCreating Mercator grid.', level=2) @@ -101,12 +100,7 @@ class MercatorGrid(Grid): self.write_coords_netcdf() settings.comm.Barrier() - self.esmf_grid = super(MercatorGrid, self).create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) - # - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/grids/grid_rotated.py b/hermesv3_gr/modules/grids/grid_rotated.py index eabf8e7b72cdedf51e7c6759b5da7ba6cf98a92d..8c9903612a1ef51efbff669a46d1f9edb0c159b4 100755 --- a/hermesv3_gr/modules/grids/grid_rotated.py +++ b/hermesv3_gr/modules/grids/grid_rotated.py @@ -22,7 +22,7 @@ import sys import os import timeit import hermesv3_gr.config.settings as settings -from grid import Grid +from .grid import Grid class RotatedGrid(Grid): @@ -40,14 +40,12 @@ class RotatedGrid(Grid): """ def __init__(self, grid_type, vertical_description_path, timestep_num, temporal_path, centre_lat, centre_lon, - west_boundary, south_boundary, inc_rlat, inc_rlon): - import ESMF - + west_boundary, south_boundary, inc_rlat, inc_rlon, comm=None): st_time = timeit.default_timer() settings.write_log('\tCreating Rotated grid.', level=2) # Initialises with parent class - super(RotatedGrid, self).__init__(grid_type, vertical_description_path, temporal_path) + super(RotatedGrid, self).__init__(grid_type, vertical_description_path, temporal_path, comm) # Setting parameters self.new_pole_longitude_degrees = -180 + centre_lon @@ -71,18 +69,10 @@ class RotatedGrid(Grid): if not os.path.exists(self.coords_netcdf_file): if settings.rank == 0: - # super(RotatedGrid, self).write_coords_netcdf() self.write_coords_netcdf() settings.comm.Barrier() - # self.write_coords_netcdf() - - self.esmf_grid = super(RotatedGrid, self).create_esmf_grid_from_file(self.coords_netcdf_file, sphere=False) - - self.x_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][1] - self.x_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][1] - self.y_lower_bound = self.esmf_grid.lower_bounds[ESMF.StaggerLoc.CENTER][0] - self.y_upper_bound = self.esmf_grid.upper_bounds[ESMF.StaggerLoc.CENTER][0] + self.calculate_bounds() self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) diff --git a/hermesv3_gr/modules/masking/masking.py b/hermesv3_gr/modules/masking/masking.py index 36b1c93f85a709e9e5e7c771e9c39b9a3248ae09..90670cae1cc825744927c0805bc3b6417b5d3acb 100755 --- a/hermesv3_gr/modules/masking/masking.py +++ b/hermesv3_gr/modules/masking/masking.py @@ -85,29 +85,6 @@ class Masking(object): settings.write_time('Masking', 'get_country_codes', timeit.default_timer() - st_time, level=3) return countries_dict - @staticmethod - def partlst(lst, num): - """ - Split a Array in N balanced arrays. - - :param lst: Array to split - :type lst: numpy.array - - :param num: Number of mini arrays. - :type num: int - - :return: Array - :type: numpy.array - """ - import itertools - # Partition @lst in @n balanced parts, in given order - parts, rest = divmod(len(lst), num) - lstiter = iter(lst) - for j in xrange(num): - plen = len(lst) / num + (1 if rest > 0 else 0) - rest -= 1 - yield list(itertools.islice(lstiter, plen)) - def create_country_iso(self, in_nc): import numpy as np from hermesv3_gr.tools.netcdf_tools import extract_vars @@ -123,9 +100,10 @@ class Masking(object): dst_var = [] num = 0 - points = np.array(zip(lat.flatten(), lon.flatten())) + aux_list = list(zip(lat.flatten(), lon.flatten())) + points = np.array(aux_list[:]) - points_list = list(self.partlst(points, settings.size)) + points_list = np.array_split(points, settings.size) for lat_aux, lon_aux in points_list[settings.rank]: num += 1 @@ -289,7 +267,7 @@ class Masking(object): values = values['data'] mask = np.ones(values.shape) - for code, factor in self.factors_mask_values.iteritems(): + for code, factor in self.factors_mask_values.items(): mask[values == code] = factor settings.write_time('Masking', 'custom_scale_mask', timeit.default_timer() - st_time, level=3) diff --git a/hermesv3_gr/modules/regrid/regrid_conservative.py b/hermesv3_gr/modules/regrid/regrid_conservative.py index f2de4d1ed0b96bac1bd6ea35928daf0d7be1e7bf..74c219e8ecbb9ec7a7f883e07ba1d69e2b25af55 100755 --- a/hermesv3_gr/modules/regrid/regrid_conservative.py +++ b/hermesv3_gr/modules/regrid/regrid_conservative.py @@ -23,7 +23,7 @@ import timeit import ESMF import hermesv3_gr.config.settings as settings -from regrid import Regrid +from .regrid import Regrid class ConservativeRegrid(Regrid): @@ -48,7 +48,7 @@ class ConservativeRegrid(Regrid): src_field.read(filename=self.pollutant_dicts[0]['path'], variable=self.pollutant_dicts[0]['name'], timeslice=0) - dst_grid = self.grid.esmf_grid + dst_grid = self.grid.create_esmf_grid_from_file(self.grid.coords_netcdf_file) dst_field = ESMF.Field(dst_grid, name='my outut field') ESMF.Regrid(src_field, dst_field, filename=self.weight_matrix_file, regrid_method=ESMF.RegridMethod.CONSERVE,) @@ -151,18 +151,18 @@ class ConservativeRegrid(Regrid): if os.path.exists(self.weight_matrix_file): pre_size = 0 post_size = 1 - print "I'm {0}".format(settings.rank), 'Writing Weight Matrix {0}'.format(self.weight_matrix_file) + print("I'm {0}".format(settings.rank), 'Writing Weight Matrix {0}'.format(self.weight_matrix_file)) # find = True while pre_size != post_size: - print "I'm {0}".format(settings.rank), pre_size, post_size + print("I'm {0}".format(settings.rank), pre_size, post_size) pre_size = post_size post_size = os.path.getsize(self.weight_matrix_file) time.sleep(1) find = True - print "I'm {0}".format(settings.rank), 'FINISHED' + print("I'm {0}".format(settings.rank), 'FINISHED') else: time.sleep(5) - print "I'm {0}".format(settings.rank), 'Waiting Weight Matrix' + print("I'm {0}".format(settings.rank), 'Waiting Weight Matrix') def apply_weights(self, values): """ diff --git a/hermesv3_gr/modules/speciation/speciation.py b/hermesv3_gr/modules/speciation/speciation.py index 6084c44978eb0db59489bd7fbf3862592a61fdd9..713a4848bda6b6ca9ef4d7d5f4bf5aaf4cd0ffbe 100755 --- a/hermesv3_gr/modules/speciation/speciation.py +++ b/hermesv3_gr/modules/speciation/speciation.py @@ -81,7 +81,7 @@ class Speciation(object): long_name_dict = df.loc[df[df.ID == 'short_description'].index[0]].to_dict() long_name_dict.pop('ID', None) profile_list = [] - for key in formulas_dict.iterkeys(): + for key in formulas_dict.keys(): profile_list.append({ 'name': key, 'formula': formulas_dict[key], @@ -146,7 +146,7 @@ class Speciation(object): raise KeyError('ERROR: {0} pollutant is not in the molecular weights file {1} .'.format( emission['name'], self.molecular_weights_path)) sys.exit(1) - exec ("{0} = np.array(emission['data'], dtype=settings.precision)".format(emission['name'])) + exec("{0} = np.array(emission['data'], dtype=settings.precision)".format(emission['name'])) emission['units'] = '' input_pollutants.append(emission['name']) @@ -182,7 +182,7 @@ class Speciation(object): raise AttributeError( "Error in speciation profile {0}: ".format(self.id) + "The output specie {0} cannot be calculated ".format(pollutant['name']) + - "with the expression {0} because{1}".format(formula, e.message)) + "with the expression {0} because{1}".format(formula, str(e))) else: sys.exit(1) speciated_emissions.append(dict_aux) diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 858edd5b76399fa965e7bfbffac3c82b36947115..e1630aadbda8a4ada642ade11eb196687b09a944 100755 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -70,7 +70,7 @@ class TemporalDistribution(object): """ def __init__(self, starting_date, timestep_type, timestep_num, timestep_freq, monthly_profile_path, monthly_profile_id, weekly_profile_path, weekly_profile_id, daily_profile_path, daily_profile_id, - hourly_profile_path, hourly_profile_id, world_info_path, auxiliar_files_dir, grid): + hourly_profile_path, hourly_profile_id, world_info_path, auxiliar_files_dir, grid, first_time=False): from timezonefinder import TimezoneFinder import pandas as pd @@ -80,40 +80,34 @@ class TemporalDistribution(object): self.grid = grid - # self.starting_date = starting_date + if not first_time: + self.date_array = self.caclulate_date_array(starting_date, timestep_type, timestep_num, timestep_freq) - self.date_array = self.caclulate_date_array(starting_date, timestep_type, timestep_num, timestep_freq) + if timestep_type in ['daily', 'hourly']: + self.daily_profile = self.get_daily_profile(daily_profile_id, daily_profile_path) + else: + self.daily_profile = None - # self.timestep_type = timestep_type - # self.timestep_num = timestep_num - # self.timestep_freq = timestep_freq - # - # self.ending_date = self.calculate_ending_date() - if timestep_type in ['daily', 'hourly']: - self.daily_profile = self.get_daily_profile(daily_profile_id, daily_profile_path) - else: - self.daily_profile = None + if self.daily_profile is None: + if timestep_type in ['monthly', 'daily', 'hourly']: + self.monthly_profile = self.get_monthly_profile(monthly_profile_id, monthly_profile_path) + else: + self.monthly_profile = None - if self.daily_profile is None: - if timestep_type in ['monthly', 'daily', 'hourly']: - self.monthly_profile = self.get_monthly_profile(monthly_profile_id, monthly_profile_path) + if timestep_type in ['daily', 'hourly']: + self.weekly_profile = self.get_weekly_profile(weekly_profile_id, weekly_profile_path) + else: + self.weekly_profile = None else: self.monthly_profile = None - - if timestep_type in ['daily', 'hourly']: - self.weekly_profile = self.get_weekly_profile(weekly_profile_id, weekly_profile_path) - else: self.weekly_profile = None - else: - self.monthly_profile = None - self.weekly_profile = None - if monthly_profile_id is not None or weekly_profile_id is not None: - warn('WARNING: Daily profile is set. Monthly or Weekly profiles will be ignored.') + if monthly_profile_id is not None or weekly_profile_id is not None: + warn('WARNING: Daily profile is set. Monthly or Weekly profiles will be ignored.') - if timestep_type in ['hourly']: - self.hourly_profile = self.get_hourly_profile(hourly_profile_id, hourly_profile_path) - else: - self.hourly_profile = None + if timestep_type in ['hourly']: + self.hourly_profile = self.get_hourly_profile(hourly_profile_id, hourly_profile_path) + else: + self.hourly_profile = None self.world_info = world_info_path self.netcdf_timezones = os.path.join(auxiliar_files_dir, 'timezones.nc') @@ -201,7 +195,7 @@ class TemporalDistribution(object): if self.grid.grid_type == 'rotated': lat_name = 'rlat' lon_name = 'rlon' - elif self.grid.grid_type.isin(['mercator', 'lcc']): + elif self.grid.grid_type in ['mercator', 'lcc']: lat_name = 'x' lon_name = 'y' else: @@ -359,12 +353,12 @@ class TemporalDistribution(object): weekdays_count = self.calculate_weekdays(date) if isinstance(profile, dict): factor = self.calculate_weekday_factor(profile, weekdays_count) - for dict_key in profile.iterkeys(): + for dict_key in profile.keys(): profile[dict_key] = profile[dict_key] + factor elif isinstance(profile, np.ndarray): # Gridded factor = self.calculate_weekday_gridded_factor(profile, weekdays_count) - for weekday in xrange(7): + for weekday in range(7): profile[weekday, :] = profile[weekday, :] + factor else: settings.write_log('ERROR: Check the .err file to get more info.') @@ -460,7 +454,7 @@ class TemporalDistribution(object): profile = self.parse_hourly_profile_id(profile_id) if profile is not None: - for profile_type, profile_id_aux in profile.iteritems(): + for profile_type, profile_id_aux in profile.items(): # Gridded monthly profile if os.path.exists(profile_id_aux): profile_aux = self.get_gridded_temporal_profile('Fhour', profile_id_aux) @@ -564,7 +558,7 @@ class TemporalDistribution(object): 'Asia/Famagusta': 'Asia/Nicosia', } - if timezone in tz_dict.iterkeys(): + if timezone in iter(tz_dict.keys()): timezone = tz_dict[timezone] return timezone @@ -625,7 +619,7 @@ class TemporalDistribution(object): dst_var = [] num = 0 - points = zip(lat.flatten(), lon.flatten()) + points = list(zip(lat.flatten(), lon.flatten())) for lat_aux, lon_aux in points: num += 1 @@ -692,12 +686,13 @@ class TemporalDistribution(object): nc_in.close() tz_list = np.chararray(timezones.shape, itemsize=32) - for id_aux in xrange(timezones.min(), timezones.max() + 1): + for id_aux in range(timezones.min(), timezones.max() + 1): try: timezone = self.get_tz_from_id(id_aux) tz_list[timezones == id_aux] = timezone except IndexError: pass + settings.write_time('TemporalDistribution', 'calculate_timezones', timeit.default_timer() - st_time, level=3) return tz_list @@ -706,8 +701,8 @@ class TemporalDistribution(object): """ Calculate the temporal factor to correct the input data of the given date for each cell. - :param date: Date of the current timestep. - :type date: datetime.datetime + :param date_aux: Date of the current timestep. + :type date_aux: datetime.datetime :return: 2D array with the factors to correct the input data to the date of this timestep. :rtype: numpy.array @@ -720,14 +715,13 @@ class TemporalDistribution(object): df = pd.DataFrame(self.timezones_array.flatten(), columns=['tz']) - df['utc'] = pd.to_datetime(date_aux) + df['local'] = pd.to_datetime(date_aux, utc=True) try: - df['local'] = df.groupby('tz')['utc'].apply( - lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert(x.name).dt.tz_localize(None)) + df['local'] = df.groupby('tz')['local'].apply( + lambda x: x.dt.tz_convert(x.name.decode("utf-8")).dt.tz_localize(None)) except pytz.exceptions.UnknownTimeZoneError: df['local'] = df.groupby('tz')['utc'].apply( - lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert( - self.parse_tz(x.name)).dt.tz_localize(None)) + lambda x: x.dt.tz_convert(self.parse_tz(x.name.decode("utf-8"))).dt.tz_localize(None)) # ===== HOURLY PROFILES ===== df['weekday'] = df['local'].dt.weekday @@ -739,7 +733,7 @@ class TemporalDistribution(object): if isinstance(weekday_profile, dict): df['weekday_factor'] = df['hour'].map(weekday_profile) else: - for hour in xrange(24): + for hour in range(24): df.loc[df['hour'] == hour, 'weekday_factor'] = weekday_profile[ hour, df[df['hour'] == hour].index] # SATURDAY @@ -747,7 +741,7 @@ class TemporalDistribution(object): if isinstance(saturday_profile, dict): df['saturday_factor'] = df['hour'].map(saturday_profile) else: - for hour in xrange(24): + for hour in range(24): df.loc[df['hour'] == hour, 'saturday_factor'] = saturday_profile[ hour, df[df['hour'] == hour].index] # SUNDAY @@ -755,7 +749,7 @@ class TemporalDistribution(object): if isinstance(sunday_profile, dict): df['sunday_factor'] = df['hour'].map(sunday_profile) else: - for hour in xrange(24): + for hour in range(24): df.loc[df['hour'] == hour, 'sunday_factor'] = sunday_profile[ hour, df[df['hour'] == hour].index] @@ -782,7 +776,7 @@ class TemporalDistribution(object): if isinstance(self.weekly_profile[month], dict): df.loc[df['month'] == month, 'weekly_factor'] = df['weekday'].map(self.weekly_profile[month]) else: - for weekday in xrange(7): + for weekday in range(7): df.loc[df['weekday'] == weekday, 'weekly_factor'] = self.weekly_profile[month][ weekday, df[df['weekday'] == weekday].index] @@ -871,7 +865,7 @@ class TemporalDistribution(object): weekdays_factors = 0 num_days = 0 - for weekday in xrange(7): + for weekday in range(7): weekdays_factors += profile[weekday] * weekdays[weekday] num_days += weekdays[weekday] @@ -894,7 +888,7 @@ class TemporalDistribution(object): weekdays_factors = np.zeros((profile.shape[-1])) num_days = 0 - for weekday in xrange(7): + for weekday in range(7): weekdays_factors += profile[weekday, :] * weekdays[weekday] num_days += weekdays[weekday] @@ -918,7 +912,7 @@ class TemporalDistribution(object): st_time = timeit.default_timer() weekdays = [MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY] - days = [weekday(date.year, date.month, d + 1) for d in xrange(monthrange(date.year, date.month)[1])] + days = [weekday(date.year, date.month, d + 1) for d in range(monthrange(date.year, date.month)[1])] weekdays_dict = {} count = 0 for day in weekdays: diff --git a/hermesv3_gr/modules/vertical/vertical.py b/hermesv3_gr/modules/vertical/vertical.py index 1b309ea4be040924ab3f08aa4078ac1ceb1aed42..34baa1a738d33c198841d2f702639e96864f8d60 100755 --- a/hermesv3_gr/modules/vertical/vertical.py +++ b/hermesv3_gr/modules/vertical/vertical.py @@ -84,7 +84,7 @@ class VerticalDistribution(object): " The v_profile '{0}' of the '{1}' file doesn't match.".format(self.id, path)) sys.exit(1) else: - return_value = zip(v_profile['layers'], v_profile['weights']) + return_value = list(zip(v_profile['layers'], v_profile['weights'])) settings.write_time('VerticalDistribution', 'get_vertical_profile', timeit.default_timer() - st_time, level=3) @@ -140,7 +140,7 @@ class VerticalDistribution(object): index = len([s for s in output_vertical_profile if s < prev_layer]) origin_diff_factor = in_weight / (layer - prev_layer) weight_list = [] - for i in xrange(len(output_vertical_profile_aux) - 1): + for i in range(len(output_vertical_profile_aux) - 1): weight = (abs(output_vertical_profile_aux[i] - output_vertical_profile_aux[i + 1])) * origin_diff_factor weight_list.append({'index': index, 'weight': weight}) index += 1 diff --git a/hermesv3_gr/modules/vertical/vertical_gfas.py b/hermesv3_gr/modules/vertical/vertical_gfas.py index 309092613d01be5355d7ad8f21ced18110aaa288..5e65c19720f22e921a30c83eb6a99c0af23d22e3 100755 --- a/hermesv3_gr/modules/vertical/vertical_gfas.py +++ b/hermesv3_gr/modules/vertical/vertical_gfas.py @@ -20,7 +20,7 @@ import timeit import hermesv3_gr.config.settings as settings -from vertical import VerticalDistribution +from .vertical import VerticalDistribution class GfasVerticalDistribution(VerticalDistribution): @@ -68,7 +68,7 @@ class GfasVerticalDistribution(VerticalDistribution): st_time = timeit.default_timer() widths = [] - for i in xrange(len(heights_list)): + for i in range(len(heights_list)): if i == 0: widths.append(heights_list[i]) else: @@ -126,10 +126,10 @@ class GfasVerticalDistribution(VerticalDistribution): st_time = timeit.default_timer() fires = np.zeros(top_fires.shape) - for i in xrange(len(self.output_heights)): + for i in range(len(self.output_heights)): if top_fires[i].sum() != 0: weight_list = self.get_weights(list(self.output_heights[0: i + 1])) - for i_weight in xrange(len(weight_list)): + for i_weight in range(len(weight_list)): fires[i_weight] += top_fires[i] * weight_list[i_weight] settings.write_time('GfasVerticalDistribution', 'apply_approach', timeit.default_timer() - st_time, level=3) @@ -185,7 +185,7 @@ class GfasVerticalDistribution(VerticalDistribution): return fire_list def calculate_weight_layer_dict(self, layer): - weight_layer_dict = {x: None for x in xrange(layer + 1)} + weight_layer_dict = {x: None for x in range(layer + 1)} if self.approach == '50_top': weight_layer_dict[layer] = 0.5 to_distribute = 0.5 @@ -213,7 +213,7 @@ class GfasVerticalDistribution(VerticalDistribution): vert_emissions.append(emis) else: weight_layer_dict = self.calculate_weight_layer_dict(layer) - for layer, weight in weight_layer_dict.iteritems(): + for layer, weight in weight_layer_dict.items(): aux_emis = emis.copy() aux_emis.loc[:, 'layer'] = layer aux_emis.loc[:, input_pollutant_list] = aux_emis[input_pollutant_list].multiply(weight) diff --git a/hermesv3_gr/modules/writing/writer.py b/hermesv3_gr/modules/writing/writer.py index 06e6f34b4136aa34365b2f02fee263ce579477ed..9fb1d1a968225a1497f02d8246a851faebbd7ec1 100755 --- a/hermesv3_gr/modules/writing/writer.py +++ b/hermesv3_gr/modules/writing/writer.py @@ -24,6 +24,9 @@ import numpy as np from mpi4py import MPI from netCDF4 import Dataset from hermesv3_gr.config import settings +from functools import reduce +import pandas as pd +from pandas import DataFrame, MultiIndex class Writer(object): @@ -128,7 +131,7 @@ class Writer(object): settings.write_log("\t\tParallel NetCDF file ready to write.", level=2) index = 0 # print "Rank {0} 2".format(rank) - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) st_time = timeit.default_timer() @@ -179,13 +182,13 @@ class Writer(object): dict_aux['data'] = None empty_dict[emi['name']] = dict_aux - self.variables_attributes = empty_dict.values() + self.variables_attributes = list(empty_dict.values()) settings.write_time('Writer', 'set_variable_attributes', timeit.default_timer() - st_time, level=3) return True - def calculate_data_by_var(self, variable, inventory_list, shape): + def calculate_data_by_var(self, variable, inventory_list, shape, change_units=True): """ Calculate the date of the given variable throw the inventory list. @@ -242,7 +245,8 @@ class Writer(object): settings.write_time('TemporalDistribution', 'calculate_data_by_var', timeit.default_timer() - temporal_time, level=2) # Unit changes - data = self.unit_change(variable, data) + if change_units: + data = self.unit_change(variable, data) if data is not None: data[data < 0] = 0 settings.write_time('Writer', 'calculate_data_by_var', timeit.default_timer() - st_time, level=3) @@ -252,7 +256,7 @@ class Writer(object): """ Implement on inner class """ - return np.array([0]) + return data @staticmethod def calculate_displacements(counts): @@ -384,9 +388,9 @@ class Writer(object): lat_dim = ('lat',) elif len(center_latitudes.shape) == 2: netcdf.createDimension('lat', center_latitudes.shape[0]) - lat_dim = ('lon', 'lat',) + lat_dim = ('lat', 'lon',) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -395,24 +399,24 @@ class Writer(object): lon_dim = ('lon',) elif len(center_longitudes.shape) == 2: netcdf.createDimension('lon', center_longitudes.shape[1]) - lon_dim = ('lon', 'lat',) + lon_dim = ('lat', 'lon',) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( - len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( + len(center_longitudes.shape))) sys.exit(1) elif roated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -560,7 +564,7 @@ class Writer(object): try: var[:] = variable['data'] except ValueError: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -602,3 +606,39 @@ class Writer(object): netcdf.setncatts(global_attributes) netcdf.close() + + def get_emis(self, emission_list): + self.set_variable_attributes(emission_list) + self.change_variable_attributes() + + data_list = [] + + fid_array = np.array(range(self.grid.full_shape[-2] * self.grid.full_shape[-1])) + fid_array = fid_array.reshape((self.grid.full_shape[-2], self.grid.full_shape[-1])) + fid_array = fid_array[self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound].flatten() + + zero_vars = [] + for var_name in self.variables_attributes.keys(): + var_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape, change_units=False) + if var_data is None: + zero_vars.append(var_name) + else: + var_data_list = [] + for tstep in range(var_data.shape[0]): + for layer in range(var_data.shape[1]): + aux_data = DataFrame(data=var_data[tstep, layer, :, :].flatten(), columns=[var_name], + index=MultiIndex.from_product([fid_array, [layer], [tstep]], + names=['FID', 'layer', 'tstep'])) + aux_data = aux_data.replace(0, np.nan) + aux_data.dropna(how='all', inplace=True) + var_data_list.append(aux_data) + data_list.append(pd.concat(var_data_list, sort=False)) + + data = pd.concat(data_list, axis=1, sort=False) + data = data.replace(np.nan, 0) + + for var_name in zero_vars: + data[var_name] = 0 + + return data diff --git a/hermesv3_gr/modules/writing/writer_cmaq.py b/hermesv3_gr/modules/writing/writer_cmaq.py index 7b3480c780f442559400412192c7367b0a61b0e6..71242c10677390be8f9aae06821640ff920c3e09 100755 --- a/hermesv3_gr/modules/writing/writer_cmaq.py +++ b/hermesv3_gr/modules/writing/writer_cmaq.py @@ -144,16 +144,17 @@ class WriterCmaq(Writer): :rtype: numpy.array """ from datetime import timedelta + t_flag = np.empty((len(hours_array), num_vars, 2)) - a = np.array([[[]]]) - - for inc_hours in hours_array: + for i_d, inc_hours in enumerate(hours_array): date = st_date + timedelta(hours=inc_hours) - b = np.array([[int(date.strftime('%Y%j'))], [int(date.strftime('%H%M%S'))]] * num_vars) - a = np.append(a, b) + y_d = int(date.strftime('%Y%j')) + hms = int(date.strftime('%H%M%S')) + for i_p in range(num_vars): + t_flag[i_d, i_p, 0] = y_d + t_flag[i_d, i_p, 1] = hms - a.shape = (len(hours_array), 2, num_vars) - return a + return t_flag @staticmethod def str_var_list(var_list): @@ -198,7 +199,7 @@ class WriterCmaq(Writer): if self.global_attributes_path is not None: df = pd.read_csv(self.global_attributes_path) - for att in atts_dict.iterkeys(): + for att in atts_dict.keys(): try: if att in int_atts: atts_dict[att] = np.int32(df.loc[df['attribute'] == att, 'value'].item()) @@ -220,7 +221,7 @@ class WriterCmaq(Writer): settings.write_log('WARNING: Check the .err file to get more information.') message = 'WARNING: No output attributes defined, check the output_attributes' message += ' parameter of the configuration file.\nUsing default values:' - for key, value in atts_dict.iteritems(): + for key, value in atts_dict.items(): message += '\n\t{0} = {1}'.format(key, value) if settings.rank == 0: warning(message) @@ -431,7 +432,7 @@ class WriterCmaq(Writer): index = 0 # data_list, var_list = self.change_variable_attributes(self.variables_attributes) - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): index += 1 var = netcdf.createVariable(var_name, 'f', ('TSTEP', 'LAY', 'ROW', 'COL',), zlib=self.compress) var.setncatts(self.variables_attributes[var_name]) @@ -440,7 +441,7 @@ class WriterCmaq(Writer): # ===== Global attributes ===== settings.write_log("\t\tCreating NetCDF metadata.", level=2) - global_attributes = self.create_global_attributes(self.variables_attributes.keys()) + global_attributes = self.create_global_attributes(list(self.variables_attributes.keys())) for attribute in self.global_attributes_order: netcdf.setncattr(attribute, global_attributes[attribute]) @@ -498,13 +499,14 @@ class WriterCmaq(Writer): tflag = netcdf.createVariable('TFLAG', 'i', ('TSTEP', 'VAR', 'DATE-TIME',)) tflag.setncatts({'units': "{:<16}".format(''), 'long_name': "{:<16}".format('TFLAG'), 'var_desc': "{:<80}".format('Timestep-valid flags: (1) YYYYDDD or (2) HHMMSS')}) - tflag[:] = self.create_tflag(self.date, self.hours, len(self.variables_attributes)) + var_tflag = self.create_tflag(self.date, self.hours, len(self.variables_attributes)) + tflag[:] = var_tflag settings.write_log("\t\t\t'TFLAG' variable created with size: {0}".format(tflag[:].shape), level=3) full_shape = None index = 0 # data_list, var_list = self.change_variable_attributes(self.variables_attributes) - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): if settings.size != 1: settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) @@ -575,7 +577,7 @@ class WriterCmaq(Writer): if mpi_numpy: data = np.ones(var[:].shape, dtype=settings.precision) * 100 - for i in xrange(settings.size): + for i in range(settings.size): try: if i == 0: var[:, :, :, :full_position[i][3]] = recvbuf[i] @@ -595,7 +597,7 @@ class WriterCmaq(Writer): elif mpi_vector: if rank_data is not None: data = np.empty(var[:].shape, dtype=settings.precision) - for i in xrange(settings.size): + for i in range(settings.size): # print 'Resizeing {0}'.format(i) if not i == settings.size - 1: data[:, :, full_position[i][0]:full_position[i][1], @@ -615,7 +617,7 @@ class WriterCmaq(Writer): settings.write_log("\t\tCreating NetCDF metadata.", level=2) if settings.rank == 0: # ===== Global attributes ===== - global_attributes = self.create_global_attributes(self.variables_attributes.keys()) + global_attributes = self.create_global_attributes(list(self.variables_attributes.keys())) for attribute in self.global_attributes_order: netcdf.setncattr(attribute, global_attributes[attribute]) diff --git a/hermesv3_gr/modules/writing/writer_monarch.py b/hermesv3_gr/modules/writing/writer_monarch.py index 8749ed5c9f245d4316b62ed3509977d17d634ea0..1bd675827d67b5d5ed7c6b63818dd000968bf497 100755 --- a/hermesv3_gr/modules/writing/writer_monarch.py +++ b/hermesv3_gr/modules/writing/writer_monarch.py @@ -154,8 +154,8 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'lat' dimension: {0}".format(self.grid.center_latitudes.shape[0]), level=3) lat_dim = ('lon', 'lat', ) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format( - len(self.grid.center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format( + len(self.grid.center_latitudes.shape))) sys.exit(1) # Longitude @@ -168,15 +168,15 @@ class WriterMonarch(Writer): settings.write_log("\t\t\t'lon' dimension: {0}".format(self.grid.center_longitudes.shape[1]), level=3) lon_dim = ('lon', 'lat', ) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( - len(self.grid.center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format( + len(self.grid.center_longitudes.shape))) sys.exit(1) elif Rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if self.grid.rlat is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(self.grid.rlat)) settings.write_log("\t\t\t'rlat' dimension: {0}".format(len(self.grid.rlat)), level=3) @@ -184,7 +184,7 @@ class WriterMonarch(Writer): # Rotated Longitude if self.grid.rlon is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(self.grid.rlon)) settings.write_log("\t\t\t'rlon' dimension: {0}".format(len(self.grid.rlon)), level=3) @@ -333,7 +333,7 @@ class WriterMonarch(Writer): var[:] = 0 index = 0 - for var_name, variable in self.variables_attributes.iteritems(): + for var_name, variable in self.variables_attributes.items(): index += 1 var = netcdf.createVariable(var_name, 'f', ('time',) + var_dim, zlib=self.compress) @@ -632,7 +632,7 @@ class WriterMonarch(Writer): full_shape = None index = 0 - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): if settings.size != 1: settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) @@ -719,7 +719,7 @@ class WriterMonarch(Writer): if mpi_numpy: data = np.ones(var[:].shape, dtype=settings.precision) * 100 - for i in xrange(settings.size): + for i in range(settings.size): try: if i == 0: var[:, :, :, :full_position[i][3]] = recvbuf[i] @@ -739,7 +739,7 @@ class WriterMonarch(Writer): elif mpi_vector: if rank_data is not None: data = np.empty(var[:].shape, dtype=settings.precision) - for i in xrange(settings.size): + for i in range(settings.size): if not i == settings.size - 1: data[:, :, full_position[i][0]:full_position[i][1], full_position[i][2]:full_position[i][3]] = \ diff --git a/hermesv3_gr/modules/writing/writer_wrf_chem.py b/hermesv3_gr/modules/writing/writer_wrf_chem.py index 364dbcdf1fca6383aa1f7e2d07bad10e4d1706d8..5f27d7998d5c8fc4e7491dbcd77c483fdf33ecc2 100755 --- a/hermesv3_gr/modules/writing/writer_wrf_chem.py +++ b/hermesv3_gr/modules/writing/writer_wrf_chem.py @@ -216,7 +216,7 @@ class WriterWrfChem(Writer): if self.global_attributes_path is not None: df = pd.read_csv(self.global_attributes_path) - for att in atts_dict.iterkeys(): + for att in atts_dict.keys(): try: if att in int_atts: atts_dict[att] = np.int32(df.loc[df['attribute'] == att, 'value'].item()) @@ -225,7 +225,7 @@ class WriterWrfChem(Writer): elif att in str_atts: atts_dict[att] = str(df.loc[df['attribute'] == att, 'value'].item()) except ValueError: - print 'A warning has occurred. Check the .err file to get more information.' + print('A warning has occurred. Check the .err file to get more information.') if settings.rank == 0: warning('The global attribute {0} is not defined; Using default value {1}'.format( att, atts_dict[att])) @@ -234,7 +234,7 @@ class WriterWrfChem(Writer): settings.write_log('WARNING: Check the .err file to get more information.') message = 'WARNING: No output attributes defined, check the output_attributes' message += ' parameter of the configuration file.\nUsing default values:' - for key, value in atts_dict.iteritems(): + for key, value in atts_dict.items(): message += '\n\t{0} = {1}'.format(key, value) if settings.rank == 0: warning(message) @@ -299,16 +299,13 @@ class WriterWrfChem(Writer): :return: """ from datetime import timedelta - import netCDF4 - aux_times_list = [] + aux_times = np.chararray((len(self.hours), 19), itemsize=1) - for hour in self.hours: + for i, hour in enumerate(self.hours): aux_date = self.date + timedelta(hours=hour) - aux_times_list.append(aux_date.strftime("%Y-%m-%d_%H:%M:%S")) - - str_out = netCDF4.stringtochar(np.array(aux_times_list)) - return str_out + aux_times[i] = list(aux_date.strftime("%Y-%m-%d_%H:%M:%S")) + return aux_times def create_parallel_netcdf(self): # TODO Documentation @@ -343,7 +340,7 @@ class WriterWrfChem(Writer): settings.write_log("\t\t\t'Times' variable created with size: {0}".format(times[:].shape), level=3) index = 0 - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): index += 1 var = netcdf.createVariable(var_name, 'f', ('Time', 'emissions_zdim', 'south_north', 'west_east',), zlib=self.compress) @@ -402,6 +399,7 @@ class WriterWrfChem(Writer): # ===== Variables ===== settings.write_log("\t\tCreating NetCDF variables.", level=2) times = netcdf.createVariable('Times', 'S1', ('Time', 'DateStrLen', )) + print(times[:].shape, self.create_times_var().shape) times[:] = self.create_times_var() settings.write_log("\t\t\t'Times' variable created with size: {0}".format(times[:].shape), level=3) @@ -410,7 +408,7 @@ class WriterWrfChem(Writer): # self.change_variable_attributes() - for var_name in self.variables_attributes.iterkeys(): + for var_name in self.variables_attributes.keys(): if settings.size != 1: settings.write_log("\t\t\tGathering {0} data.".format(var_name), level=3) rank_data = self.calculate_data_by_var(var_name, emission_list, self.grid.shape) @@ -458,7 +456,7 @@ class WriterWrfChem(Writer): if rank_data is not None: data = np.empty(var[:].shape, dtype=settings.precision) - for i in xrange(settings.size): + for i in range(settings.size): # print 'Resizeing {0}'.format(i) if not i == settings.size - 1: data[:, :, full_position[i][0]:full_position[i][1], diff --git a/hermesv3_gr/tools/coordinates_tools.py b/hermesv3_gr/tools/coordinates_tools.py index 56d929adacdf4388307d2d307a46d17fcf57a53b..00baa67df654e5175ed5c59b8853a6d907f02fbf 100755 --- a/hermesv3_gr/tools/coordinates_tools.py +++ b/hermesv3_gr/tools/coordinates_tools.py @@ -296,9 +296,9 @@ if __name__ == '__main__': import numpy as np new_pole_lon_d = 20.0 # lonpole tlm0d new_pole_lat_d = 35.0 # latpole tph0d - print latlon2rotated(new_pole_lon_d, new_pole_lat_d, 20.0, 35.0) - print latlon2rotated(new_pole_lon_d, new_pole_lat_d, -20.2485, -9.9036) - print rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, 0, 0) - print rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, -51., -35.) - print rotated2latlon(new_pole_lon_d, new_pole_lat_d, np.array([-51., -51., -51., -51.]), - np.array([-35., -34.9, -34.8, -34.7])) + print(latlon2rotated(new_pole_lon_d, new_pole_lat_d, 20.0, 35.0)) + print(latlon2rotated(new_pole_lon_d, new_pole_lat_d, -20.2485, -9.9036)) + print(rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, 0, 0)) + print(rotated2latlon_single(new_pole_lon_d, new_pole_lat_d, -51., -35.)) + print(rotated2latlon(new_pole_lon_d, new_pole_lat_d, np.array([-51., -51., -51., -51.]), + np.array([-35., -34.9, -34.8, -34.7]))) diff --git a/hermesv3_gr/tools/download_benchmark.py b/hermesv3_gr/tools/download_benchmark.py index 5d58bc8331a1a907235a16519c8e8e83d4dc1960..ef08a180d59f574e87a8c3429e96da19509cb06a 100755 --- a/hermesv3_gr/tools/download_benchmark.py +++ b/hermesv3_gr/tools/download_benchmark.py @@ -36,7 +36,7 @@ def query_yes_no(question, default="yes"): while True: sys.stdout.write(question + prompt) - choice = raw_input().lower() + choice = input().lower() if default is not None and choice == '': return valid[default] elif choice in valid: @@ -47,12 +47,12 @@ def query_yes_no(question, default="yes"): def check_args(args, exe_str): if len(args) == 0: - print("Missing destination path after '{0}'. e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES".format(exe_str)) + print(("Missing destination path after '{0}'. e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES".format(exe_str))) sys.exit(1) elif len(args) > 1: - print("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES".format(exe_str)) + print(("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES".format(exe_str))) sys.exit(1) else: dir_path = args[0] diff --git a/hermesv3_gr/tools/netcdf_tools.py b/hermesv3_gr/tools/netcdf_tools.py index 955468ed7fa610be247a54e718c96628bbe11e4e..9e0fa356af80f61245069cf47bd80f75ec002705 100755 --- a/hermesv3_gr/tools/netcdf_tools.py +++ b/hermesv3_gr/tools/netcdf_tools.py @@ -21,11 +21,7 @@ import sys from netCDF4 import Dataset from mpi4py import MPI - -ICOMM = MPI.COMM_WORLD -COMM = ICOMM.Split(color=0, key=0) -RANK = COMM.Get_rank() -SIZE = COMM.Get_size() +from functools import reduce def open_netcdf(netcdf_path): @@ -166,9 +162,9 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lat_dim = ('lat',) elif len(center_latitudes.shape) == 2: netcdf.createDimension('lat', center_latitudes.shape[0]) - lat_dim = ('lon', 'lat', ) + lat_dim = ('lat', 'lon', ) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -177,23 +173,23 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lon_dim = ('lon',) elif len(center_longitudes.shape) == 2: netcdf.createDimension('lon', center_longitudes.shape[1]) - lon_dim = ('lon', 'lat', ) + lon_dim = ('lat', 'lon', ) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -337,7 +333,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, try: var[:] = variable['data'] except ValueError: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -433,9 +429,9 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lat_dim = ('lat',) elif len(center_latitudes.shape) == 2: netcdf.createDimension('lat', center_latitudes.shape[0]) - lat_dim = ('lon', 'lat', ) + lat_dim = ('lat', 'lon', ) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -444,23 +440,23 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lon_dim = ('lon',) elif len(center_longitudes.shape) == 2: netcdf.createDimension('lon', center_longitudes.shape[1]) - lon_dim = ('lon', 'lat', ) + lon_dim = ('lat', 'lon', ) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -605,9 +601,8 @@ def create_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, # if variable['data'] is not 0: # print var[:].shape, variable['data'].shape, variable['data'].max() shape = tuple() - exec ("shape = (len(hours), {0}.size, {1}.size, {2}.size)".format(var_dim[0], var_dim[1], var_dim[2])) - # exit() - print shape + exec("shape = (len(hours), {0}.size, {1}.size, {2}.size)".format(var_dim[0], var_dim[1], var_dim[2])) + var[:] = np.zeros(shape) # Grid mapping diff --git a/hermesv3_gr/tools/sample_files.py b/hermesv3_gr/tools/sample_files.py index d8fee3fedd73e61643223ba7837addd38ff04065..17c7eac88ce876a1fa9b35511ccc8c7991a71ef0 100755 --- a/hermesv3_gr/tools/sample_files.py +++ b/hermesv3_gr/tools/sample_files.py @@ -131,7 +131,7 @@ def query_yes_no(question, default="yes"): while True: sys.stdout.write(question + prompt) - choice = raw_input().lower() + choice = input().lower() if default is not None and choice == '': return valid[default] elif choice in valid: @@ -142,12 +142,12 @@ def query_yes_no(question, default="yes"): def check_args(args, exe_str): if len(args) == 0: - print("Missing destination path after '{0}'. e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str)) + print(("Missing destination path after '{0}'. e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str))) sys.exit(1) elif len(args) > 1: - print("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + - "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str)) + print(("Too much arguments through '{0}'. Only destination path is needed e.g.:".format(exe_str) + + "\n\t{0} /home/user/HERMES/HERMES_IN".format(exe_str))) sys.exit(1) else: dir_path = args[0] @@ -169,7 +169,7 @@ def copy_files(file_list, directory): for element in file_list: if dict == type(element): - copy_files(element.values()[0], os.path.join(directory, element.keys()[0])) + copy_files(list(element.values())[0], os.path.join(directory, list(element.keys())[0])) else: copy2(element, directory) return True diff --git a/preproc/cams_glob_ant_preproc.py b/preproc/cams_glob_ant_preproc.py index 2d6ae51ac05dfa2b04b2ff0dbbeabe7eba886f3b..44113e5a34fcf0c6885820f42811b7f23d266786 100755 --- a/preproc/cams_glob_ant_preproc.py +++ b/preproc/cams_glob_ant_preproc.py @@ -213,7 +213,7 @@ def create_bounds(coordinates, number_vertices=2): def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cell_area, date): # TODO Documentation - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -341,7 +341,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/cams_glob_ocean_preproc.py b/preproc/cams_glob_ocean_preproc.py index b4c42dbb50bfc9e211326239e0082930dd87392b..8b9fae21e60e5e33ca4d880c65d2cb5e1f04a812 100755 --- a/preproc/cams_glob_ocean_preproc.py +++ b/preproc/cams_glob_ocean_preproc.py @@ -184,7 +184,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/cams_glob_ship_preproc.py b/preproc/cams_glob_ship_preproc.py index 49e98d62a2872eab5a73424261f826d9d5213794..9e2a184dccace5876f303251ee037b618abb7897 100755 --- a/preproc/cams_glob_ship_preproc.py +++ b/preproc/cams_glob_ship_preproc.py @@ -190,7 +190,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/cams_glob_soil_preproc.py b/preproc/cams_glob_soil_preproc.py index cade121cba8934862a9c38c190f0682d7b7d628b..3fc4e7e436ea41167c2dea54a433159796e2c0a4 100755 --- a/preproc/cams_glob_soil_preproc.py +++ b/preproc/cams_glob_soil_preproc.py @@ -204,7 +204,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/cams_reg_ap_preproc.py b/preproc/cams_reg_ap_preproc.py index 57b37949e529cc7535a6fd832a73d8100d6e80b0..46eb47b90c87d1840ad8f7c80ec6d3560cdeac3e 100755 --- a/preproc/cams_reg_ap_preproc.py +++ b/preproc/cams_reg_ap_preproc.py @@ -78,14 +78,14 @@ def calculate_grid_definition(in_path): # Longitudes lons = np.sort(np.unique(dataframe.Lon_rounded)) lons_interval = lons[1:] - lons[:-1] - print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - dataframe.Lon_rounded.min(), dataframe.Lon_rounded.max(), lons_interval.min(), len(lons)) + print('Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( + dataframe.Lon_rounded.min(), dataframe.Lon_rounded.max(), lons_interval.min(), len(lons))) # Latitudes lats = np.sort(np.unique(dataframe.Lat_rounded)) lats_interval = lats[1:] - lats[:-1] - print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - dataframe.Lat_rounded.min(), dataframe.Lat_rounded.max(), lats_interval.min(), len(lats)) + print('Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( + dataframe.Lat_rounded.min(), dataframe.Lat_rounded.max(), lats_interval.min(), len(lats))) lats = np.arange(-90 + lats_interval.min() / 2, 90, lats_interval.min(), dtype=np.float64) lons = np.arange(-180 + lons_interval.min() / 2, 180, lons_interval.min(), dtype=np.float64) @@ -172,7 +172,7 @@ def do_transformation(year): #dataframe = pd.concat([df_np, df_p]) for name, group in dataframe.groupby('GNFR_Sector'): - print 'gnfr', name + print('gnfr', name) pollutant_list = create_pollutant_empty_list(in_file, len(c_lats), len(c_lons)) # Other mobile sources ignoring sea cells (shipping emissions) @@ -182,7 +182,7 @@ def do_transformation(year): group = group.groupby(['row_lat', 'col_lon']).sum().reset_index() - for i in xrange(len(pollutant_list)): + for i in range(len(pollutant_list)): # pollutant_list[i]['data'][group.col_lon, group.row_lat] = group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'][group.row_lat, group.col_lon] += group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'] = pollutant_list[i]['data'].reshape((1,) + pollutant_list[i]['data'].shape) @@ -396,8 +396,8 @@ def check_vocs(year): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - gnfr, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum) + print('{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + gnfr, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum)) return True def do_pm_transformation(year): @@ -475,8 +475,8 @@ def check_pm(year): [new_pm] = extract_vars(pm_path, [pm]) pm_sum += new_pm['data'].sum() - print '{0} PM2.5 sum: {1}; PM sum: {2}; %diff: {3}'.format( - gnfr, pm25_sum, pm_sum, 100*(pm25_sum - pm_sum) / pm25_sum) + print('{0} PM2.5 sum: {1}; PM sum: {2}; %diff: {3}'.format( + gnfr, pm25_sum, pm_sum, 100*(pm25_sum - pm_sum) / pm25_sum)) return True if __name__ == '__main__': diff --git a/preproc/cams_reg_ap_preproc_pm_ratios.py b/preproc/cams_reg_ap_preproc_pm_ratios.py index d2ff01c8cd5efb5aacdd8ce52cb286ec459d96ca..ee18136a25b2fbad7a409ff1a740831e1e939850 100755 --- a/preproc/cams_reg_ap_preproc_pm_ratios.py +++ b/preproc/cams_reg_ap_preproc_pm_ratios.py @@ -60,7 +60,7 @@ def extract_vars(netcdf_path, variables_list, attributes_list=()): data_list.append(dict_aux) netcdf.close() - print data_list + print(data_list) return data_list @@ -104,7 +104,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, if not (regular_latlon or lcc or rotated): regular_latlon = True - print netcdf_path + print(netcdf_path) netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== @@ -119,7 +119,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat',) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -130,21 +130,21 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat',) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -218,7 +218,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lons.axis = "X" lons.long_name = "longitude coordinate" lons.standard_name = "longitude" - print 'lons:', lons[:].shape, center_longitudes.shape + print('lons:', lons[:].shape, center_longitudes.shape) lons[:] = center_longitudes if boundary_longitudes is not None: lons.bounds = "lon_bnds" @@ -290,7 +290,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, try: var[:] = variable['data'] except Exception: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -391,22 +391,22 @@ def create_pm_ratio(pm): import numpy as np [country_values, lat, lon] = extract_vars(TNO_WORLD_MASK, ['timezone_id', 'lat', 'lon']) country_values = country_values['data'].reshape((country_values['data'].shape[1], country_values['data'].shape[1])) - print country_values + print(country_values) #sys.exit() - print OUTPUT_PATH + print(OUTPUT_PATH) if not os.path.exists(OUTPUT_PATH): os.makedirs(OUTPUT_PATH) complete_output_path = os.path.join(OUTPUT_PATH, 'ratio_{0}_{1}.nc'.format(pm, YEAR)) if not os.path.exists(complete_output_path): - print 'Creating ratio file for {0}\npath: {1}'.format(pm, complete_output_path) + print('Creating ratio file for {0}\npath: {1}'.format(pm, complete_output_path)) data_list = [] for gnfr in get_sector_list(pm): - print gnfr + print(gnfr) mask_factor = np.zeros(country_values.shape) iso_codes = get_iso_codes() - for country_code, factor in get_country_code_and_factor(pm, gnfr, YEAR).iteritems(): + for country_code, factor in get_country_code_and_factor(pm, gnfr, YEAR).items(): try: mask_factor[country_values == iso_codes[country_code]] = factor except Exception: @@ -421,7 +421,7 @@ def create_pm_ratio(pm): }) write_netcdf(complete_output_path, lat['data'], lon['data'], data_list) else: - print 'Ratio file for {0} already created\npath: {1}'.format(pm, complete_output_path) + print('Ratio file for {0} already created\npath: {1}'.format(pm, complete_output_path)) return True @@ -475,7 +475,7 @@ def get_pm_list(): del df['ISO3'], df['gnfr'], df['fr'], df['year'] df = df.drop_duplicates().dropna() pm_list = df.pmcode.values - print df.pmcode.values + print(df.pmcode.values) return df.pmcode.values @@ -493,7 +493,7 @@ def get_sector_list(pm): df = df[df.pmcode == pm] del df['ISO3'], df['pmcode'], df['year'], df['fr'] df = df.drop_duplicates().dropna() - print df.gnfr.values + print(df.gnfr.values) return df.gnfr.values diff --git a/preproc/cams_reg_ap_preproc_voc_ratios.py b/preproc/cams_reg_ap_preproc_voc_ratios.py index a3bc49c9abd8e2f27cb20055903f36322e366eff..6833243a832d2834f81dbbdff3b2b5b09c9540d1 100755 --- a/preproc/cams_reg_ap_preproc_voc_ratios.py +++ b/preproc/cams_reg_ap_preproc_voc_ratios.py @@ -60,7 +60,7 @@ def extract_vars(netcdf_path, variables_list, attributes_list=()): data_list.append(dict_aux) netcdf.close() - print data_list + print(data_list) return data_list @@ -104,7 +104,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, if not (regular_latlon or lcc or rotated): regular_latlon = True - print netcdf_path + print(netcdf_path) netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== @@ -119,7 +119,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat',) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -130,21 +130,21 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat',) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -218,7 +218,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lons.axis = "X" lons.long_name = "longitude coordinate" lons.standard_name = "longitude" - print 'lons:', lons[:].shape, center_longitudes.shape + print('lons:', lons[:].shape, center_longitudes.shape) lons[:] = center_longitudes if boundary_longitudes is not None: lons.bounds = "lon_bnds" @@ -290,7 +290,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, try: var[:] = variable['data'] except Exception: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -391,22 +391,22 @@ def create_voc_ratio(voc): import numpy as np [country_values, lat, lon] = extract_vars(TNO_WORLD_MASK, ['timezone_id', 'lat', 'lon']) country_values = country_values['data'].reshape((country_values['data'].shape[1], country_values['data'].shape[1])) - print country_values + print(country_values) #sys.exit() - print OUTPUT_PATH + print(OUTPUT_PATH) if not os.path.exists(OUTPUT_PATH): os.makedirs(OUTPUT_PATH) complete_output_path = os.path.join(OUTPUT_PATH, 'ratio_{0}_{1}.nc'.format(voc, YEAR)) if not os.path.exists(complete_output_path): - print 'Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path) + print('Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path)) data_list = [] for gnfr in get_sector_list(voc): - print gnfr + print(gnfr) mask_factor = np.zeros(country_values.shape) iso_codes = get_iso_codes() - for country_code, factor in get_country_code_and_factor(voc, gnfr, YEAR).iteritems(): + for country_code, factor in get_country_code_and_factor(voc, gnfr, YEAR).items(): try: mask_factor[country_values == iso_codes[country_code]] = factor except Exception: @@ -421,7 +421,7 @@ def create_voc_ratio(voc): }) write_netcdf(complete_output_path, lat['data'], lon['data'], data_list) else: - print 'Ratio file for {0} already created\npath: {1}'.format(voc, complete_output_path) + print('Ratio file for {0} already created\npath: {1}'.format(voc, complete_output_path)) return True @@ -472,14 +472,14 @@ def get_voc_list(): import pandas as pd df = pd.read_csv(CSV_PATH, sep=',') - print list(df.columns.values) + print(list(df.columns.values)) # sys.exit() del df['year'], df['ISO3'], df['gnfr'], df['fr'] df = df.drop_duplicates().dropna() voc_list = df.vcode.values - for i in xrange(len(voc_list)): + for i in range(len(voc_list)): voc_list[i] = voc_list[i].replace('v', 'voc') - print df.vcode.values + print(df.vcode.values) return df.vcode.values @@ -498,7 +498,7 @@ def get_sector_list(voc): df = df[df.vcode == voc] del df['ISO3'], df['vcode'], df['year'], df['fr'] df = df.drop_duplicates().dropna() - print df.gnfr.values + print(df.gnfr.values) return df.gnfr.values diff --git a/preproc/cams_reg_ghg_preproc.py b/preproc/cams_reg_ghg_preproc.py index 8447f4ccd90a83eb5be42af090e9ae7080b69e0d..59810213aa9d62ed603dbbdfa6d3f3e67203fdcb 100755 --- a/preproc/cams_reg_ghg_preproc.py +++ b/preproc/cams_reg_ghg_preproc.py @@ -74,14 +74,14 @@ def calculate_grid_definition(in_path): # Longitudes lons = np.sort(np.unique(dataframe.Lon_rounded)) lons_interval = lons[1:] - lons[:-1] - print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - dataframe.Lon_rounded.min(), dataframe.Lon_rounded.max(), lons_interval.min(), len(lons)) + print('Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( + dataframe.Lon_rounded.min(), dataframe.Lon_rounded.max(), lons_interval.min(), len(lons))) # Latitudes lats = np.sort(np.unique(dataframe.Lat_rounded)) lats_interval = lats[1:] - lats[:-1] - print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - dataframe.Lat_rounded.min(), dataframe.Lat_rounded.max(), lats_interval.min(), len(lats)) + print('Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( + dataframe.Lat_rounded.min(), dataframe.Lat_rounded.max(), lats_interval.min(), len(lats))) lats = np.arange(-90 + lats_interval.min() / 2, 90, lats_interval.min(), dtype=np.float64) lons = np.arange(-180 + lons_interval.min() / 2, 180, lons_interval.min(), dtype=np.float64) @@ -168,7 +168,7 @@ def do_transformation(year): #dataframe = pd.concat([df_np, df_p]) for name, group in dataframe.groupby('GNFR_Sector'): - print 'gnfr', name + print('gnfr', name) pollutant_list = create_pollutant_empty_list(in_file, len(c_lats), len(c_lons)) # Other mobile sources ignoring sea cells (shipping emissions) @@ -178,7 +178,7 @@ def do_transformation(year): group = group.groupby(['row_lat', 'col_lon']).sum().reset_index() - for i in xrange(len(pollutant_list)): + for i in range(len(pollutant_list)): # pollutant_list[i]['data'][group.col_lon, group.row_lat] = group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'][group.row_lat, group.col_lon] += group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'] = pollutant_list[i]['data'].reshape((1,) + pollutant_list[i]['data'].shape) @@ -392,8 +392,8 @@ def check_vocs(year): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - gnfr, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum) + print('{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + gnfr, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum)) return True def do_pm_transformation(year): @@ -471,8 +471,8 @@ def check_pm(year): [new_pm] = extract_vars(pm_path, [pm]) pm_sum += new_pm['data'].sum() - print '{0} PM2.5 sum: {1}; PM sum: {2}; %diff: {3}'.format( - gnfr, pm25_sum, pm_sum, 100*(pm25_sum - pm_sum) / pm25_sum) + print('{0} PM2.5 sum: {1}; PM sum: {2}; %diff: {3}'.format( + gnfr, pm25_sum, pm_sum, 100*(pm25_sum - pm_sum) / pm25_sum)) return True if __name__ == '__main__': diff --git a/preproc/ceds_preproc.py b/preproc/ceds_preproc.py index 5a5385b578f50b9a1e62ac6649cfe4c86a9eba11..7a3ee0b5afb564bc4c8f5da58c911cf075a70b90 100755 --- a/preproc/ceds_preproc.py +++ b/preproc/ceds_preproc.py @@ -248,7 +248,7 @@ def do_transformation(year): if not os.path.exists(file_path): os.makedirs(file_path) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', @@ -301,10 +301,10 @@ def do_air_transformation(year): elif sector == 'air_crs': data_aux = data[:, 15:24 + 1, :, :].sum(axis=1) else: - print 'ERROR' + print('ERROR') sys.exit(1) - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): emission = { 'name': pollutant_name, 'units': 'kg.m-2.s-1', diff --git a/preproc/eclipsev5a_preproc.py b/preproc/eclipsev5a_preproc.py index 9dd76589f629e4f728c1fecdcc94d4e1b6429e1c..55c3b8b7c37ee46f9c27626e4dc90163e212231d 100755 --- a/preproc/eclipsev5a_preproc.py +++ b/preproc/eclipsev5a_preproc.py @@ -99,7 +99,7 @@ def create_bounds(coordinates, number_vertices=2): def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cell_area, date): # TODO Documentation - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -236,9 +236,9 @@ def get_output_name(pollutant, sector, year, month): def do_single_transformation(pollutant, sector, data, c_lats, c_lons, cell_area): # TODO Docuemtnation - for i in xrange(len(LIST_YEARS)): + for i in range(len(LIST_YEARS)): - for month in xrange(12): + for month in range(12): # print i, list_years[i], month + 1 if pollutant == 'NOx': pollutant_name = 'nox_no2' @@ -270,7 +270,7 @@ def do_transformation(): # TODO Documentation for pollutant in LIST_POLLUTANTS: file_name = os.path.join(INPUT_PATH, INPUT_NAME.replace('', pollutant)) - print file_name + print(file_name) nc = Dataset(file_name, mode='r') c_lats = nc.variables['lat'][:] c_lons = nc.variables['lon'][:] @@ -325,7 +325,7 @@ def do_flaring_transformation(): if var_name is not None: data = nc_in.variables[var][:] data = np.nan_to_num(data) - for i in xrange(len(LIST_YEARS)): + for i in range(len(LIST_YEARS)): output_name = get_flaring_output_name(var_name, 'flaring', LIST_YEARS[i]) data_aux = data[i, :, :] data_aux = (data_aux * YEAR_FACTOR) / cell_area diff --git a/preproc/edgarv432_ap_preproc.py b/preproc/edgarv432_ap_preproc.py index 67ee72b251ef1069a26b152265113927716bf0de..6da9f0aab7ecd462d9f474850c0221502094bf0d 100755 --- a/preproc/edgarv432_ap_preproc.py +++ b/preproc/edgarv432_ap_preproc.py @@ -146,7 +146,7 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr month=None): # TODO Documentation # Creating NetCDF & Dimensions - print output_name_path + print(output_name_path) nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) nc_output.createDimension('lon', center_lons.shape[0]) @@ -233,7 +233,7 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr def do_yearly_transformation(year): # TODO Documentation for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): + for ipcc in list(ipcc_to_sector_dict().keys()): file_path = os.path.join( INPUT_PATH, YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', @@ -241,7 +241,7 @@ def do_yearly_transformation(year): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') if pollutant in ['PM2.5_bio', 'PM2.5_fossil']: @@ -285,7 +285,7 @@ def do_yearly_transformation(year): def do_monthly_transformation(year): # TODO Documentation for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): + for ipcc in list(ipcc_to_sector_dict().keys()): file_path = os.path.join( INPUT_PATH, YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', @@ -293,7 +293,7 @@ def do_monthly_transformation(year): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') if pollutant in ['PM2.5_bio', 'PM2.5_fossil']: @@ -328,7 +328,7 @@ def do_monthly_transformation(year): nc_month_factors = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE.replace('', sector))) month_factors = nc_month_factors.variables[sector][:] - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): data_aux = data * month_factors[month - 1, :, :] write_netcdf(os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant.lower(), year, str(month).zfill(2))), @@ -344,8 +344,8 @@ def do_monthly_transformation(year): def do_2010_monthly_transformation(): # TODO Documentation for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): - for month in xrange(1, 12 + 1, 1): + for ipcc in list(ipcc_to_sector_dict().keys()): + for month in range(1, 12 + 1, 1): file_path = os.path.join( INPUT_PATH, MONTHLY_INPUT_NAME.replace('', pollutant).replace('', @@ -353,7 +353,7 @@ def do_2010_monthly_transformation(): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') # print pollutant # print pollutant in ['PM2.5_bio', 'PM2.5_fossil'] diff --git a/preproc/edgarv432_voc_preproc.py b/preproc/edgarv432_voc_preproc.py index 091ce93c54cfc2ab6ab47c4fdf93c9db9f94f16d..a300c80632ab94bc272484e2c82cf7b4cce6e755 100755 --- a/preproc/edgarv432_voc_preproc.py +++ b/preproc/edgarv432_voc_preproc.py @@ -138,7 +138,7 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr month=None): # TODO Documentation # Creating NetCDF & Dimensions - print output_name_path + print(output_name_path) nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) nc_output.createDimension('lon', center_lons.shape[0]) @@ -227,9 +227,9 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr def do_yearly_transformation(year): # TODO Documentation - print year + print(year) for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): + for ipcc in list(ipcc_to_sector_dict().keys()): file_path = os.path.join( INPUT_PATH, YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', @@ -237,7 +237,7 @@ def do_yearly_transformation(year): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') data = nc_in.variables['emi_{0}'.format(pollutant.lower())][:] @@ -250,7 +250,7 @@ def do_yearly_transformation(year): nc_in.close() sector = ipcc_to_sector_dict()[ipcc] - if pollutant in ['voc{0}'.format(x) for x in xrange(1, 9 + 1, 1)]: + if pollutant in ['voc{0}'.format(x) for x in range(1, 9 + 1, 1)]: pollutant_aux = pollutant.replace('voc', 'voc0') else: pollutant_aux = pollutant @@ -274,9 +274,9 @@ def do_yearly_transformation(year): def do_monthly_transformation(year): # TODO Documentation - print year + print(year) for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): + for ipcc in list(ipcc_to_sector_dict().keys()): file_path = os.path.join( INPUT_PATH, YEARLY_INPUT_NAME.replace('', pollutant).replace('', str(year)).replace('', @@ -284,7 +284,7 @@ def do_monthly_transformation(year): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') data = nc_in.variables['emi_{0}'.format(pollutant.lower())][:] @@ -298,7 +298,7 @@ def do_monthly_transformation(year): sector = ipcc_to_sector_dict()[ipcc] - if pollutant in ['voc{0}'.format(x) for x in xrange(1, 9 + 1, 1)]: + if pollutant in ['voc{0}'.format(x) for x in range(1, 9 + 1, 1)]: pollutant_aux = pollutant.replace('voc', 'voc0') else: pollutant_aux = pollutant @@ -314,7 +314,7 @@ def do_monthly_transformation(year): nc_month_factors = Dataset(os.path.join(INPUT_PATH, MONTHLY_PATTERN_FILE.replace('', sector))) month_factors = nc_month_factors.variables[sector][:] - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): data_aux = data * month_factors[month - 1, :, :] write_netcdf(os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format( pollutant_aux.lower(), year, str(month).zfill(2))), @@ -330,8 +330,8 @@ def do_monthly_transformation(year): def do_2010_monthly_transformation(): # TODO Documentation for pollutant in LIST_POLLUTANTS: - for ipcc in ipcc_to_sector_dict().keys(): - for month in xrange(1, 12 + 1, 1): + for ipcc in list(ipcc_to_sector_dict().keys()): + for month in range(1, 12 + 1, 1): file_path = os.path.join( INPUT_PATH, MONTHLY_INPUT_NAME.replace('', pollutant).replace('', @@ -339,7 +339,7 @@ def do_2010_monthly_transformation(): if os.path.exists(file_path): grid_area = get_grid_area(file_path) - print file_path + print(file_path) nc_in = Dataset(file_path, mode='r') data = nc_in.variables['emi_{0}'.format(pollutant.lower())][:] @@ -353,7 +353,7 @@ def do_2010_monthly_transformation(): sector = ipcc_to_sector_dict()[ipcc] - if pollutant in ['voc{0}'.format(x) for x in xrange(1, 9 + 1, 1)]: + if pollutant in ['voc{0}'.format(x) for x in range(1, 9 + 1, 1)]: pollutant_aux = pollutant.replace('voc', 'voc0') else: pollutant_aux = pollutant diff --git a/preproc/emep_preproc.py b/preproc/emep_preproc.py index 15f44b0a9684f6ba1313477b9b552846be350a69..23bdf30013b5cdf39edca46e159ce9a599ec5a6f 100755 --- a/preproc/emep_preproc.py +++ b/preproc/emep_preproc.py @@ -30,10 +30,11 @@ Besides citing HERMESv3_GR, users must also acknowledge the use of the correspon """ # ============== CONFIGURATION PARAMETERS ====================== -INPUT_PATH = '/esarchive/recon/ceip/emepv18/original_files' -OUTPUT_PATH = '/esarchive/recon/ceip/emepv18/yearly_mean' +INPUT_PATH = '/esarchive/recon/ceip/emepv19/original_files' +OUTPUT_PATH = '/esarchive/recon/ceip/emepv19/yearly_mean' INPUT_NAME = '__2018_GRID_.txt' -# list_years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016] +# list_years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, +# 2017] LIST_YEARS = [2015] LIST_POLLUTANTS = ['NOx', 'NMVOC', 'SOx', 'NH3', 'PM2_5', 'PM10', 'CO'] # ============================================================== @@ -65,14 +66,14 @@ def calculate_grid_definition(in_path): # Longitudes lons = np.sort(np.unique(df.LONGITUDE)) lons_interval = lons[1:] - lons[:-1] - print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - df.LONGITUDE.min(), df.LONGITUDE.max(), lons_interval.min(), len(lons)) + print('Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( + df.LONGITUDE.min(), df.LONGITUDE.max(), lons_interval.min(), len(lons))) # Latitudes lats = np.sort(np.unique(df.LATITUDE)) lats_interval = lats[1:] - lats[:-1] - print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - df.LATITUDE.min(), df.LATITUDE.max(), lats_interval.min(), len(lats)) + print('Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( + df.LATITUDE.min(), df.LATITUDE.max(), lats_interval.min(), len(lats))) lats = np.arange(-90 + lats_interval.min()/2, 90, lats_interval.min(), dtype=np.float64) lons = np.arange(-180 + lons_interval.min()/2, 180, lons_interval.min(), dtype=np.float64) @@ -96,7 +97,7 @@ def do_transformation(year): INPUT_NAME.replace('', str(year)).replace('', sector).replace('', pollutant)) if os.path.exists(in_file): - print in_file + print(in_file) c_lats, c_lons, lat_interval, lon_interval = calculate_grid_definition(in_file) b_lats = create_bounds(c_lats, number_vertices=2) b_lons = create_bounds(c_lons, number_vertices=2) @@ -105,6 +106,8 @@ def do_transformation(year): name = 'nox_no2' elif name == 'pm2_5': name = 'pm25' + elif name == 'sox': + name = 'so2' elif name == 'voc': name = 'nmvoc' diff --git a/preproc/gfas12_h_preproc.py b/preproc/gfas12_h_preproc.py index 9cc93fef153e21e8666355ab350c815145f1cf56..e38d1d3b6f86ec88109841b3a992e39e181a6343 100755 --- a/preproc/gfas12_h_preproc.py +++ b/preproc/gfas12_h_preproc.py @@ -37,8 +37,8 @@ INPUT_PATH = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/' INPUT_NAME = 'gfas_hourly_.grb' OUTPUT_PATH = '/esarchive/recon/ecmwf/gfas' -STARTING_DATE = datetime(year=2018, month=11, day=01) -ENDIND_DATE = datetime(year=2018, month=11, day=01) +STARTING_DATE = datetime(year=2018, month=11, day=0o1) +ENDIND_DATE = datetime(year=2018, month=11, day=0o1) PARAMETERS_FILE = '/esarchive/recon/ecmwf/gfas/original_files/ga_mc_sfc_gfas_ecmf/GFAS_hourly_Parameters.csv' # ============================================================== @@ -118,7 +118,7 @@ def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cel :type date: datetime.datetime """ - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -277,6 +277,6 @@ if __name__ == '__main__': if os.path.isfile(f): do_transformation(f, date_aux, OUTPUT_PATH, var_list) else: - print 'ERROR: file {0} not found'.format(f) + print('ERROR: file {0} not found'.format(f)) date_aux = date_aux + timedelta(days=1) diff --git a/preproc/gfas12_preproc.py b/preproc/gfas12_preproc.py index 328cbc04fe86e2693536b63a7a62af2109f644c2..86ffcf9814641b344a4ef5744a6912cacb11e365 100755 --- a/preproc/gfas12_preproc.py +++ b/preproc/gfas12_preproc.py @@ -118,7 +118,7 @@ def write_netcdf(output_name_path, data_list, center_lats, center_lons, grid_cel :type date: datetime.datetime """ - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -276,6 +276,6 @@ if __name__ == '__main__': if os.path.isfile(f): do_transformation(f, date_aux, OUTPUT_PATH, var_list) else: - print 'ERROR: file {0} not found'.format(f) + print('ERROR: file {0} not found'.format(f)) date_aux = date_aux + timedelta(days=1) diff --git a/preproc/htapv2_preproc.py b/preproc/htapv2_preproc.py index a81aebe5a968ff8e068bff24918a984008d13fec..284fd6dec94ae87003e43f87db0e9d73a802022d 100755 --- a/preproc/htapv2_preproc.py +++ b/preproc/htapv2_preproc.py @@ -73,7 +73,7 @@ def do_transformation_annual(filename, out_path, pollutant, sector, year): """ from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds - print filename + print(filename) [c_lats, c_lons] = extract_vars(filename, ['lat', 'lon']) if pollutant == 'pm25': @@ -108,7 +108,7 @@ def do_transformation_annual(filename, out_path, pollutant, sector, year): os.makedirs(out_path) out_path = os.path.join(out_path, '{0}_{1}.nc'.format(pollutant, year)) - print out_path + print(out_path) write_netcdf(out_path, c_lats['data'], c_lons['data'], [data], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), cell_area=get_grid_area(filename), global_attributes=global_attributes,) @@ -139,8 +139,8 @@ def do_transformation(filename_list, out_path, pollutant, sector, year): from hermesv3_gr.tools.netcdf_tools import extract_vars, write_netcdf, get_grid_area from hermesv3_gr.tools.coordinates_tools import create_bounds - for month in xrange(1, 13): - print filename_list[month - 1] + for month in range(1, 13): + print(filename_list[month - 1]) [c_lats, c_lons] = extract_vars(filename_list[month - 1], ['lat', 'lon']) if pollutant == 'pm25': @@ -239,7 +239,7 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): nmvoc_ratio_list = do_ratio_list() - print sector + print(sector) if sector == 'ENERGY': ratio_var = 'pow' @@ -263,16 +263,16 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): nmvoc_ratio_list.pop('voc24', None) nmvoc_ratio_list.pop('voc25', None) - print type(nmvoc_ratio_list), nmvoc_ratio_list + print(type(nmvoc_ratio_list), nmvoc_ratio_list) - for month in xrange(1, 13): - print filename_list[month - 1] + for month in range(1, 13): + print(filename_list[month - 1]) c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) [data] = extract_vars(filename_list[month - 1], ['emi_nmvoc']) - for voc, ratio_file in nmvoc_ratio_list.iteritems(): - print voc, ratio_file + for voc, ratio_file in nmvoc_ratio_list.items(): + print(voc, ratio_file) pollutant = voc [ratio] = extract_vars(ratio_file, [ratio_var]) @@ -303,7 +303,7 @@ def do_nmvoc_month_transformation(filename_list, out_path, sector, year): os.makedirs(out_path_aux) out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(pollutant, year, str(month).zfill(2))) - print out_path_aux + print(out_path_aux) write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) @@ -325,18 +325,18 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year nmvoc_ratio_list = do_ratio_list() - print sector + print(sector) - print type(nmvoc_ratio_list), nmvoc_ratio_list + print(type(nmvoc_ratio_list), nmvoc_ratio_list) - for month in xrange(1, 13): - print filename_list[month - 1] + for month in range(1, 13): + print(filename_list[month - 1]) c_lats, c_lons = extract_vars(filename_list[month - 1], ['lat', 'lon']) [ind, exf, sol] = extract_vars(filename_list[month - 1], ['emiss_ind', 'emiss_exf', 'emiss_sol']) - for voc, ratio_file in nmvoc_ratio_list.iteritems(): - print voc, ratio_file + for voc, ratio_file in nmvoc_ratio_list.items(): + print(voc, ratio_file) data = { 'name': voc, 'units': 'kg m-2 s-1', @@ -382,7 +382,7 @@ def do_nmvoc_industry_month_transformation(filename_list, out_path, sector, year os.makedirs(out_path_aux) out_path_aux = os.path.join(out_path_aux, '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) - print out_path_aux + print(out_path_aux) write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), global_attributes=global_attributes,) @@ -439,7 +439,7 @@ def do_nmvoc_year_transformation(filename, out_path, sector, year): os.makedirs(out_path_aux) out_path_aux = os.path.join(out_path_aux, '{0}_{1}.nc'.format(pollutant, year)) - print out_path_aux + print(out_path_aux) write_netcdf(out_path_aux, c_lats['data'], c_lons['data'], [data_aux], boundary_latitudes=create_bounds(c_lats['data']), boundary_longitudes=create_bounds(c_lons['data']), @@ -510,7 +510,7 @@ def check_vocs(year): :return: """ from hermesv3_gr.tools.netcdf_tools import extract_vars - for month in xrange(1, 12 + 1, 1): + for month in range(1, 12 + 1, 1): for snap in ['ENERGY', 'INDUSTRY', 'RESIDENTIAL', 'TRANSPORT']: nmvoc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', 'nmvoc_{0}'.format(snap.lower()), 'nmvoc_{0}{1}.nc'.format(year, str(month).zfill(2))) @@ -518,26 +518,26 @@ def check_vocs(year): nmvoc_sum = new_voc['data'].sum() voc_sum = 0 - for voc in ['voc{0}'.format(str(x).zfill(2)) for x in xrange(1, 25 + 1, 1)]: + for voc in ['voc{0}'.format(str(x).zfill(2)) for x in range(1, 25 + 1, 1)]: voc_path = os.path.join(OUTPUT_PATH, 'monthly_mean', '{0}_{1}'.format(voc, snap.lower()), '{0}_{1}{2}.nc'.format(voc, year, str(month).zfill(2))) if os.path.exists(voc_path): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum) / nmvoc_sum, month) + print('{0} month: {4}; NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + snap, nmvoc_sum, voc_sum, 100 * (nmvoc_sum - voc_sum) / nmvoc_sum, month)) if __name__ == '__main__': for y in LIST_YEARS: - for pollutant_dict in get_pollutant_dict().iteritems(): + for pollutant_dict in get_pollutant_dict().items(): for current_sector in get_sector_dict()[pollutant_dict[0]]['month']: input_name_aux = INPUT_NAME.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) input_name_aux = input_name_aux.replace('', pollutant_dict[1]) file_list = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) - for aux_month in xrange(1, 13)] + for aux_month in range(1, 13)] do_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), pollutant_dict[0], current_sector, y) @@ -564,7 +564,7 @@ if __name__ == '__main__': input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) file_list = [os.path.join(INPUT_PATH, input_name_aux.replace('', str(aux_month))) - for aux_month in xrange(1, 13)] + for aux_month in range(1, 13)] if current_sector == 'INDUSTRY_3subsectors': do_nmvoc_industry_month_transformation(file_list, os.path.join(OUTPUT_PATH, 'monthly_mean'), @@ -580,5 +580,5 @@ if __name__ == '__main__': input_name_aux = input_name_aux.replace('', current_sector) input_name_aux = input_name_aux.replace('', str(y)) input_name_aux = os.path.join(INPUT_PATH, input_name_aux) - print input_name_aux + print(input_name_aux) do_nmvoc_year_transformation(input_name_aux, os.path.join(OUTPUT_PATH, 'yearly_mean'), current_sector, y) diff --git a/preproc/tno_mac_iii_preproc.py b/preproc/tno_mac_iii_preproc.py index e96da9f64c991bc4c11a36852246f40e09a6931f..3b447d6db08041c7e996d5070bed45a758f782d2 100755 --- a/preproc/tno_mac_iii_preproc.py +++ b/preproc/tno_mac_iii_preproc.py @@ -73,14 +73,14 @@ def calculate_grid_definition(in_path): # Longitudes lons = np.sort(np.unique(dataframe.Lon)) lons_interval = lons[1:] - lons[:-1] - print 'Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( - dataframe.Lon.min(), dataframe.Lon.max(), lons_interval.min(), len(lons)) + print('Lon min: {0}; Lon max: {1}; Lon inc: {2}; Lon num: {3}'.format( + dataframe.Lon.min(), dataframe.Lon.max(), lons_interval.min(), len(lons))) # Latitudes lats = np.sort(np.unique(dataframe.Lat)) lats_interval = lats[1:] - lats[:-1] - print 'Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( - dataframe.Lat.min(), dataframe.Lat.max(), lats_interval.min(), len(lats)) + print('Lat min: {0}; Lat max: {1}; Lat inc: {2}; Lat num: {3}'.format( + dataframe.Lat.min(), dataframe.Lat.max(), lats_interval.min(), len(lats))) lats = np.arange(-90 + lats_interval.min() / 2, 90, lats_interval.min(), dtype=np.float64) lons = np.arange(-180 + lons_interval.min() / 2, 180, lons_interval.min(), dtype=np.float64) @@ -152,8 +152,8 @@ def do_transformation(year): dataframe = pd.read_table(in_file, sep=';') - df_np = dataframe[dataframe.SourceType != 'P'] - df_p = dataframe[dataframe.SourceType == 'P'] + df_np = dataframe[dataframe.SourceType != 'P'].copy() + df_p = dataframe[dataframe.SourceType == 'P'].copy() df_np.loc[:, 'row_lat'] = np.array((df_np.Lat - (-90 + lat_interval / 2)) / lat_interval, dtype=np.int32) df_np.loc[:, 'col_lon'] = np.array((df_np.Lon - (-180 + lon_interval / 2)) / lon_interval, dtype=np.int32) @@ -164,7 +164,7 @@ def do_transformation(year): dataframe = pd.concat([df_np, df_p]) for name, group in dataframe.groupby('SNAP'): - print 'snap', name + print('snap', name) pollutant_list = create_pollutant_empty_list(in_file, len(c_lats), len(c_lons)) # Other mobile sources ignoring sea cells (shipping emissions) @@ -174,7 +174,7 @@ def do_transformation(year): group = group.groupby(['row_lat', 'col_lon']).sum().reset_index() - for i in xrange(len(pollutant_list)): + for i in range(len(pollutant_list)): # pollutant_list[i]['data'][group.col_lon, group.row_lat] = group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'][group.row_lat, group.col_lon] += group[pollutant_list[i]['TNO_name']] pollutant_list[i]['data'] = pollutant_list[i]['data'].reshape((1,) + pollutant_list[i]['data'].shape) @@ -362,8 +362,8 @@ def check_vocs(year): [new_voc] = extract_vars(voc_path, [voc]) voc_sum += new_voc['data'].sum() - print '{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( - snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum) + print('{0} NMVOC sum: {1}; VOCs sum: {2}; %diff: {3}'.format( + snap, nmvoc_sum, voc_sum, 100*(nmvoc_sum - voc_sum) / nmvoc_sum)) return True diff --git a/preproc/tno_mac_iii_preproc_voc_ratios.py b/preproc/tno_mac_iii_preproc_voc_ratios.py index 9aea92690893c217c28bd069905ab604ff598147..80b41feeb77e95e89e41bf9aaa1c8c7582f36c62 100755 --- a/preproc/tno_mac_iii_preproc_voc_ratios.py +++ b/preproc/tno_mac_iii_preproc_voc_ratios.py @@ -101,7 +101,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, if not (regular_latlon or lcc or rotated): regular_latlon = True - print netcdf_path + print(netcdf_path) netcdf = Dataset(netcdf_path, mode='w', format="NETCDF4") # ===== Dimensions ===== @@ -116,7 +116,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lat', center_latitudes.shape[0]) lat_dim = ('lon', 'lat',) else: - print 'ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape)) + print('ERROR: Latitudes must be on a 1D or 2D array instead of {0}'.format(len(center_latitudes.shape))) sys.exit(1) # Longitude @@ -127,21 +127,21 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, netcdf.createDimension('lon', center_longitudes.shape[1]) lon_dim = ('lon', 'lat',) else: - print 'ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape)) + print('ERROR: Longitudes must be on a 1D or 2D array instead of {0}'.format(len(center_longitudes.shape))) sys.exit(1) elif rotated: var_dim = ('rlat', 'rlon',) # Rotated Latitude if rotated_lats is None: - print 'ERROR: For rotated grids is needed the rotated latitudes.' + print('ERROR: For rotated grids is needed the rotated latitudes.') sys.exit(1) netcdf.createDimension('rlat', len(rotated_lats)) lat_dim = ('rlat', 'rlon',) # Rotated Longitude if rotated_lons is None: - print 'ERROR: For rotated grids is needed the rotated longitudes.' + print('ERROR: For rotated grids is needed the rotated longitudes.') sys.exit(1) netcdf.createDimension('rlon', len(rotated_lons)) lon_dim = ('rlat', 'rlon',) @@ -215,7 +215,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, lons.axis = "X" lons.long_name = "longitude coordinate" lons.standard_name = "longitude" - print 'lons:', lons[:].shape, center_longitudes.shape + print('lons:', lons[:].shape, center_longitudes.shape) lons[:] = center_longitudes if boundary_longitudes is not None: lons.bounds = "lon_bnds" @@ -287,7 +287,7 @@ def write_netcdf(netcdf_path, center_latitudes, center_longitudes, data_list, try: var[:] = variable['data'] except Exception: - print 'VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape) + print('VAR ERROR, netcdf shape: {0}, variable shape: {1}'.format(var[:].shape, variable['data'].shape)) # Grid mapping if regular_latlon: @@ -388,19 +388,19 @@ def create_voc_ratio(voc): import numpy as np [country_values, lat, lon] = extract_vars(TNO_WORLD_MASK, ['timezone_id', 'lat', 'lon']) country_values = country_values['data'].reshape((country_values['data'].shape[1], country_values['data'].shape[1])) - print OUTPUT_PATH + print(OUTPUT_PATH) if not os.path.exists(OUTPUT_PATH): os.makedirs(OUTPUT_PATH) complete_output_path = os.path.join(OUTPUT_PATH, 'ratio_{0}.nc'.format(voc)) if not os.path.exists(complete_output_path): - print 'Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path) + print('Creating ratio file for {0}\npath: {1}'.format(voc, complete_output_path)) data_list = [] for snap in get_sector_list(voc): - print snap + print(snap) mask_factor = np.zeros(country_values.shape) iso_codes = get_iso_codes() - for country_code, factor in get_country_code_and_factor(voc, snap).iteritems(): + for country_code, factor in get_country_code_and_factor(voc, snap).items(): try: mask_factor[country_values == iso_codes[country_code]] = factor except Exception: @@ -415,7 +415,7 @@ def create_voc_ratio(voc): }) write_netcdf(complete_output_path, lat['data'], lon['data'], data_list) else: - print 'Ratio file for {0} already created\npath: {1}'.format(voc, complete_output_path) + print('Ratio file for {0} already created\npath: {1}'.format(voc, complete_output_path)) return True @@ -468,7 +468,7 @@ def get_voc_list(): del df['ISO3'], df['snap'], df['output substance name'], df['fr'] df = df.drop_duplicates().dropna() voc_list = df.vcode.values - for i in xrange(len(voc_list)): + for i in range(len(voc_list)): voc_list[i] = voc_list[i].replace('v', 'voc') return df.vcode.values diff --git a/preproc/wiedinmyer_preproc.py b/preproc/wiedinmyer_preproc.py index 7ccdab724b60607e79235aa0d42c0ffe22bf5f6e..fd0344b72437bba0eea624aa2b038248729001e9 100755 --- a/preproc/wiedinmyer_preproc.py +++ b/preproc/wiedinmyer_preproc.py @@ -81,7 +81,7 @@ def do_transformation(filename): :type filename: str """ import numpy as np - print filename + print(filename) from hermesv3_gr.tools.netcdf_tools import get_grid_area from cf_units import Unit @@ -101,10 +101,10 @@ def do_transformation(filename): for output_pollutant in LIST_POLLUTANTS: input_pollutant = out_pollutant_to_in_pollutant(output_pollutant) try: - print input_pollutant + print(input_pollutant) data = nc_in.variables[input_pollutant][:] except RuntimeWarning: - print 'ERROR reading {0}'.format(input_pollutant) + print('ERROR reading {0}'.format(input_pollutant)) data = np.nan_to_num(data) data = data/grid_area # To pass from Gg/year to Gg/m2.year data = data*factor @@ -119,7 +119,7 @@ def do_transformation(filename): if not os.path.exists(out_path_aux): os.makedirs(out_path_aux) write_netcdf(os.path.join(out_path_aux, '{0}_{1}.nc'.format(output_pollutant, YEAR)), - data, data_attributes, lats, lons, grid_area, YEAR, 01) + data, data_attributes, lats, lons, grid_area, YEAR, 0o1) nc_in.close() @@ -157,7 +157,7 @@ def write_netcdf(output_name_path, data, data_atts, center_lats, center_lons, gr """ from hermesv3_gr.tools.coordinates_tools import create_bounds - print output_name_path + print(output_name_path) # Creating NetCDF & Dimensions nc_output = Dataset(output_name_path, mode='w', format="NETCDF4") nc_output.createDimension('nv', 2) @@ -243,4 +243,4 @@ if __name__ == '__main__': do_transformation(os.path.join(INPUT_PATH, INPUT_NAME)) - print 'Time(s):', timeit.default_timer() - starting_time + print('Time(s):', timeit.default_timer() - starting_time) diff --git a/run_test.py b/run_test.py index 6eb2926793f63a9c62eee79c06273d12f95fab82..655465f0f26d182068b5ffe928597a593a781365 100755 --- a/run_test.py +++ b/run_test.py @@ -1,5 +1,4 @@ # coding=utf-8 -"""Script to run the tests for EarthDiagnostics and generate the code coverage report""" import os import sys diff --git a/setup.py b/setup.py index 34e4e16dd1182e6836bbe496e33c5f8d1e29ac8a..2050aedae17edbe8de56b1fb020fe03485034865 100755 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ setup( 'ESMPy>=7.1.0.dev0', 'holidays', 'pytz', - 'timezonefinder<4.0.0', + 'timezonefinder>=4.0.0', 'mpi4py', 'pytest', ],