From 78e6b11a1ccb160212b5319a524ee94ca9e31b96 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 5 Mar 2019 16:52:24 +0100 Subject: [PATCH 01/11] Added 365 daily profile --- data/profiles/temporal/TemporalProfile_Daily.csv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 data/profiles/temporal/TemporalProfile_Daily.csv diff --git a/data/profiles/temporal/TemporalProfile_Daily.csv b/data/profiles/temporal/TemporalProfile_Daily.csv new file mode 100644 index 0000000..2106568 --- /dev/null +++ b/data/profiles/temporal/TemporalProfile_Daily.csv @@ -0,0 +1,2 @@ +TP_D,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365 +D001,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 -- GitLab From 878c7e2705549bfc9995cc2b8d3c24cdf592b685 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 5 Mar 2019 16:52:53 +0100 Subject: [PATCH 02/11] Changed version --- CHANGELOG | 8 +++++++- hermesv3_gr/__init__.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d42afd3..c9ed59b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,4 +7,10 @@ 2019/03/01 HEMRESv3_GR stable version - GFAS emission inventory as point source emission. \ No newline at end of file + GFAS emission inventory as point source emission. + +1.X.X + 2019/XX/XX + + Temporal gridded profiles (CAMS-81 products) (dst resolution) + diff --git a/hermesv3_gr/__init__.py b/hermesv3_gr/__init__.py index 6c8e6b9..5becc17 100644 --- a/hermesv3_gr/__init__.py +++ b/hermesv3_gr/__init__.py @@ -1 +1 @@ -__version__ = "0.0.0" +__version__ = "1.0.0" -- GitLab From 5d4f4d9c680406f020cb17aba9ea441992bfef7e Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 5 Mar 2019 17:25:20 +0100 Subject: [PATCH 03/11] Same column ID name (before was different name for each profile type) --- data/profiles/temporal/TemporalProfile_Daily.csv | 2 +- data/profiles/temporal/TemporalProfile_Hourly.csv | 2 +- data/profiles/temporal/TemporalProfile_Monthly.csv | 2 +- data/profiles/temporal/TemporalProfile_Weekly.csv | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/profiles/temporal/TemporalProfile_Daily.csv b/data/profiles/temporal/TemporalProfile_Daily.csv index 2106568..4a1893b 100644 --- a/data/profiles/temporal/TemporalProfile_Daily.csv +++ b/data/profiles/temporal/TemporalProfile_Daily.csv @@ -1,2 +1,2 @@ -TP_D,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365 +ID,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365 D001,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 diff --git a/data/profiles/temporal/TemporalProfile_Hourly.csv b/data/profiles/temporal/TemporalProfile_Hourly.csv index 857270e..28dad6b 100644 --- a/data/profiles/temporal/TemporalProfile_Hourly.csv +++ b/data/profiles/temporal/TemporalProfile_Hourly.csv @@ -1,4 +1,4 @@ -TP_H,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23 +ID,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23 H001,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1. H002,0.79,0.72,0.72,0.71,0.74,0.8,0.92,1.08,1.19,1.22,1.21,1.21,1.17,1.15,1.14,1.13,1.1,1.07,1.04,1.02,1.02,1.01,0.96,0.88 H003,0.38,0.36,0.36,0.36,0.37,0.69,1.19,1.53,1.57,1.56,1.35,1.16,1.07,1.06,1,0.98,0.99,1.12,1.41,1.52,1.39,1.35,1.19,0.42 diff --git a/data/profiles/temporal/TemporalProfile_Monthly.csv b/data/profiles/temporal/TemporalProfile_Monthly.csv index b66babd..f904688 100644 --- a/data/profiles/temporal/TemporalProfile_Monthly.csv +++ b/data/profiles/temporal/TemporalProfile_Monthly.csv @@ -1,4 +1,4 @@ -TP_M,1,2,3,4,5,6,7,8,9,10,11,12 +ID,1,2,3,4,5,6,7,8,9,10,11,12 M001,1,1,1,1,1,1,1,1,1,1,1,1 M002,1.2,1.15,1.05,1,0.9,0.85,0.8,0.875,0.95,1,1.075,1.15 M003,1.7,1.5,1.3,1,0.7,0.4,0.2,0.4,0.7,1.05,1.4,1.65 diff --git a/data/profiles/temporal/TemporalProfile_Weekly.csv b/data/profiles/temporal/TemporalProfile_Weekly.csv index 5d9b66a..c96b586 100644 --- a/data/profiles/temporal/TemporalProfile_Weekly.csv +++ b/data/profiles/temporal/TemporalProfile_Weekly.csv @@ -1,4 +1,4 @@ -TP_W,0,1,2,3,4,5,6 +ID,0,1,2,3,4,5,6 W001,1,1,1,1,1,1,1 W002,1.06,1.06,1.06,1.06,1.06,0.85,0.85 W003,1.08,1.08,1.08,1.08,1.08,0.8,0.8 -- GitLab From 65fbc82d9f1cb2872a0a9082973c21f7d1ac86bc Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 5 Mar 2019 18:44:11 +0100 Subject: [PATCH 04/11] Re-implementing temporal profiles --- conf/hermes.conf | 1 + .../temporal/TemporalProfile_Daily.csv | 1 + hermesv3_gr/config/config.py | 1 + .../emision_inventories/emission_inventory.py | 46 ++++-- hermesv3_gr/modules/temporal/temporal.py | 133 ++++++++++++------ tests/unit/test_temporal.py | 2 +- 6 files changed, 131 insertions(+), 53 deletions(-) diff --git a/conf/hermes.conf b/conf/hermes.conf index 0b1e1fc..56d26d2 100644 --- a/conf/hermes.conf +++ b/conf/hermes.conf @@ -71,6 +71,7 @@ cross_table = /conf/EI_configuration.csv p_vertical = /data/profiles/vertical/Vertical_profile.csv p_month = /data/profiles/temporal/TemporalProfile_Monthly.csv +p_week = /data/profiles/temporal/TemporalProfile_Weekly.csv p_day = /data/profiles/temporal/TemporalProfile_Daily.csv p_hour = /data/profiles/temporal/TemporalProfile_Hourly.csv p_speciation = /data/profiles/speciation/Speciation_profile_cb05_aero5_MONARCH.csv diff --git a/data/profiles/temporal/TemporalProfile_Daily.csv b/data/profiles/temporal/TemporalProfile_Daily.csv index 4a1893b..bb647a6 100644 --- a/data/profiles/temporal/TemporalProfile_Daily.csv +++ b/data/profiles/temporal/TemporalProfile_Daily.csv @@ -1,2 +1,3 @@ ID,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365 D001,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +D002,0.5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 diff --git a/hermesv3_gr/config/config.py b/hermesv3_gr/config/config.py index c568e4a..b61857d 100644 --- a/hermesv3_gr/config/config.py +++ b/hermesv3_gr/config/config.py @@ -141,6 +141,7 @@ class Config(ArgParser): p.add_argument('--p_month', required=True, help='Path to the file that contains all the needed monthly profiles.') p.add_argument('--p_week', required=True, help='Path to the file that contains all the needed weekly profiles.') + p.add_argument('--p_day', required=True, help='Path to the file that contains all the needed daily profiles.') p.add_argument('--p_hour', required=True, help='Path to the file that contains all the needed hourly profiles.') p.add_argument('--p_speciation', required=True, help='Path to the file that contains all the needed speciation profiles.') diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index 4a899f9..87fe767 100644 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -76,7 +76,7 @@ class EmissionInventory(object): """ def __init__(self, options, grid, current_date, inventory_name, source_type, sector, pollutants, inputs_path, input_frequency, vertical_output_profile, reference_year=2010, factors=None, regrid_mask=None, - p_vertical=None, p_month=None, p_week=None, p_hour=None, p_speciation=None): + p_vertical=None, p_month=None, p_week=None, p_day=None, p_hour=None, p_speciation=None): from hermesv3_gr.modules.masking.masking import Masking st_time = timeit.default_timer() @@ -135,8 +135,8 @@ class EmissionInventory(object): if not((p_month is None) and (p_week is None) and (p_hour is None)): self.temporal = TemporalDistribution( current_date, options.output_timestep_type, options.output_timestep_num, options.output_timestep_freq, - options.p_month, p_month, options.p_week, p_week, options.p_hour, p_hour, options.world_info, - options.auxiliar_files_path, grid) + options.p_month, p_month, options.p_week, p_week, options.p_day, p_day, options.p_hour, p_hour, + options.world_info, options.auxiliar_files_path, grid) else: self.temporal = None settings.write_log('\t\tNone temporal profile set.', level=2) @@ -296,13 +296,15 @@ class EmissionInventory(object): path = options.cross_table df = pd.read_csv(path, sep=';', index_col=False) for column in ['ei', 'sector', 'ref_year', 'active', 'factor_mask', 'regrid_mask', 'pollutants', 'path', - 'frequency', 'source_type', 'p_vertical', 'p_month', 'p_week', 'p_hour', 'p_speciation']: + 'frequency', 'source_type', 'p_vertical', 'p_month', 'p_week', 'p_day', 'p_hour', + 'p_speciation']: df_cols = list(df.columns.values) if column not in df_cols: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise AttributeError('ERROR: Column {0} is not in the {1} file.'.format(column, path)) sys.exit(1) + df = df[df['active'] == 1] num = 1 emission_inventory_list = [] @@ -312,11 +314,30 @@ class EmissionInventory(object): num += 1 pollutants = list(map(str, re.split(', |,|; |;| ', emission_inventory.pollutants))) + #MONTHLY try: # gridded temporal profile p_month = emission_inventory.p_month.replace('', options.input_dir) except AttributeError: p_month = emission_inventory.p_month + # WEEKLY + try: + # gridded temporal profile + p_week = emission_inventory.p_week.replace('', options.input_dir) + except AttributeError: + p_week = emission_inventory.p_week + # DAILY + try: + # gridded temporal profile + p_day = emission_inventory.p_day.replace('', options.input_dir) + except AttributeError: + p_day = emission_inventory.p_day + # HOURLY + try: + # gridded temporal profile + p_hour = emission_inventory.p_hour.replace('', options.input_dir) + except AttributeError: + p_hour = emission_inventory.p_hour emission_inventory_path = emission_inventory.path.replace('', options.data_path) emission_inventory_path = emission_inventory_path.replace('', options.input_dir) @@ -333,8 +354,9 @@ class EmissionInventory(object): regrid_mask=emission_inventory.regrid_mask, p_vertical=emission_inventory.p_vertical, p_month=p_month, - p_week=emission_inventory.p_week, - p_hour=emission_inventory.p_hour, + p_week=p_week, + p_day=p_day, + p_hour=p_hour, p_speciation=emission_inventory.p_speciation)) else: emission_inventory_list.append( @@ -347,8 +369,9 @@ class EmissionInventory(object): regrid_mask=emission_inventory.regrid_mask, p_vertical=emission_inventory.p_vertical, p_month=p_month, - p_week=emission_inventory.p_week, - p_hour=emission_inventory.p_hour, + p_week=p_week, + p_day=p_day, + p_hour=p_hour, p_speciation=emission_inventory.p_speciation)) elif emission_inventory.source_type == 'point': if emission_inventory.ei == 'GFASv12': @@ -359,7 +382,7 @@ class EmissionInventory(object): emission_inventory.frequency, vertical_output_profile, reference_year=emission_inventory.ref_year, factors=emission_inventory.factor_mask, regrid_mask=emission_inventory.regrid_mask, p_vertical=emission_inventory.p_vertical, - p_month=p_month, p_week=emission_inventory.p_week, p_hour=emission_inventory.p_hour, + p_month=p_month, p_week=p_week, p_day=p_day, p_hour=p_hour, p_speciation=emission_inventory.p_speciation)) else: emission_inventory_list.append( @@ -372,8 +395,9 @@ class EmissionInventory(object): regrid_mask=emission_inventory.regrid_mask, p_vertical=emission_inventory.p_vertical, p_month=p_month, - p_week=emission_inventory.p_week, - p_hour=emission_inventory.p_hour, + p_week=p_week, + p_day=p_day, + p_hour=p_hour, p_speciation=emission_inventory.p_speciation)) else: settings.write_log('ERROR: Check the .err file to get more info.') diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 9341229..bc5305d 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -68,8 +68,8 @@ class TemporalDistribution(object): :type auxiliar_files_dir: str """ def __init__(self, starting_date, timestep_type, timestep_num, timestep_freq, monthly_profile_path, - month_profile_id, weekly_profile_path, weekly_profile_id, hourly_profile_path, hourly_profile_id, - world_info_path, auxiliar_files_dir, grid): + monthly_profile_id, weekly_profile_path, weekly_profile_id, daily_profile_path, daily_profile_id, + hourly_profile_path, hourly_profile_id, world_info_path, auxiliar_files_dir, grid): from timezonefinder import TimezoneFinder import pandas as pd @@ -86,45 +86,18 @@ class TemporalDistribution(object): self.timestep_freq = timestep_freq self.ending_date = self.calculate_ending_date() - if month_profile_id is not None: - if len(month_profile_id) > 4: - if os.path.exists(month_profile_id): - self.monthly_profile = self.read_gridded_profile(month_profile_id, 'Fmonth') - self.monthly_profile = self.monthly_profile.reshape( - (self.monthly_profile.shape[0], self.monthly_profile.shape[1] * self.monthly_profile.shape[2])) - - self.monthly_profile_path = month_profile_id - else: - settings.write_log('ERROR: Check the .err file to get more info.') - if settings.rank == 0: - raise IOError('ERROR: Gridded monthly profile file not found: {0}'.format(month_profile_id)) - sys.exit(1) - else: - self.monthly_profile = self.get_temporal_monthly_profile(monthly_profile_path, month_profile_id) - else: - self.monthly_profile = None - settings.write_log("\t\t\tNo temporal monthly profile is set.", level=3) - self.weekly_profile_id = weekly_profile_id - self.weekly_profile_path = weekly_profile_path - if weekly_profile_id is not None: - settings.write_log("\t\t\tGetting temporal daily profile id '{0}' from {1} .".format( - weekly_profile_id, weekly_profile_path), level=3) + # print monthly_profile_path, monthly_profile_id, weekly_profile_path, weekly_profile_id, daily_profile_path, daily_profile_id, hourly_profile_path, hourly_profile_id + # sys.exit() + self.daily_profile = self.get_daily_profile(daily_profile_id, daily_profile_path) + if self.daily_profile is not None: + self.monthly_profile = self.get_monthly_profile(monthly_profile_id, monthly_profile_path) + self.weekly_profile = self.get_weekly_profile(weekly_profile_id, weekly_profile_path) else: - settings.write_log("\t\t\tNo temporal daily profile is set.", level=3) + self.monthly_profile = None + self.weekly_profile = None - self.hourly_profile_path = hourly_profile_path - if hourly_profile_id is not None and len(hourly_profile_id) is 4: - self.hourly_profile = self.get_temporal_hourly_profile(hourly_profile_id) - settings.write_log("\t\t\tGetting temporal hourly profile id '{0}' from {1} .".format( - hourly_profile_id, hourly_profile_path), level=3) - else: - if hourly_profile_id is None: - settings.write_log("\t\t\tNo temporal hourly profile is set.", level=3) - else: - settings.write_log("\t\t\tGetting temporal hourly profile ids {0} from {1} .".format( - hourly_profile_id, hourly_profile_path), level=3) - self.hourly_profile = hourly_profile_id + self.hourly_profile = self.get_hourly_profile(hourly_profile_id, hourly_profile_path) self.world_info = world_info_path self.netcdf_timezones = os.path.join(auxiliar_files_dir, 'timezones.nc') @@ -141,6 +114,84 @@ class TemporalDistribution(object): settings.write_time('TemporalDistribution', 'Init', timeit.default_timer() - st_time, level=2) + def get_csv_temporal_profile(self, profile_id, profile_path): + import pandas as pd + profile = pd.read_csv(profile_path) + profile = profile.loc[profile['ID'] == profile_id, :].drop(columns='ID') + if len(profile) is 0: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: Profile ID {0} is not in the {1} file.'.format(profile_id, profile_path)) + sys.exit(1) + elif len(profile) > 1: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: Profile ID {0} is repeated {2} times in the {1} file.'.format( + profile_id, profile_path, len(profile))) + sys.exit(1) + return profile.reset_index(drop=True) + + def get_gridded_temporal_profile(self, profile_value, profile_path): + from netCDF4 import Dataset + nc_in = Dataset(profile_path, mode='r') + try: + var = nc_in.variables[profile_value][:] + except KeyError: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: The variable {0} is not in the {1} file.'.format( + profile_value, profile_path)) + sys.exit(1) + + return var + + def get_daily_profile(self, profile_id, profile_path): + import calendar + + if calendar.isleap(self.starting_date.year): + days_of_year = 366 + else: + days_of_year = 365 + + try: + if np.isnan(profile_id): + profile = None + except TypeError: + if os.path.exists(profile_id): + profile = self.get_gridded_temporal_profile('FD', profile_id) + profile_sum = profile.sum() / (profile.shape[-1] * profile.shape[-2]) + + elif len(profile_id) is 4: + profile = self.get_csv_temporal_profile(profile_id, profile_path) + profile_sum = profile.T[0].sum() + profile = profile.to_dict('records') + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: An error has occured in the profile {0}.'.format(profile_id)) + sys.exit(1) + + # CHECK if it is nomalized + if round(days_of_year, 2) != round(profile_sum, 2): + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: The sum of the temporal weight factors is not {0}.'.format( + days_of_year)) + sys.exit(1) + print profile + sys.exit() + return profile + + + def get_monthly_profile(self, profile_id, profile_path): + pass + + def get_weekly_profile(self, profile_id, profile_path): + pass + + def get_hourly_profile(self, profile_id, profile_path): + pass + def calculate_ending_date(self): """ Calculate the date of the last timestep. @@ -475,7 +526,7 @@ class TemporalDistribution(object): del df['hour'] if self.weekly_profile_id is not None: - daily_profile = self.get_temporal_daily_profile(date) + daily_profile = self.get_temporal_weekly_profile(date) df['week_factor'] = df['weekday'].map(daily_profile) else: df['week_factor'] = 1 @@ -595,7 +646,7 @@ class TemporalDistribution(object): return profile - def get_temporal_daily_profile(self, date): + def get_temporal_weekly_profile(self, date): """ Extract the daily profile of the given ID in a dictionary format. @@ -628,7 +679,7 @@ class TemporalDistribution(object): else: profile = None - settings.write_time('TemporalDistribution', 'get_temporal_daily_profile', timeit.default_timer() - st_time, + settings.write_time('TemporalDistribution', 'get_temporal_weekly_profile', timeit.default_timer() - st_time, level=3) return profile diff --git a/tests/unit/test_temporal.py b/tests/unit/test_temporal.py index 6174229..093b55f 100644 --- a/tests/unit/test_temporal.py +++ b/tests/unit/test_temporal.py @@ -425,4 +425,4 @@ class TestTemporalDistribution(unittest.TestCase): # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') # - # print temporal.get_temporal_daily_profile(date) + # print temporal.get_temporal_weekly_profile(date) -- GitLab From 3de4c3e53415d52233bca941b684df64333466c2 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Wed, 6 Mar 2019 15:59:38 +0100 Subject: [PATCH 05/11] WIP --- .gitignore | 2 ++ hermesv3_gr/modules/temporal/temporal.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c10666e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea +*.pyc diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index bc5305d..786b42c 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -175,8 +175,8 @@ class TemporalDistribution(object): if round(days_of_year, 2) != round(profile_sum, 2): settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: - raise AttributeError('ERROR: The sum of the temporal weight factors is not {0}.'.format( - days_of_year)) + raise AttributeError('ERROR: The sum of the temporal weight factors is not {0} (is {1}).'.format( + days_of_year, profile_sum)) sys.exit(1) print profile sys.exit() -- GitLab From e8d2790ccc1719b1ec6f10d035c47166d3bc4d62 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Thu, 7 Mar 2019 19:17:55 +0100 Subject: [PATCH 06/11] REady to test --- conf/EI_configuration.csv | 2 +- .../temporal/TemporalProfile_Hourly.csv | 2 +- .../temporal/TemporalProfile_Monthly.csv | 2 +- .../temporal/TemporalProfile_Weekly.csv | 2 +- .../emision_inventories/emission_inventory.py | 2 +- .../gfas_emission_inventory.py | 4 +- .../point_gfas_emission_inventory.py | 4 +- .../point_source_emission_inventory.py | 5 +- hermesv3_gr/modules/temporal/temporal.py | 957 ++++++++++++------ tests/unit/test_temporal.py | 2 +- 10 files changed, 643 insertions(+), 339 deletions(-) diff --git a/conf/EI_configuration.csv b/conf/EI_configuration.csv index e885b76..34a0ac3 100644 --- a/conf/EI_configuration.csv +++ b/conf/EI_configuration.csv @@ -1,4 +1,4 @@ -ei;sector;ref_year;active;factor_mask;regrid_mask;pollutants;path;frequency;source_type;p_vertical;p_month;p_week;p_hour;p_speciation;comment +ei;sector;ref_year;active;factor_mask;regrid_mask;pollutants;path;frequency;source_type;p_vertical;p_month;p_week;p_hour;p_speciation;p_day GFASv12;;2015;1;;;co,nox_no,pm25,oc,bc,so2,ch3oh,c2h5oh,c3h8,c2h4,c3h6,c5h8,terpenes,hialkenes,hialkanes,ch2o,c2h4o,c3h6o,nh3,c2h6s,c2h6,c7h8,c6h6,c8h10,c4h8,c5h10,c6h12,c8h16,c4h10,c5h12,c6h14,c7h16;/ecmwf/gfas/daily_mean;daily;point;method=sovief,approach=uniform;;;H001;E001; HTAPv2;energy;2010;1;;;co,nox_no2,pm10,pm25,oc,bc,so2,nh3,voc01,voc02,voc03,voc04,voc05,voc06,voc07,voc08,voc09,voc12,voc13,voc14,voc15,voc16,voc17,voc21,voc22,voc23,voc24;/jrc/htapv2/monthly_mean;monthly;area;V001;;W002;H002;E002; HTAPv2;industry;2010;1;;;co,nox_no2,pm10,pm25,oc,bc,so2,nh3,voc01,voc02,voc03,voc04,voc05,voc06,voc07,voc08,voc09,voc12,voc13,voc14,voc15,voc16,voc17,voc18,voc19,voc20,voc21,voc22,voc23,voc24;/jrc/htapv2/monthly_mean;monthly;area;V002;;W003;H004;E003; diff --git a/data/profiles/temporal/TemporalProfile_Hourly.csv b/data/profiles/temporal/TemporalProfile_Hourly.csv index 28dad6b..0434817 100644 --- a/data/profiles/temporal/TemporalProfile_Hourly.csv +++ b/data/profiles/temporal/TemporalProfile_Hourly.csv @@ -1,7 +1,7 @@ ID,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23 H001,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1. H002,0.79,0.72,0.72,0.71,0.74,0.8,0.92,1.08,1.19,1.22,1.21,1.21,1.17,1.15,1.14,1.13,1.1,1.07,1.04,1.02,1.02,1.01,0.96,0.88 -H003,0.38,0.36,0.36,0.36,0.37,0.69,1.19,1.53,1.57,1.56,1.35,1.16,1.07,1.06,1,0.98,0.99,1.12,1.41,1.52,1.39,1.35,1.19,0.42 +H003,0.38,0.36,0.36,0.36,0.37,0.5,1.19,1.53,1.57,1.56,1.35,1.16,1.07,1.06,1,0.98,0.99,1.12,1.41,1.52,1.39,1.35,1,0.42 H004,0.75,0.75,0.78,0.82,0.88,0.95,1.02,1.09,1.16,1.22,1.28,1.3,1.22,1.24,1.25,1.16,1.08,1.01,0.95,0.9,0.85,0.81,0.78,0.75 H005,0.5,0.35,0.2,0.1,0.1,0.2,0.75,1.25,1.4,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.4,1.25,1.1,1,0.9,0.8,0.7 H006,0.19,0.09,0.06,0.05,0.09,0.22,0.86,1.84,1.86,1.41,1.24,1.2,1.32,1.44,1.45,1.59,2.03,2.08,1.51,1.06,0.74,0.62,0.61,0.44 diff --git a/data/profiles/temporal/TemporalProfile_Monthly.csv b/data/profiles/temporal/TemporalProfile_Monthly.csv index f904688..1c2108f 100644 --- a/data/profiles/temporal/TemporalProfile_Monthly.csv +++ b/data/profiles/temporal/TemporalProfile_Monthly.csv @@ -1,4 +1,4 @@ -ID,1,2,3,4,5,6,7,8,9,10,11,12 +ID,January,February,March,April,May,June,July,August,September,October,November,December M001,1,1,1,1,1,1,1,1,1,1,1,1 M002,1.2,1.15,1.05,1,0.9,0.85,0.8,0.875,0.95,1,1.075,1.15 M003,1.7,1.5,1.3,1,0.7,0.4,0.2,0.4,0.7,1.05,1.4,1.65 diff --git a/data/profiles/temporal/TemporalProfile_Weekly.csv b/data/profiles/temporal/TemporalProfile_Weekly.csv index c96b586..2d66279 100644 --- a/data/profiles/temporal/TemporalProfile_Weekly.csv +++ b/data/profiles/temporal/TemporalProfile_Weekly.csv @@ -1,4 +1,4 @@ -ID,0,1,2,3,4,5,6 +ID,Monday,Tuesday,Wednesday,Thursday,Friday,Saturday,Sunday W001,1,1,1,1,1,1,1 W002,1.06,1.06,1.06,1.06,1.06,0.85,0.85 W003,1.08,1.08,1.08,1.08,1.08,0.8,0.8 diff --git a/hermesv3_gr/modules/emision_inventories/emission_inventory.py b/hermesv3_gr/modules/emision_inventories/emission_inventory.py index 87fe767..2ce190d 100644 --- a/hermesv3_gr/modules/emision_inventories/emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/emission_inventory.py @@ -132,7 +132,7 @@ class EmissionInventory(object): # Creating Temporal Object # It will also create the necessaries timezone files - if not((p_month is None) and (p_week is None) and (p_hour is None)): + if not((p_month is None) and (p_week is None) and (p_day is None) and (p_hour is None)): self.temporal = TemporalDistribution( current_date, options.output_timestep_type, options.output_timestep_num, options.output_timestep_freq, options.p_month, p_month, options.p_week, p_week, options.p_day, p_day, options.p_hour, p_hour, diff --git a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py index 08baa77..b237bc5 100755 --- a/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/gfas_emission_inventory.py @@ -71,7 +71,7 @@ class GfasEmissionInventory(EmissionInventory): def __init__(self, options, grid, current_date, inventory_name, source_type, sector, pollutants, inputs_path, frequency, vertical_output_profile, reference_year=2010, factors=None, regrid_mask=None, p_vertical=None, p_month=None, p_week=None, - p_hour=None, p_speciation=None): + p_day=None, p_hour=None, p_speciation=None): from hermesv3_gr.modules.vertical.vertical_gfas import GfasVerticalDistribution st_time = timeit.default_timer() @@ -81,7 +81,7 @@ class GfasEmissionInventory(EmissionInventory): options, grid, current_date, inventory_name, source_type, sector, pollutants, inputs_path, frequency, vertical_output_profile, reference_year=reference_year, factors=factors, regrid_mask=regrid_mask, p_vertical=None, - p_month=p_month, p_week=p_week, p_hour=p_hour, p_speciation=p_speciation) + p_month=p_month, p_week=p_week, p_day=p_day, p_hour=p_hour, p_speciation=p_speciation) self.approach = self.get_approach(p_vertical) self.method = self.get_method(p_vertical) diff --git a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py index 4a5dd7b..98a90ec 100755 --- a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py @@ -73,7 +73,7 @@ class PointGfasEmissionInventory(EmissionInventory): def __init__(self, options, grid, current_date, inventory_name, source_type, sector, pollutants, inputs_path, frequency, vertical_output_profile, reference_year=2010, factors=None, regrid_mask=None, p_vertical=None, p_month=None, p_week=None, - p_hour=None, p_speciation=None): + p_day=None, p_hour=None, p_speciation=None): from hermesv3_gr.modules.vertical.vertical_gfas import GfasVerticalDistribution st_time = timeit.default_timer() @@ -82,7 +82,7 @@ class PointGfasEmissionInventory(EmissionInventory): options, grid, current_date, inventory_name, source_type, sector, pollutants, inputs_path, frequency, vertical_output_profile, reference_year=reference_year, factors=factors, regrid_mask=regrid_mask, p_vertical=None, - p_month=p_month, p_week=p_week, p_hour=p_hour, p_speciation=p_speciation) + p_month=p_month, p_week=p_week, p_day=p_day, p_hour=p_hour, p_speciation=p_speciation) # self.approach = self.get_approach(p_vertical) self.method = self.get_method(p_vertical) diff --git a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py index b287c6c..60e32b8 100755 --- a/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_source_emission_inventory.py @@ -75,7 +75,7 @@ class PointSourceEmissionInventory(EmissionInventory): def __init__(self, options, grid, current_date, inventory_name, source_type, sector, pollutants, inputs_path, frequency, vertical_output_profile, reference_year=2010, factors=None, regrid_mask=None, - p_vertical=None, p_month=None, p_week=None, p_hour=None, p_speciation=None): + p_vertical=None, p_month=None, p_week=None, p_day=None, p_hour=None, p_speciation=None): st_time = timeit.default_timer() settings.write_log('\t\tCreating point source emission inventory.', level=3) @@ -83,7 +83,8 @@ class PointSourceEmissionInventory(EmissionInventory): super(PointSourceEmissionInventory, self).__init__( options, grid, current_date, inventory_name, source_type, sector, pollutants, inputs_path, frequency, vertical_output_profile, reference_year=reference_year, factors=factors, regrid_mask=regrid_mask, - p_vertical=p_vertical, p_month=p_month, p_week=p_week, p_hour=p_hour, p_speciation=p_speciation) + p_vertical=p_vertical, p_month=p_month, p_week=p_week, p_day=p_day, p_hour=p_hour, + p_speciation=p_speciation) self.crs = {'init': 'epsg:4326'} self.location = None diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 786b42c..aecb691 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -23,6 +23,7 @@ import sys import timeit import hermesv3_gr.config.settings as settings import numpy as np +from warnings import warn class TemporalDistribution(object): @@ -45,8 +46,8 @@ class TemporalDistribution(object): :param monthly_profile_path: Path to the file that contains all the monthly profiles. :type monthly_profile_path: str - :param month_profile_id: ID of the monthly profile to use. - :type month_profile_id: str + :param monthly_profile_id: ID of the monthly profile to use. + :type monthly_profile_id: str :param daily_profile_path: Path to the file that contains all the daily profiles. :type weekly_profile_path: str @@ -79,30 +80,38 @@ class TemporalDistribution(object): self.grid = grid - self.starting_date = starting_date + # self.starting_date = starting_date - self.timestep_type = timestep_type - self.timestep_num = timestep_num - self.timestep_freq = timestep_freq + self.date_array = self.caclulate_date_array(starting_date, timestep_type, timestep_num, timestep_freq) - self.ending_date = self.calculate_ending_date() + # self.timestep_type = timestep_type + # self.timestep_num = timestep_num + # self.timestep_freq = timestep_freq + # + # self.ending_date = self.calculate_ending_date() - # print monthly_profile_path, monthly_profile_id, weekly_profile_path, weekly_profile_id, daily_profile_path, daily_profile_id, hourly_profile_path, hourly_profile_id - # sys.exit() self.daily_profile = self.get_daily_profile(daily_profile_id, daily_profile_path) - if self.daily_profile is not None: + + if self.daily_profile is None: self.monthly_profile = self.get_monthly_profile(monthly_profile_id, monthly_profile_path) self.weekly_profile = self.get_weekly_profile(weekly_profile_id, weekly_profile_path) else: self.monthly_profile = None self.weekly_profile = None + if monthly_profile_id is not None or weekly_profile_id is not None: + warn('WARNING: Daily profile is set. Monthly or Weekly profiles will be ignored.') self.hourly_profile = self.get_hourly_profile(hourly_profile_id, hourly_profile_path) + # print 'Monthly profile: ', self.monthly_profile + # print 'Weekly profile: ', self.weekly_profile + # print 'Daily profile: ', self.daily_profile + # print 'Hourly profile: ', self.hourly_profile + self.world_info = world_info_path self.netcdf_timezones = os.path.join(auxiliar_files_dir, 'timezones.nc') - self.hours_since = [] + # self.hours_since = [] self.world_info_df = pd.read_csv(self.world_info, sep=';') self.tf = TimezoneFinder() @@ -114,6 +123,56 @@ class TemporalDistribution(object): settings.write_time('TemporalDistribution', 'Init', timeit.default_timer() - st_time, level=2) + def caclulate_date_array(self, st_date, time_step_type, time_step_num, time_step_freq): + from datetime import timedelta + from calendar import monthrange, isleap + + if time_step_type == 'hourly': + end_date = st_date + (time_step_num - 1) * timedelta(hours=time_step_freq) + elif time_step_type == 'daily': + end_date = st_date + (time_step_num - 1) * timedelta(hours=time_step_freq * 24) + elif time_step_type == 'monthly': + delta_year = (time_step_num - 1) * time_step_freq // 12 + delta_month = (time_step_num - 1) * time_step_freq % 12 + end_date = st_date.replace(year=st_date.year + delta_year, + month=st_date.month + delta_month) + elif time_step_type == 'yearly': + delta_year = (time_step_num - 1) * time_step_freq + end_date = st_date.replace(year=st_date.year + delta_year) + else: + end_date = st_date + + date_aux = st_date + date_array = [] + while date_aux <= end_date: + date_array.append(date_aux) # 3600 seconds per hour + + if time_step_type == 'hourly': + delta = timedelta(hours=time_step_freq) + elif time_step_type == 'daily': + delta = timedelta(hours=time_step_freq * 24) + elif time_step_type == 'monthly': + days = monthrange(date_aux.year, date_aux.month)[1] + delta = timedelta(hours=days * 24) + elif time_step_type == 'yearly': + if isleap(date_aux.year): + delta = timedelta(hours=366 * 24) + else: + delta = timedelta(hours=365 * 24) + else: + delta = None + + date_aux = date_aux + delta + return date_array + + def get_month_array(self, date_array): + from datetime import date + + month_array = [date(year=x.year, month=x.month, day=1) for x in date_array] + month_array = list(set(month_array)) + + return month_array + def get_csv_temporal_profile(self, profile_id, profile_path): import pandas as pd profile = pd.read_csv(profile_path) @@ -142,119 +201,322 @@ class TemporalDistribution(object): raise AttributeError('ERROR: The variable {0} is not in the {1} file.'.format( profile_value, profile_path)) sys.exit(1) - + var = var.reshape((var.shape[0], var.shape[1] * var.shape[2])) return var def get_daily_profile(self, profile_id, profile_path): + """ + Function to read the daily profile. + Return options: + - None -> If no daily profile set. + - dict -> 365 (or 365) dict keys with the Julian day as key. + - numpy.ndarray -> 3D array with the gridded profiles. + + :param profile_id: That parameter can be None if no daily profile to set; A path to the gridded profile; + Or a string (length=4) with the ID of the profile (it have to appear in the profile path). + + :param profile_path: Path to the CSV file that contains all the daily profiles. The sum of the profile have to + be the amount of days of the year to simulate. + :type profile_path: str + + :return: Daily profile (numpy.ndarray if gridded or dict for the rest) + :rtype: None, numpy.ndarray, dict + """ import calendar - if calendar.isleap(self.starting_date.year): + if calendar.isleap(self.date_array[0].year): days_of_year = 366 else: days_of_year = 365 try: if np.isnan(profile_id): - profile = None + profile_id = None + except TypeError: + profile_id = profile_id + + # No Daily profile + if profile_id is None: + profile = None + # Gridded monthly profile + elif os.path.exists(profile_id): + profile = self.get_gridded_temporal_profile('FD', profile_id) + profile_sum = profile.sum() / profile.shape[-1] + # Plane daily profile + elif len(profile_id) is 4: + profile = self.get_csv_temporal_profile(profile_id, profile_path) + profile_sum = profile.T[0].sum() + profile.columns = profile.columns.astype(int) + profile = profile.to_dict('records')[0] + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: An error has occurred in the profile {0}.'.format(profile_id)) + sys.exit(1) + + # CHECK if it is normalized + if profile is not None and round(days_of_year, 2) != round(profile_sum, 2): + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: The sum of the temporal weight factors is not {0} (is {1}).'.format( + days_of_year, profile_sum)) + sys.exit(1) + + return profile + + def get_monthly_profile(self, profile_id, profile_path): + """ + Function to read the monthly profile. + Return options: + - None -> If no monthly profile set. + - dict -> 12 dict keys with the month number (from 1 to 12) as key. + - numpy.ndarray -> 3D array with the gridded profiles. + + :param profile_id: That parameter can be None if no monthly profile to set; A path to the gridded profile; + Or a string (length=4) with the ID of the profile (it have to appear in the profile path). + + :param profile_path: Path to the CSV file that contains all the monthly profiles. The sum of the profile have to + be 12. + :type profile_path: str + + :return: Monthly profile (numpy.ndarray if gridded or dict for the rest) + :rtype: None, numpy.ndarray, dict + """ + try: + if np.isnan(profile_id): + profile_id = None except TypeError: - if os.path.exists(profile_id): - profile = self.get_gridded_temporal_profile('FD', profile_id) - profile_sum = profile.sum() / (profile.shape[-1] * profile.shape[-2]) + profile_id = profile_id + + # No Daily profile + if profile_id is None: + profile = None + # Gridded monthly profile + elif os.path.exists(profile_id): + profile = self.get_gridded_temporal_profile('FM', profile_id) + profile_sum = profile.sum() / profile.shape[-1] + # Plane daily profile + elif len(profile_id) is 4: + profile = self.get_csv_temporal_profile(profile_id, profile_path) + profile_sum = profile.T[0].sum() + profile.rename(columns={ + 'January': 1, 'February': 2, 'March': 3, 'April': 4, 'May': 5, 'June': 6, 'July': 7, 'August': 8, + 'September': 9, 'October': 10, 'November': 11, 'December': 12}, inplace=True) + profile = profile.to_dict('records')[0] + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: An error has occurred in the profile {0}.'.format(profile_id)) + sys.exit(1) + # CHECK if it is normalized + if profile is not None and round(12, 2) != round(profile_sum, 2): + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: The sum of the temporal weight factors is not {0} (is {1}).'.format( + 12, profile_sum)) + sys.exit(1) + + return profile + + def rebalance_weekly_profile(self, date, profile): + + if isinstance(profile, dict): + factor = self.calculate_weekday_factor(profile, self.calculate_weekdays(date)) + for dict_key in profile.iterkeys(): + profile[dict_key] = profile[dict_key] + factor + elif isinstance(profile, np.ndarray): + # Gridded + factor = self.calculate_weekday_gridded_factor(profile, self.calculate_weekdays(date)) + for weekday in xrange(7): + profile[weekday, :, :] = profile[weekday, :, :] + factor + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise TypeError("ERROR: Profile type '{0}' not implemented".format(type(profile))) + sys.exit(1) + + return profile + + def get_weekly_profile(self, profile_id, profile_path): + try: + if np.isnan(profile_id): + return None + except TypeError: + profile_id = profile_id + + profile = {0: None} + for month_date in self.get_month_array(self.date_array): + # No Daily profile + if profile_id is None: + return None + # Gridded monthly profile + elif os.path.exists(profile_id): + profile_aux = self.get_gridded_temporal_profile('FW', profile_id) + profile_sum = profile_aux.sum() / profile_aux.shape[-1] + # Plane daily profile elif len(profile_id) is 4: - profile = self.get_csv_temporal_profile(profile_id, profile_path) - profile_sum = profile.T[0].sum() - profile = profile.to_dict('records') + profile_aux = self.get_csv_temporal_profile(profile_id, profile_path) + profile_sum = profile_aux.T[0].sum() + profile_aux.rename(columns={'Monday': 0, 'Tuesday': 1, 'Wednesday': 2, 'Thursday': 3, 'Friday': 4, + 'Saturday': 5, 'Sunday': 6}, inplace=True) + profile_aux = profile_aux.to_dict('records')[0] else: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: - raise AttributeError('ERROR: An error has occured in the profile {0}.'.format(profile_id)) + raise AttributeError('ERROR: An error has occurred in the profile {0}.'.format(profile_id)) sys.exit(1) - # CHECK if it is nomalized - if round(days_of_year, 2) != round(profile_sum, 2): + # CHECK if it is normalized + if profile_aux is not None and round(7, 2) != round(profile_sum, 2): settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise AttributeError('ERROR: The sum of the temporal weight factors is not {0} (is {1}).'.format( - days_of_year, profile_sum)) + 7, profile_sum)) sys.exit(1) - print profile - sys.exit() + else: + if profile[0] is None: + profile[0] = profile_aux.copy() + profile[month_date.month] = self.rebalance_weekly_profile(month_date, profile_aux) return profile + @staticmethod + def parse_hourly_profile_id(profile_id): + import re - def get_monthly_profile(self, profile_id, profile_path): - pass + dict_aux = {} + try: + list_aux = list(map(str, re.split(' , | ,|, |,| ', profile_id))) + except TypeError: + return None + if len(list_aux) is 1: + dict_aux['weekday'] = profile_id + dict_aux['saturday'] = profile_id + dict_aux['sunday'] = profile_id + else: + for element in list_aux: + key_value_list = list(map(str, re.split(':| :|: | : |=| =|= | = ', element))) + dict_aux[key_value_list[0]] = key_value_list[1] - def get_weekly_profile(self, profile_id, profile_path): - pass + return dict_aux def get_hourly_profile(self, profile_id, profile_path): - pass - - def calculate_ending_date(self): """ - Calculate the date of the last timestep. - - :return: Date of the last timestep - :rtype: datetime.datetime - """ - from datetime import timedelta - - st_time = timeit.default_timer() + Function to read the hourly profile. + Return options: + - None -> If no hourly profile set. + - dict -> 24 dict keys with the month number (from 0 to 23) as key. + - numpy.ndarray -> 3D array with the gridded profiles. - if self.timestep_type == 'hourly': - end_date = self.starting_date + (self.timestep_num - 1) * timedelta(hours=self.timestep_freq) - elif self.timestep_type == 'daily': - end_date = self.starting_date + (self.timestep_num - 1) * timedelta(hours=self.timestep_freq * 24) - elif self.timestep_type == 'monthly': - delta_year = (self.timestep_num - 1) * self.timestep_freq // 12 - delta_month = (self.timestep_num - 1) * self.timestep_freq % 12 - end_date = self.starting_date.replace(year=self.starting_date.year + delta_year, - month=self.starting_date.month + delta_month) - elif self.timestep_type == 'yearly': - delta_year = (self.timestep_num - 1) * self.timestep_freq - end_date = self.starting_date.replace(year=self.starting_date.year + delta_year) - else: - end_date = self.starting_date - - settings.write_time('TemporalDistribution', 'calculate_ending_date', timeit.default_timer() - st_time, level=3) - - return end_date - - def calculate_timedelta(self, date): - """ - Calculate the difference of time to the next timestep. + :param profile_id: That parameter can be None if no hourly profile to set; A path to the gridded profile; + Or a string (length=4) with the ID of the profile (it have to appear in the profile path). - :param date: Date of the current timestep. - :type date: datetime.datetime + :param profile_path: Path to the CSV file that contains all the hourly profiles. The sum of the profile have to + be 24. + :type profile_path: str - :return: Difference of time to the next timestep. - :rtype: datetime.timedelta + :return: Hourly profile (numpy.ndarray if gridded or dict for the rest) + :rtype: None, numpy.ndarray, dict """ - from datetime import timedelta - from calendar import monthrange, isleap - - st_time = timeit.default_timer() + try: + if np.isnan(profile_id): + profile = None + except TypeError: + profile = self.parse_hourly_profile_id(profile_id) + + if profile is not None: + for profile_type, profile_id_aux in profile.iteritems(): + # Gridded monthly profile + if os.path.exists(profile_id_aux): + profile_aux = self.get_gridded_temporal_profile('FH', profile_id_aux) + profile_sum = profile_aux.sum() / profile_aux.shape[-1] + # Plane daily profile + elif len(profile_id_aux) is 4: + profile_aux = self.get_csv_temporal_profile(profile_id_aux, profile_path) + profile_sum = profile_aux.T[0].sum() + profile_aux.columns = profile_aux.columns.astype(int) + profile_aux = profile_aux.to_dict('records')[0] + else: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: An error has occurred in the profile {0}.'.format(profile_id_aux)) + sys.exit(1) - if self.timestep_type == 'hourly': - delta = timedelta(hours=self.timestep_freq) - elif self.timestep_type == 'daily': - delta = timedelta(hours=self.timestep_freq * 24) - elif self.timestep_type == 'monthly': - days = monthrange(date.year, date.month)[1] - delta = timedelta(hours=days * 24) - elif self.timestep_type == 'yearly': - if isleap(date.year): - delta = timedelta(hours=366 * 24) - else: - delta = timedelta(hours=365 * 24) - else: - delta = None + # CHECK if it is normalized + if profile_aux is not None and round(24, 2) != round(profile_sum, 2): + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise AttributeError('ERROR: The sum of the temporal weight factors ' + + '{0} is not {1} (is {2}).'.format(profile_id_aux, 24, profile_sum)) + sys.exit(1) + else: + profile[profile_type] = profile_aux - settings.write_time('TemporalDistribution', 'calculate_ending_date', timeit.default_timer() - st_time, level=3) + return profile - return delta + # def calculate_ending_date(self): + # """ + # Calculate the date of the last timestep. + # + # :return: Date of the last timestep + # :rtype: datetime.datetime + # """ + # from datetime import timedelta + # + # st_time = timeit.default_timer() + # + # if self.timestep_type == 'hourly': + # end_date = self.starting_date + (self.timestep_num - 1) * timedelta(hours=self.timestep_freq) + # elif self.timestep_type == 'daily': + # end_date = self.starting_date + (self.timestep_num - 1) * timedelta(hours=self.timestep_freq * 24) + # elif self.timestep_type == 'monthly': + # delta_year = (self.timestep_num - 1) * self.timestep_freq // 12 + # delta_month = (self.timestep_num - 1) * self.timestep_freq % 12 + # end_date = self.starting_date.replace(year=self.starting_date.year + delta_year, + # month=self.starting_date.month + delta_month) + # elif self.timestep_type == 'yearly': + # delta_year = (self.timestep_num - 1) * self.timestep_freq + # end_date = self.starting_date.replace(year=self.starting_date.year + delta_year) + # else: + # end_date = self.starting_date + # + # settings.write_time('TemporalDistribution', 'calculate_ending_date', timeit.default_timer() - st_time, level=3) + # + # return end_date + # + # def calculate_timedelta(self, date): + # """ + # Calculate the difference of time to the next timestep. + # + # :param date: Date of the current timestep. + # :type date: datetime.datetime + # + # :return: Difference of time to the next timestep. + # :rtype: datetime.timedelta + # """ + # from datetime import timedelta + # from calendar import monthrange, isleap + # + # st_time = timeit.default_timer() + # + # if self.timestep_type == 'hourly': + # delta = timedelta(hours=self.timestep_freq) + # elif self.timestep_type == 'daily': + # delta = timedelta(hours=self.timestep_freq * 24) + # elif self.timestep_type == 'monthly': + # days = monthrange(date.year, date.month)[1] + # delta = timedelta(hours=days * 24) + # elif self.timestep_type == 'yearly': + # if isleap(date.year): + # delta = timedelta(hours=366 * 24) + # else: + # delta = timedelta(hours=365 * 24) + # else: + # delta = None + # + # settings.write_time('TemporalDistribution', 'calculate_ending_date', timeit.default_timer() - st_time, level=3) + # + # return delta def get_tz_from_id(self, tz_id): """ @@ -417,30 +679,30 @@ class TemporalDistribution(object): return True - def read_gridded_profile(self, path, value): - # TODO Documentation - """ - - :param path: - :param value: - :return: - """ - from netCDF4 import Dataset - - st_time = timeit.default_timer() - - settings.write_log('\t\t\tGetting gridded temporal monthly profile from {0} .'.format(path), level=3) - - nc_in = Dataset(path) - profile = nc_in.variables[value][:, self.grid.x_lower_bound:self.grid.x_upper_bound, - self.grid.y_lower_bound:self.grid.y_upper_bound] - nc_in.close() - - profile[profile <= 0] = 1 - - settings.write_time('TemporalDistribution', 'read_gridded_profile', timeit.default_timer() - st_time, level=3) - - return profile + # def read_gridded_profile(self, path, value): + # # TODO Documentation + # """ + # + # :param path: + # :param value: + # :return: + # """ + # from netCDF4 import Dataset + # + # st_time = timeit.default_timer() + # + # settings.write_log('\t\t\tGetting gridded temporal monthly profile from {0} .'.format(path), level=3) + # + # nc_in = Dataset(path) + # profile = nc_in.variables[value][:, self.grid.x_lower_bound:self.grid.x_upper_bound, + # self.grid.y_lower_bound:self.grid.y_upper_bound] + # nc_in.close() + # + # profile[profile <= 0] = 1 + # + # settings.write_time('TemporalDistribution', 'read_gridded_profile', timeit.default_timer() - st_time, level=3) + # + # return profile def calculate_timezones(self): """ @@ -469,7 +731,7 @@ class TemporalDistribution(object): return tz_list - def calculate_2d_temporal_factors(self, date): + def calculate_2d_temporal_factors(self, date_aux): """ Calculate the temporal factor to correct the input data of the given date for each cell. @@ -479,15 +741,15 @@ class TemporalDistribution(object): :return: 2D array with the factors to correct the input data to the date of this timestep. :rtype: numpy.array """ + from datetime import date import pytz import pandas as pd st_time = timeit.default_timer() df = pd.DataFrame(self.timezones_array.flatten(), columns=['tz']) - df['i'] = df.index - df['utc'] = pd.to_datetime(date) + df['utc'] = pd.to_datetime(date_aux) try: df['local'] = df.groupby('tz')['utc'].apply( lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert(x.name).dt.tz_localize(None)) @@ -495,65 +757,101 @@ class TemporalDistribution(object): df['local'] = df.groupby('tz')['utc'].apply( lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert( self.parse_tz(x.name)).dt.tz_localize(None)) - df.set_index('local', inplace=True) - df['month'] = df.index.month - df['weekday'] = df.index.weekday - df['hour'] = df.index.hour + df['day'] = df['local'].dt.day + + # ===== HOURLY PROFILES ===== + df['weekday'] = df['local'].dt.weekday + df['hour'] = df['local'].dt.hour if self.hourly_profile is not None: if isinstance(self.hourly_profile, dict): - df['hour_factor'] = df['hour'].map(self.hourly_profile) - else: - profile_ids = self.parse_hourly_profile_id() - weekday_profile = self.get_temporal_hourly_profile(profile_ids['weekday']) - saturday_profile = self.get_temporal_hourly_profile(profile_ids['saturday']) - sunday_profile = self.get_temporal_hourly_profile(profile_ids['sunday']) - df['weekday_factor'] = df['hour'].map(weekday_profile) - df['saturday_factor'] = df['hour'].map(saturday_profile) - df['sunday_factor'] = df['hour'].map(sunday_profile) - - del df['tz'], df['utc'] - df['hour_factor'] = 0 - - df.loc[df['weekday'] <= 4, 'hour_factor'] = df['weekday_factor'][df['weekday'] <= 4].values - df.loc[df['weekday'] == 5, 'hour_factor'] = df['saturday_factor'][df['weekday'] == 5].values - df.loc[df['weekday'] == 6, 'hour_factor'] = df['sunday_factor'][df['weekday'] == 6].values - - del df['weekday_factor'], df['saturday_factor'], df['sunday_factor'] + # WEEKDAY + weekday_profile = self.hourly_profile['weekday'] + if isinstance(weekday_profile, dict): + df['weekday_factor'] = df['hour'].map(weekday_profile) + else: + for hour in xrange(24): + df.loc[df['hour'] == hour, 'weekday_factor'] = weekday_profile[ + hour, df[df['hour'] == hour].index] + # SATURDAY + saturday_profile = self.hourly_profile['saturday'] + if isinstance(saturday_profile, dict): + df['saturday_factor'] = df['hour'].map(saturday_profile) + else: + for hour in xrange(24): + df.loc[df['hour'] == hour, 'saturday_factor'] = saturday_profile[ + hour, df[df['hour'] == hour].index] + # SUNDAY + sunday_profile = self.hourly_profile['sunday'] + if isinstance(sunday_profile, dict): + df['sunday_factor'] = df['hour'].map(sunday_profile) + else: + for hour in xrange(24): + df.loc[df['hour'] == hour, 'sunday_factor'] = sunday_profile[ + hour, df[df['hour'] == hour].index] + + # Selecting profile type depending on the weekday + df.loc[df['weekday'] <= 4, 'hourly_factor'] = df['weekday_factor'][df['weekday'] <= 4].values + df.loc[df['weekday'] == 5, 'hourly_factor'] = df['saturday_factor'][df['weekday'] == 5].values + df.loc[df['weekday'] == 6, 'hourly_factor'] = df['sunday_factor'][df['weekday'] == 6].values + + df.drop(columns=['weekday_factor', 'saturday_factor', 'sunday_factor'], inplace=True) else: - df['hour_factor'] = 1 - del df['hour'] + df['hourly_factor'] = 1 + + df.drop(columns=['hour'], inplace=True) + + # ===== WEEKLY PROFILES ===== + df['month'] = df['local'].dt.month + if self.weekly_profile is not None: + for month in np.unique(df['month']): + if month not in self.weekly_profile: + self.weekly_profile[month] = self.rebalance_weekly_profile( + date(year=self.date_array[0].year, month=month, day=1), self.weekly_profile[0]) + + if isinstance(self.weekly_profile[month], dict): + df.loc[df['month'] == month, 'weekly_factor'] = df['weekday'].map(self.weekly_profile[month]) + else: + for weekday in xrange(7): + df.loc[df['weekday'] == weekday, 'weekly_factor'] = self.weekly_profile[month][ + weekday, df[df['weekday'] == weekday].index] - if self.weekly_profile_id is not None: - daily_profile = self.get_temporal_weekly_profile(date) - df['week_factor'] = df['weekday'].map(daily_profile) else: - df['week_factor'] = 1 - del df['weekday'] - - if self.monthly_profile is None: - df['month_factor'] = 1 - elif isinstance(self.monthly_profile, dict): - df['month_factor'] = df['month'].map(self.monthly_profile) - elif isinstance(self.monthly_profile, np.ndarray): - for m, df_aux in df.groupby('month'): - try: - df.loc[df['month'] == m, 'month_factor'] = \ - self.monthly_profile[m - 1, df.loc[df['month'] == m, 'i'].values] - except IndexError: - settings.write_log('ERROR: Check the .err file to get more info.') - if settings.rank == 0: - raise IOError("The gridded temporal profile {0} ".format(self.monthly_profile_path) + - "is not on the output grid resolution.") - sys.exit(1) + df['weekly_factor'] = 1 + + df.drop(columns=['weekday'], inplace=True) + + # ===== MONTHLY PROFILES ===== + if self.monthly_profile is not None: + if isinstance(self.monthly_profile, dict): + df['monthly_factor'] = df['month'].map(self.monthly_profile) + else: + for month in np.unique(df['month']): + df.loc[df['month'] == month, 'monthly_factor'] = self.monthly_profile[ + month - 1, df[df['month'] == month].index] + else: + df['monthly_factor'] = 1 + + df.drop(columns=['month'], inplace=True) + + # ===== DAILY PROFILES ===== + if self.daily_profile is not None: + if isinstance(self.monthly_profile, dict): + df['daily_factor'] = df['day'].map(self.daily_profile) + else: + for day in np.unique(df['day']): + df.loc[df['day'] == day, 'daily_factor'] = self.daily_profile[ + day - 1, df[df['day'] == day].index] else: - df['month_factor'] = 1 - del df['month'] + df['daily_factor'] = 1 + + df.drop(columns=['day'], inplace=True) + + df['factor'] = df['monthly_factor'] * df['weekly_factor'] * df['daily_factor'] * df['hourly_factor'] - df['factor'] = df['month_factor'] * df['week_factor'] * df['hour_factor'] + df.drop(columns=['monthly_factor', 'weekly_factor', 'daily_factor', 'hourly_factor'], inplace=True) - # TODO make timezones_aray 2D factors = np.array(df['factor'].values).reshape((self.timezones_array.shape[1], self.timezones_array.shape[2])) del df @@ -573,19 +871,15 @@ class TemporalDistribution(object): settings.write_log("\tCalculating temporal factors.", level=2) factors = [] - date = self.starting_date - count = 0 + # date = self.starting_date - while date <= self.ending_date: - count += 1 + for i, date_aux in enumerate(self.date_array): settings.write_log("\t\t{0} temporal factor ({1}/{2}).".format( - date.strftime('%Y/%m/%d %H:%M:%S'), count, self.timestep_num), level=3) + date_aux.strftime('%Y/%m/%d %H:%M:%S'), i+1, len(self.date_array)), level=3) - factors.append(self.calculate_2d_temporal_factors(date)) + factors.append(self.calculate_2d_temporal_factors(date_aux)) - date_aux = date - self.starting_date - self.hours_since.append(date_aux.seconds / 3600 + date_aux.days * 24) # 3600 seconds per hour - date = date + self.calculate_timedelta(date) + # self.hours_since.append((date_aux - self.date_array[0]).seconds / 3600 + date_aux.days * 24) factors = np.array(factors) @@ -593,125 +887,132 @@ class TemporalDistribution(object): level=3) return factors - def parse_hourly_profile_id(self): - """ - Parse the hourly profile ID to get a dictionary with the ID for "weekday", "saturday" and "sunday" - - :return: - """ - import re - - dict_aux = {} - list_aux = list(map(str, re.split(' , | ,|, |,| ', self.hourly_profile))) - for element in list_aux: - key_value_list = list(map(str, re.split(':| :|: | : |=| =|= | = ', element))) - dict_aux[key_value_list[0]] = key_value_list[1] - - return dict_aux - - def get_temporal_hourly_profile(self, profile_id, date=None): - """ - Extract the hourly profile of the given ID in a dictionary format. - - The hour (0 to 23) is the key (int) and the value (float) is the factor. - - :param profile_id: ID of the hourly profile to use. - :type profile_id: str - - :param date: Date of the timestep to simulate. Not necessary for a single ID. - :type date: datetime.datetime - - :return: Hourly profile where the hour (0 to 23) is the key (int) and the value (float) is the factor. - :rtype: dict - """ - import pandas as pd - - st_time = timeit.default_timer() - if date is None: - df = pd.read_csv(self.hourly_profile_path) - try: - profile = df.loc[df[df.TP_H == profile_id].index[0]].to_dict() - except IndexError: - settings.write_log('ERROR: Check the .err file to get more info.') - if settings.rank == 0: - raise AttributeError('ERROR: Hourly profile ID {0} is not in the {1} file.'.format( - profile_id, self.hourly_profile_path)) - sys.exit(1) - profile.pop('TP_H', None) - profile = {int(k): float(v) for k, v in profile.items()} - else: - profile = None - settings.write_time('TemporalDistribution', 'get_temporal_hourly_profile', timeit.default_timer() - st_time, - level=3) - - return profile - - def get_temporal_weekly_profile(self, date): - """ - Extract the daily profile of the given ID in a dictionary format. - - The weekday (0 to 6) is the key (int) and the value (float) is the factor. - - :param date: Date of the timestep to simulate. - :type date: datetime.datetime - - :return: Daily profile where the weekday (0 to 6) is the key (int) and the value (float) is the factor. - :rtype: dict - """ - import pandas as pd - - st_time = timeit.default_timer() + # def get_temporal_hourly_profile(self, profile_id, date=None): + # """ + # Extract the hourly profile of the given ID in a dictionary format. + # + # The hour (0 to 23) is the key (int) and the value (float) is the factor. + # + # :param profile_id: ID of the hourly profile to use. + # :type profile_id: str + # + # :param date: Date of the timestep to simulate. Not necessary for a single ID. + # :type date: datetime.datetime + # + # :return: Hourly profile where the hour (0 to 23) is the key (int) and the value (float) is the factor. + # :rtype: dict + # """ + # import pandas as pd + # + # st_time = timeit.default_timer() + # if date is None: + # df = pd.read_csv(self.hourly_profile_path) + # try: + # profile = df.loc[df[df.TP_H == profile_id].index[0]].to_dict() + # except IndexError: + # settings.write_log('ERROR: Check the .err file to get more info.') + # if settings.rank == 0: + # raise AttributeError('ERROR: Hourly profile ID {0} is not in the {1} file.'.format( + # profile_id, self.hourly_profile_path)) + # sys.exit(1) + # profile.pop('TP_H', None) + # profile = {int(k): float(v) for k, v in profile.items()} + # else: + # profile = None + # settings.write_time('TemporalDistribution', 'get_temporal_hourly_profile', timeit.default_timer() - st_time, + # level=3) + # + # return profile + # + # def get_temporal_weekly_profile(self, date): + # """ + # Extract the daily profile of the given ID in a dictionary format. + # + # The weekday (0 to 6) is the key (int) and the value (float) is the factor. + # + # :param date: Date of the timestep to simulate. + # :type date: datetime.datetime + # + # :return: Daily profile where the weekday (0 to 6) is the key (int) and the value (float) is the factor. + # :rtype: dict + # """ + # import pandas as pd + # + # st_time = timeit.default_timer() + # + # if self.weekly_profile_id is not None: + # df = pd.read_csv(self.weekly_profile_path) + # try: + # profile = df.loc[df[df.TP_W == self.weekly_profile_id].index[0]].to_dict() + # except IndexError: + # settings.write_log('ERROR: Check the .err file to get more info.') + # if settings.rank == 0: + # raise AttributeError('ERROR: Daily profile ID {0} is not in the {1} file.'.format( + # self.weekly_profile_id, self.weekly_profile_path)) + # sys.exit(1) + # profile.pop('TP_W', None) + # profile_aux = {int(k): float(v) for k, v in profile.items()} + # rebalance_factor = self.calculate_rebalance_factor(profile_aux, date) + # profile = {int(k): float(v) + rebalance_factor for k, v in profile.items()} + # else: + # profile = None + # + # settings.write_time('TemporalDistribution', 'get_temporal_weekly_profile', timeit.default_timer() - st_time, + # level=3) + # + # return profile + # + # def calculate_rebalance_factor(self, profile, date): + # """ + # Calculate the necessary factor make consistent the full month data. + # + # This is needed for the months that if you sum the daily factor of each day of the month it doesn't sum as + # the number of days of the month. + # + # :param profile: Daily profile. + # :type profile: dict + # + # :param date: Date of the timestep to simulate. + # :type date: datetime.datetime + # + # :return: Rebalance factor to be sum to the daily factor. + # :rtype: float + # """ + # st_time = timeit.default_timer() + # + # weekdays = self.calculate_weekdays(date) + # rebalance_factor = self.calculate_weekday_factor(profile, weekdays) + # + # settings.write_time('TemporalDistribution', 'calculate_rebalance_factor', timeit.default_timer() - st_time, + # level=3) + # + # return rebalance_factor - if self.weekly_profile_id is not None: - df = pd.read_csv(self.weekly_profile_path) - try: - profile = df.loc[df[df.TP_W == self.weekly_profile_id].index[0]].to_dict() - except IndexError: - settings.write_log('ERROR: Check the .err file to get more info.') - if settings.rank == 0: - raise AttributeError('ERROR: Daily profile ID {0} is not in the {1} file.'.format( - self.weekly_profile_id, self.weekly_profile_path)) - sys.exit(1) - profile.pop('TP_W', None) - profile_aux = {int(k): float(v) for k, v in profile.items()} - rebalance_factor = self.calculate_rebalance_factor(profile_aux, date) - profile = {int(k): float(v) + rebalance_factor for k, v in profile.items()} - else: - profile = None - - settings.write_time('TemporalDistribution', 'get_temporal_weekly_profile', timeit.default_timer() - st_time, - level=3) - - return profile - - def calculate_rebalance_factor(self, profile, date): + @staticmethod + def calculate_weekday_factor(profile, weekdays): + # TODO Documentation """ - Calculate the necessary factor make consistent the full month data. - - This is needed for the months that if you sum the daily factor of each day of the month it doesn't sum as - the number of days of the month. - - :param profile: Daily profile. - :type profile: dict - - :param date: Date of the timestep to simulate. - :type date: datetime.datetime + Operate with all the days of the month to get the sum of daily factors of the full month. - :return: Rebalance factor to be sum to the daily factor. - :rtype: float + :param profile: + :param weekdays: + :return: """ st_time = timeit.default_timer() - weekdays = self.calculate_weekdays(date) - rebalance_factor = self.calculate_weekday_factor_full_month(profile, weekdays) + weekdays_factors = 0 + num_days = 0 + for weekday in xrange(7): + weekdays_factors += profile[weekday] * weekdays[weekday] + num_days += weekdays[weekday] - settings.write_time('TemporalDistribution', 'calculate_rebalance_factor', timeit.default_timer() - st_time, - level=3) + settings.write_time('TemporalDistribution', 'calculate_weekday_factor', + timeit.default_timer() - st_time, level=3) - return rebalance_factor + return (num_days - weekdays_factors) / num_days @staticmethod - def calculate_weekday_factor_full_month(profile, weekdays): + def calculate_weekday_gridded_factor(profile, weekdays): # TODO Documentation """ Operate with all the days of the month to get the sum of daily factors of the full month. @@ -722,16 +1023,18 @@ class TemporalDistribution(object): """ st_time = timeit.default_timer() - weekdays_factors = 0 + weekdays_factors = np.zeros((profile.shape[-2], profile.shape[-1])) num_days = 0 for weekday in xrange(7): - weekdays_factors += profile[weekday] * weekdays[weekday] + weekdays_factors += profile[weekday, :, :] * weekdays[weekday] num_days += weekdays[weekday] - settings.write_time('TemporalDistribution', 'calculate_weekday_factor_full_month', + factor = (num_days - weekdays_factors) / num_days + + settings.write_time('TemporalDistribution', 'calculate_weekday_gridded_factor', timeit.default_timer() - st_time, level=3) - return (num_days - weekdays_factors) / num_days + return factor @staticmethod def calculate_weekdays(date): @@ -757,48 +1060,48 @@ class TemporalDistribution(object): settings.write_time('TemporalDistribution', 'calculate_weekdays', timeit.default_timer() - st_time, level=3) return weekdays_dict - @staticmethod - def get_temporal_monthly_profile(profile_path, profile_id): - """ - Extract the monthly profile of the given ID in a dictionary format. - - The month (1 to 12) is the key (int) and the value (float) is the factor. - - :param profile_path: Path to the file that contains all the monthly profiles. - :type profile_path: str - - :param profile_id: ID of the monthly profile to use. - :type profile_id: str - - :return: Monthly profile where the month (1 to 12) is the key (int) and the value (float) is the factor. - :rtype: dict - """ - import pandas as pd - - st_time = timeit.default_timer() - - settings.write_log("\t\t\tGetting temporal monthly profile id '{0}' from {1} .".format( - profile_id, profile_path), level=3) - - if profile_id is not None: - df = pd.read_csv(profile_path) - try: - profile = df.loc[df[df.TP_M == profile_id].index[0]].to_dict() - except IndexError: - settings.write_log('ERROR: Check the .err file to get more info.') - if settings.rank == 0: - raise AttributeError('ERROR: Monthly profile ID {0} is not in the {1} file.'.format( - profile_id, profile_path)) - sys.exit(1) - profile.pop('TP_M', None) - profile = {int(k): float(v) for k, v in profile.items()} - else: - profile = None - - settings.write_time('TemporalDistribution', 'get_temporal_monthly_profile', timeit.default_timer() - st_time, - level=2) - - return profile + # @staticmethod + # def get_temporal_monthly_profile(profile_path, profile_id): + # """ + # Extract the monthly profile of the given ID in a dictionary format. + # + # The month (1 to 12) is the key (int) and the value (float) is the factor. + # + # :param profile_path: Path to the file that contains all the monthly profiles. + # :type profile_path: str + # + # :param profile_id: ID of the monthly profile to use. + # :type profile_id: str + # + # :return: Monthly profile where the month (1 to 12) is the key (int) and the value (float) is the factor. + # :rtype: dict + # """ + # import pandas as pd + # + # st_time = timeit.default_timer() + # + # settings.write_log("\t\t\tGetting temporal monthly profile id '{0}' from {1} .".format( + # profile_id, profile_path), level=3) + # + # if profile_id is not None: + # df = pd.read_csv(profile_path) + # try: + # profile = df.loc[df[df.TP_M == profile_id].index[0]].to_dict() + # except IndexError: + # settings.write_log('ERROR: Check the .err file to get more info.') + # if settings.rank == 0: + # raise AttributeError('ERROR: Monthly profile ID {0} is not in the {1} file.'.format( + # profile_id, profile_path)) + # sys.exit(1) + # profile.pop('TP_M', None) + # profile = {int(k): float(v) for k, v in profile.items()} + # else: + # profile = None + # + # settings.write_time('TemporalDistribution', 'get_temporal_monthly_profile', timeit.default_timer() - st_time, + # level=2) + # + # return profile @staticmethod def calculate_delta_hours(st_date, time_step_type, time_step_num, time_step_freq): diff --git a/tests/unit/test_temporal.py b/tests/unit/test_temporal.py index 093b55f..ff3c372 100644 --- a/tests/unit/test_temporal.py +++ b/tests/unit/test_temporal.py @@ -395,7 +395,7 @@ class TestTemporalDistribution(unittest.TestCase): # '/home/Earth/ctena/Models/HERMESv3/IN/data/profiles/tz_world_country_iso3166.csv', # '/home/Earth/ctena/Models/HERMESv3/IN/data/auxiliar_files/testing') # - # self.assertEqual(round(temporal.calculate_weekday_factor_full_month( + # self.assertEqual(round(temporal.calculate_weekday_factor( # {0: 0.8, 1: 1.2, 2: 0.5, 3: 1.5, 4: 0.9, 5: 0.9, 6: 1.2}, {0: 5, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4}), 5), # round(0.2/29, 5)) # -- GitLab From b34296ec95dd8335faf8997c2fa63d8acbf15086 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Mon, 11 Mar 2019 18:38:15 +0100 Subject: [PATCH 07/11] fixed errors --- .../temporal/TemporalProfile_Daily.csv | 1 + hermesv3_gr/modules/temporal/temporal.py | 66 +++++++++++-------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/data/profiles/temporal/TemporalProfile_Daily.csv b/data/profiles/temporal/TemporalProfile_Daily.csv index bb647a6..b8bf779 100644 --- a/data/profiles/temporal/TemporalProfile_Daily.csv +++ b/data/profiles/temporal/TemporalProfile_Daily.csv @@ -1,3 +1,4 @@ ID,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365 D001,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 D002,0.5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +D003,3.14427876472473,3.24422168731689,3.16431975364685,2.7415509223938,2.75758385658264,2.85272240638733,3.40248823165894,3.13399147987366,3.00567245483398,2.92604064941406,2.61825752258301,2.79310417175293,2.80736494064331,2.4142735004425,2.01011276245117,2.25339698791504,3.14531016349792,3.05528044700623,2.65034437179565,3.01638174057007,3.02531218528748,2.75312566757202,2.64230012893677,2.71385312080383,2.34908246994019,2.33163714408875,2.4130482673645,2.16486954689026,1.83078026771545,0.855960845947266,1.73355102539062,2.62159442901611,2.50417160987854,2.10799264907837,2.909827709198,2.92748069763184,3.12294292449951,3.13945364952087,3.18258881568909,2.77711272239685,2.41922330856323,2.24666810035706,1.9712278842926,1.85282206535339,1.49873316287994,1.23413383960724,1.7275767326355,1.72716128826141,2.07724213600159,2.10157537460327,2.58075046539307,1.66518247127533,1.6506724357605,1.48166179656982,1.99437737464905,1.80160796642303,1.19522142410278,1.21702098846436,1.35041439533234,0.981676995754242,0.848678171634674,0.817809879779816,1.18084990978241,1.95513951778412,1.85822868347168,1.43376362323761,0.966481685638428,1.15326988697052,0.932643532752991,0.862170517444611,0.921442627906799,0.99095344543457,1.71480429172516,2.25939893722534,2.19928216934204,1.72081315517426,1.85774409770966,1.35058057308197,1.23751902580261,1.3896244764328,1.18478202819824,1.71665954589844,1.25195288658142,2.02826404571533,1.39557802677155,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.741944074630737,0.935135722160339,1.58248388767242,0.811842501163483,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.546149432659149,0.393509566783905,0.771123230457306,0.910511672496796,0.431994199752808,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.74380624294281,0.72565495967865,0.463243246078491,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.596069037914276,0.832319855690002,0.474188029766083,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.850692689418793,0.719050705432892,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.449079424142838,0.745883047580719,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.677978336811066,0.696524262428284,0.570787310600281,0.393509566783905,0.393509566783905,0.393509566783905,0.393509566783905,0.566010653972626,0.760233819484711,1.03383255004883,1.07040524482727,0.989063501358032,1.03414416313171,1.26143002510071,1.34372019767761,1.56831312179565,1.6423237323761,1.05075860023499,0.654690384864807,1.05146467685699,2.26547002792358,2.32483911514282,2.22279858589172,1.27072727680206,1.14600789546967,1.22469127178192,1.73448550701141,2.40921306610107,2.67673373222351,2.86710786819458,2.28641819953918,2.02244210243225,1.86184930801392,1.60514187812805,2.09168982505798,2.11677074432373,1.59747850894928,1.74225282669067,1.57279908657074,1.53381741046906,1.59356713294983,1.85482966899872,1.19655060768127,0.99366021156311,1.78427350521088,1.83352160453796,1.76202404499054,2.11918663978577,2.0318706035614,2.0173192024231,2.22432851791382,2.41059064865112,2.46313381195068,2.8343358039856,2.76256823539734,2.26326179504395,1.65942275524139,1.71945631504059,1.97526383399963,1.31551706790924 \ No newline at end of file diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index aecb691..a98310b 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -89,30 +89,35 @@ class TemporalDistribution(object): # self.timestep_freq = timestep_freq # # self.ending_date = self.calculate_ending_date() - - self.daily_profile = self.get_daily_profile(daily_profile_id, daily_profile_path) + if timestep_type in ['daily', 'hourly']: + self.daily_profile = self.get_daily_profile(daily_profile_id, daily_profile_path) + else: + self.daily_profile = None if self.daily_profile is None: - self.monthly_profile = self.get_monthly_profile(monthly_profile_id, monthly_profile_path) - self.weekly_profile = self.get_weekly_profile(weekly_profile_id, weekly_profile_path) + if timestep_type in ['monthly', 'daily', 'hourly']: + self.monthly_profile = self.get_monthly_profile(monthly_profile_id, monthly_profile_path) + else: + self.monthly_profile = None + + if timestep_type in ['daily', 'hourly']: + self.weekly_profile = self.get_weekly_profile(weekly_profile_id, weekly_profile_path) + else: + self.weekly_profile = None else: self.monthly_profile = None self.weekly_profile = None if monthly_profile_id is not None or weekly_profile_id is not None: warn('WARNING: Daily profile is set. Monthly or Weekly profiles will be ignored.') - self.hourly_profile = self.get_hourly_profile(hourly_profile_id, hourly_profile_path) - - # print 'Monthly profile: ', self.monthly_profile - # print 'Weekly profile: ', self.weekly_profile - # print 'Daily profile: ', self.daily_profile - # print 'Hourly profile: ', self.hourly_profile + if timestep_type in ['hourly']: + self.hourly_profile = self.get_hourly_profile(hourly_profile_id, hourly_profile_path) + else: + self.hourly_profile = None self.world_info = world_info_path self.netcdf_timezones = os.path.join(auxiliar_files_dir, 'timezones.nc') - # self.hours_since = [] - self.world_info_df = pd.read_csv(self.world_info, sep=';') self.tf = TimezoneFinder() @@ -194,7 +199,8 @@ class TemporalDistribution(object): from netCDF4 import Dataset nc_in = Dataset(profile_path, mode='r') try: - var = nc_in.variables[profile_value][:] + var = nc_in.variables[profile_value][:, self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound] except KeyError: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -240,14 +246,18 @@ class TemporalDistribution(object): profile = None # Gridded monthly profile elif os.path.exists(profile_id): - profile = self.get_gridded_temporal_profile('FD', profile_id) + profile = self.get_gridded_temporal_profile('Fday', profile_id) profile_sum = profile.sum() / profile.shape[-1] + if not calendar.isleap(self.date_array[0].year): + profile = np.append(profile, profile[-1, :].reshape((1, profile.shape[-1])), axis=0) # Plane daily profile elif len(profile_id) is 4: profile = self.get_csv_temporal_profile(profile_id, profile_path) profile_sum = profile.T[0].sum() profile.columns = profile.columns.astype(int) profile = profile.to_dict('records')[0] + if not calendar.isleap(self.date_array[0].year): + profile[366] = profile[365] else: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: @@ -293,7 +303,7 @@ class TemporalDistribution(object): profile = None # Gridded monthly profile elif os.path.exists(profile_id): - profile = self.get_gridded_temporal_profile('FM', profile_id) + profile = self.get_gridded_temporal_profile('Fmonth', profile_id) profile_sum = profile.sum() / profile.shape[-1] # Plane daily profile elif len(profile_id) is 4: @@ -320,22 +330,21 @@ class TemporalDistribution(object): return profile def rebalance_weekly_profile(self, date, profile): - + weekdays_count = self.calculate_weekdays(date) if isinstance(profile, dict): - factor = self.calculate_weekday_factor(profile, self.calculate_weekdays(date)) + factor = self.calculate_weekday_factor(profile, weekdays_count) for dict_key in profile.iterkeys(): profile[dict_key] = profile[dict_key] + factor elif isinstance(profile, np.ndarray): # Gridded - factor = self.calculate_weekday_gridded_factor(profile, self.calculate_weekdays(date)) + factor = self.calculate_weekday_gridded_factor(profile, weekdays_count) for weekday in xrange(7): - profile[weekday, :, :] = profile[weekday, :, :] + factor + profile[weekday, :] = profile[weekday, :] + factor else: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: raise TypeError("ERROR: Profile type '{0}' not implemented".format(type(profile))) sys.exit(1) - return profile def get_weekly_profile(self, profile_id, profile_path): @@ -352,7 +361,7 @@ class TemporalDistribution(object): return None # Gridded monthly profile elif os.path.exists(profile_id): - profile_aux = self.get_gridded_temporal_profile('FW', profile_id) + profile_aux = self.get_gridded_temporal_profile('Fweek', profile_id) profile_sum = profile_aux.sum() / profile_aux.shape[-1] # Plane daily profile elif len(profile_id) is 4: @@ -428,7 +437,7 @@ class TemporalDistribution(object): for profile_type, profile_id_aux in profile.iteritems(): # Gridded monthly profile if os.path.exists(profile_id_aux): - profile_aux = self.get_gridded_temporal_profile('FH', profile_id_aux) + profile_aux = self.get_gridded_temporal_profile('Fhour', profile_id_aux) profile_sum = profile_aux.sum() / profile_aux.shape[-1] # Plane daily profile elif len(profile_id_aux) is 4: @@ -758,9 +767,6 @@ class TemporalDistribution(object): lambda x: pd.to_datetime(x).dt.tz_localize(pytz.utc).dt.tz_convert( self.parse_tz(x.name)).dt.tz_localize(None)) - - df['day'] = df['local'].dt.day - # ===== HOURLY PROFILES ===== df['weekday'] = df['local'].dt.weekday df['hour'] = df['local'].dt.hour @@ -808,7 +814,8 @@ class TemporalDistribution(object): for month in np.unique(df['month']): if month not in self.weekly_profile: self.weekly_profile[month] = self.rebalance_weekly_profile( - date(year=self.date_array[0].year, month=month, day=1), self.weekly_profile[0]) + date(year=df.loc[df['month'] == month, 'local'].dt.year.values[0], month=month, day=1), + self.weekly_profile[0]) if isinstance(self.weekly_profile[month], dict): df.loc[df['month'] == month, 'weekly_factor'] = df['weekday'].map(self.weekly_profile[month]) @@ -836,8 +843,9 @@ class TemporalDistribution(object): df.drop(columns=['month'], inplace=True) # ===== DAILY PROFILES ===== + df['day'] = df['local'].dt.dayofyear if self.daily_profile is not None: - if isinstance(self.monthly_profile, dict): + if isinstance(self.daily_profile, dict): df['daily_factor'] = df['day'].map(self.daily_profile) else: for day in np.unique(df['day']): @@ -1023,10 +1031,10 @@ class TemporalDistribution(object): """ st_time = timeit.default_timer() - weekdays_factors = np.zeros((profile.shape[-2], profile.shape[-1])) + weekdays_factors = np.zeros((profile.shape[-1])) num_days = 0 for weekday in xrange(7): - weekdays_factors += profile[weekday, :, :] * weekdays[weekday] + weekdays_factors += profile[weekday, :] * weekdays[weekday] num_days += weekdays[weekday] factor = (num_days - weekdays_factors) / num_days -- GitLab From 2d24985fa91160d928488a2dddbe243d6d9373bd Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 12 Mar 2019 13:12:51 +0100 Subject: [PATCH 08/11] fixed problem with GFAS point parallel runs --- .../point_gfas_emission_inventory.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py index 98a90ec..e9a4c8e 100755 --- a/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py +++ b/hermesv3_gr/modules/emision_inventories/point_gfas_emission_inventory.py @@ -264,11 +264,10 @@ class PointGfasEmissionInventory(EmissionInventory): grid_shp = self.grid.to_shapefile(full_grid=False) temp_coords = Dataset(os.path.join(self.grid.temporal_path, 'temporal_coords.nc'), mode='r') - cell_area = temp_coords.variables['cell_area'][:].flatten() + + cell_area = temp_coords.variables['cell_area'][self.grid.x_lower_bound:self.grid.x_upper_bound, + self.grid.y_lower_bound:self.grid.y_upper_bound].flatten() grid_shp['dst_area'] = cell_area[grid_shp['FID']] - # grid_shp['dst_area'] = grid_shp.to_crs({'init': 'epsg:3857'}).area - # print grid_shp.crs['units'] - # sys.exit() gdf = gpd.sjoin(gdf.to_crs(grid_shp.crs), grid_shp, how='inner') @@ -283,14 +282,15 @@ class PointGfasEmissionInventory(EmissionInventory): gdf[pollutant['name']] = aux * gdf['src_area'] # print 'masa {0}: {1} '.format(pollutant['name'], gdf[pollutant['name']].sum()) - gdf[pollutant['name']] = (aux / gdf['dst_area']) * netcdf.variables['cell_area'][:].flatten()[ - gdf['src_index']] + gdf[pollutant['name']] = (aux / gdf['dst_area']) * netcdf.variables['cell_area'][:].flatten()[gdf['src_index']] # print netcdf.variables['bc'][:].sum() netcdf.close() settings.write_time('PointGfasEmissionInventory', 'do_regrid', timeit.default_timer() - st_time, level=2) del gdf['src_index'], gdf['index_right'] + # print gdf.sum().to_csv('/home/Earth/ctena/temp/gdf_serie.csv'.format(settings.rank)) + # sys.exit(1) self.emissions = gdf return True -- GitLab From 69a23c485c39abd74935bc06080ab5f05a76e9ed Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 12 Mar 2019 13:53:43 +0100 Subject: [PATCH 09/11] Added check for gridded profile dimensions --- hermesv3_gr/modules/grids/grid.py | 1 + hermesv3_gr/modules/grids/grid_global.py | 6 ++++-- hermesv3_gr/modules/grids/grid_lcc.py | 5 +++-- hermesv3_gr/modules/grids/grid_mercator.py | 6 ++++-- hermesv3_gr/modules/grids/grid_rotated.py | 7 +++++-- hermesv3_gr/modules/temporal/temporal.py | 7 +++++++ 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/hermesv3_gr/modules/grids/grid.py b/hermesv3_gr/modules/grids/grid.py index 0c424d1..9263e64 100644 --- a/hermesv3_gr/modules/grids/grid.py +++ b/hermesv3_gr/modules/grids/grid.py @@ -72,6 +72,7 @@ class Grid(object): self.y_lower_bound = None self.y_upper_bound = None self.shape = None + self.full_shape = None self.crs = None diff --git a/hermesv3_gr/modules/grids/grid_global.py b/hermesv3_gr/modules/grids/grid_global.py index ebbd97e..ee0b1d1 100644 --- a/hermesv3_gr/modules/grids/grid_global.py +++ b/hermesv3_gr/modules/grids/grid_global.py @@ -85,9 +85,11 @@ class GlobalGrid(Grid): self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) + settings.comm.Barrier() - self.cell_area = self.get_cell_area()[self.x_lower_bound:self.x_upper_bound, - self.y_lower_bound:self.y_upper_bound] + total_area = self.get_cell_area() + self.full_shape = (timestep_num, len(self.vertical_description), total_area.shape[-2], total_area.shape[-1]) + self.cell_area = total_area[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound] settings.write_time('GlobalGrid', 'Init', timeit.default_timer() - st_time, level=1) diff --git a/hermesv3_gr/modules/grids/grid_lcc.py b/hermesv3_gr/modules/grids/grid_lcc.py index 96ea0ec..321aebd 100644 --- a/hermesv3_gr/modules/grids/grid_lcc.py +++ b/hermesv3_gr/modules/grids/grid_lcc.py @@ -125,8 +125,9 @@ class LccGrid(Grid): # print 'Rank {0} _3_\n'.format(settings.rank) settings.comm.Barrier() # print 'Rank {0} _4_\n'.format(settings.rank) - self.cell_area = self.get_cell_area()[self.x_lower_bound:self.x_upper_bound, - self.y_lower_bound:self.y_upper_bound] + total_area = self.get_cell_area() + self.full_shape = (timestep_num, len(self.vertical_description), total_area.shape[-2], total_area.shape[-1]) + self.cell_area = total_area[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound] settings.write_time('LccGrid', 'Init', timeit.default_timer() - st_time, level=1) diff --git a/hermesv3_gr/modules/grids/grid_mercator.py b/hermesv3_gr/modules/grids/grid_mercator.py index f3104fb..6a0f651 100644 --- a/hermesv3_gr/modules/grids/grid_mercator.py +++ b/hermesv3_gr/modules/grids/grid_mercator.py @@ -110,11 +110,13 @@ class MercatorGrid(Grid): self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) + # print 'Rank {0} _3_\n'.format(settings.rank) settings.comm.Barrier() # print 'Rank {0} _4_\n'.format(settings.rank) - self.cell_area = self.get_cell_area()[self.x_lower_bound:self.x_upper_bound, - self.y_lower_bound:self.y_upper_bound] + total_area = self.get_cell_area() + self.full_shape = (timestep_num, len(self.vertical_description), total_area.shape[-2], total_area.shape[-1]) + self.cell_area = total_area[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound] settings.write_time('MercatorGrid', 'Init', timeit.default_timer() - st_time, level=1) diff --git a/hermesv3_gr/modules/grids/grid_rotated.py b/hermesv3_gr/modules/grids/grid_rotated.py index 8566300..eabf8e7 100644 --- a/hermesv3_gr/modules/grids/grid_rotated.py +++ b/hermesv3_gr/modules/grids/grid_rotated.py @@ -87,8 +87,11 @@ class RotatedGrid(Grid): self.shape = (timestep_num, len(self.vertical_description), self.x_upper_bound-self.x_lower_bound, self.y_upper_bound-self.y_lower_bound) - self.cell_area = self.get_cell_area()[self.x_lower_bound:self.x_upper_bound, - self.y_lower_bound:self.y_upper_bound] + settings.comm.Barrier() + + total_area = self.get_cell_area() + self.full_shape = (timestep_num, len(self.vertical_description), total_area.shape[-2], total_area.shape[-1]) + self.cell_area = total_area[self.x_lower_bound:self.x_upper_bound, self.y_lower_bound:self.y_upper_bound] settings.write_time('RotatedGrid', 'Init', timeit.default_timer() - st_time, level=1) diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index a98310b..05c2e0c 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -198,6 +198,13 @@ class TemporalDistribution(object): def get_gridded_temporal_profile(self, profile_value, profile_path): from netCDF4 import Dataset nc_in = Dataset(profile_path, mode='r') + if nc_in.dimensions['latitude'].size * nc_in.dimensions['longitude'].size != self.grid.full_shape[-2] * self.grid.full_shape[-1]: + settings.write_log('ERROR: Check the .err file to get more info.') + if settings.rank == 0: + raise IOError('ERROR: The temporal gridded profile {0} is not in the desired resolution like {1}.'.format( + profile_path, self.grid.coords_netcdf_file)) + sys.exit(1) + try: var = nc_in.variables[profile_value][:, self.grid.x_lower_bound:self.grid.x_upper_bound, self.grid.y_lower_bound:self.grid.y_upper_bound] -- GitLab From 9e20a360ec5fa81c2fc7af6d73bf184d5f86f616 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 12 Mar 2019 14:04:51 +0100 Subject: [PATCH 10/11] modified erro message --- hermesv3_gr/modules/temporal/temporal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermesv3_gr/modules/temporal/temporal.py b/hermesv3_gr/modules/temporal/temporal.py index 05c2e0c..462380d 100644 --- a/hermesv3_gr/modules/temporal/temporal.py +++ b/hermesv3_gr/modules/temporal/temporal.py @@ -201,7 +201,7 @@ class TemporalDistribution(object): if nc_in.dimensions['latitude'].size * nc_in.dimensions['longitude'].size != self.grid.full_shape[-2] * self.grid.full_shape[-1]: settings.write_log('ERROR: Check the .err file to get more info.') if settings.rank == 0: - raise IOError('ERROR: The temporal gridded profile {0} is not in the desired resolution like {1}.'.format( + raise IOError('ERROR: The temporal gridded profile {0} is not in the destiny resolution like {1}.'.format( profile_path, self.grid.coords_netcdf_file)) sys.exit(1) -- GitLab From a41afbfcc9971c2a6a57107d3cc4e3c44d4bf427 Mon Sep 17 00:00:00 2001 From: Carles Tena Date: Tue, 12 Mar 2019 15:08:19 +0100 Subject: [PATCH 11/11] Done --- CHANGELOG | 12 +++++++++--- hermesv3_gr/__init__.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c9ed59b..f686928 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,8 +9,14 @@ HEMRESv3_GR stable version GFAS emission inventory as point source emission. -1.X.X - 2019/XX/XX +1.0.1 + 2019/03/12 + + Temporal gridded profiles (dst resolution) + Fixed bug for GFAS point in parallel mode. + Daily profiles to Weekly profiles (7 elements). + New Daily profiles (365/366) elements (Gridded & CSV) + Added checks for the used temporal profiles. + Fixed bug on hourly profile - Temporal gridded profiles (CAMS-81 products) (dst resolution) diff --git a/hermesv3_gr/__init__.py b/hermesv3_gr/__init__.py index 5becc17..5c4105c 100644 --- a/hermesv3_gr/__init__.py +++ b/hermesv3_gr/__init__.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.0.1" -- GitLab