#!/usr/bin/env python

from collections import OrderedDict
from datetime import datetime, timedelta
import glob
import os
import shutil

from shapely.geometry import LineString, Polygon
from scipy.interpolate import SmoothBivariateSpline
from xarray import DataArray, Dataset

import cartopy.crs as ccrs
import matplotlib.pyplot as plt
import numpy as np
import pandas
import scipy.integrate
import scipy.ndimage as ndi
import shapely
import skimage.morphology
import xarray

import doas
import dplume

from c2sm.misc.conv import ppm_to_mass
import c2sm.misc
import omi

from smartcarb import config as cfg
from smartcarb import io
from smartcarb import misc
from smartcarb import map


import scipy.optimize as sopt



# xlim and ylim for visualization of Gauss plume
FILENAME2LIM = {
    'Sentinel_7_CO2_2015011311_o0190_l270.nc': (-15,90,-15,90, 20),
    'Sentinel_7_CO2_2015012411_o0354_l270.nc': (-25,25,-15,110, 20),
}

def test_est_background(filename, tracer=None, ssp=None, nsat=6, nmax=100,
                        co2_ssp=None, use_ppm=True):
    """
    Estimate XCO2 background of Berlin plume by normalized convolution using
    data outside plume. The estimate is compared to the true background
    known from model tracer.
    """
    #data = dplume.io.read_results(tracer, ssp, ns, nsat=nsat, origin='Berlin',
    #                              nmax=nmax)
    #res = []
    #for fn in data.filename:
    #    d = dplume.io.read_output(fn, ssp=co2_ssp)
    #    d = add_plume_area(d)
    #    data = estimate_background(d, 'CO2')
    #    est = data['CO2 background']
    #    res.append([est.attrs['bias'], est.attrs['scatter']])
    #
    #bias, scatter = np.array(res).T

    bias = []
    scatter = []

    #for filename in filenames:

    #    lon_eq = int(filename.split('_')[5][1:])
    #    if lon_eq not in [0, 401, 805, 1207, 1610, 2012]:
    #        continue
        
    #    try:
    #        data,res,curve = load_1d_results(filename)
    #    except FileNotFoundError:
    #        continue

    #    print(filename)
    #    d = np.array(data.CO2_background - data.CO2_true_background)

    #    bias.append( np.nanmean(d))
    #    scatter.append(np.sqrt(np.nanmean(d**2)))

    #bias = np.array(bias)
    #scatter =np.array(scatter)


    ts = pandas.read_csv(filename, parse_dates=True, index_col=0, sep='\t')
    ts = misc.filter_dataset(ts, nsat=nsat)
    if use_ppm:
        bias = ts['CO2 background bias (ppm)']
        scatter = ts['CO2 background scatter (ppm)']
    else:
        bias = -1.0 * ts['CO2 background bias (Mt/yr)']
        scatter = ts['CO2 background scatter (Mt/yr)']

    bias = bias[np.isfinite(bias)]
    scatter = scatter[np.isfinite(scatter)]

    print('size:', len(bias), len(scatter))
    print('bias:')
    print('- mean:', np.mean(bias))
    print('- std:', np.std(bias))
    print('scatter:')
    print('- mean:', np.mean(scatter))
    print('- std:', np.std(scatter))

    # histograms
    fig = plt.figure(figsize=(7,3))

    ax = fig.add_subplot(1,2,1)
    if use_ppm:
        h = ax.hist(bias, bins=np.arange(-0.20, 0.21, 0.025))[0]
        ax.set_xlabel('mean bias (ppm)')
    else:
        h = ax.hist(bias, bins=np.arange(-20, 21, 2.5))[0]
        ax.set_xlabel('mean bias (Mt/yr)')
    ax.set_ylabel('number of plumes')
    ax.set_ylim(0, h.max()+1)
    ax.grid(True)

    ax = fig.add_subplot(1,2,2)
    if use_ppm:
        h = ax.hist(scatter, bins=np.arange(0.0, 0.41, 0.025))[0]
        ax.set_xlabel('standard deviation (ppm)')
    else:
        h = ax.hist(scatter, bins=np.arange(0, 41, 2.5))[0]
        ax.set_xlabel('standard deviation (Mt/yr)')
    #ax.set_ylabel('number of plumes')
    ax.set_ylim(0, h.max()+1)
    ax.grid(True)

    plt.tight_layout()

    #product = '%s_%s_and_%.1f_ppm' % (tracer, ssp, co2_ssp)
    #filename = 'Berlin_est_CO2_bg_hist_with_%s.png' % product
    #plt.savefig(filename)
    #plt.savefig(filename.replace('.png', '.pdf'))

    return bias, scatter



def convert_ppm_to_mass(data, tracers, gas='CO2'):
    for tracer in tracers:
        values = data[tracer]
        attrs = values.attrs

        name = '%s_mass' % tracer
        data[name] = ppm_to_mass(values, data.psurf, gas)

        if 'noise_level' in attrs:
            level = ppm_to_mass(attrs['noise_level'], np.nanmean(data.psurf), gas)
            attrs['noise_level'] = level

        data[name].attrs.update(attrs)
        data[name].attrs['units'] = 'kg m-2'

    return data



def distance(q,p):
    """Distance between two points."""
    return np.sqrt((q[0] - p[0])**2 + (q[1] - p[1])**2)


def find_min_distance(p,curve):

    def f(t):
        q = curve(t=t)
        return np.sqrt((q[0] - p[0])**2 + (q[1] - p[1])**2)

    t0 = curve.get_parameter(x=p[0], y=p[1])

    res = sopt.minimize(f, x0=t0,
            method='CG',
            options={'maxiter': 100},
    )
    q = curve(t=float(res.x))
    return q, res.fun, res


def find_min_distance_to_curve(p, curve):
    """
    p :: point x0,y0
    f :: function
    c :: coefficients of 2D curve
    """
    s = 1e-3
    constr = [{
        'type': 'eq',
        'fun':  lambda q: s * (curve(x=q[0], y=q[1])[0] - q[0]),
    },{
        'type': 'eq',
        'fun':  lambda q: s * (curve(x=q[0], y=q[1])[1] - q[1]),
    }]

    q0 = curve(x=p[0], y=p[1])

    res = sopt.minimize(distance, x0=q0, args=(p,),
            method='SLSQP', constraints=constr[0],
            options={'maxiter': 100},
    )

    if not res.success:
        #print(res['message'], 'with', p)
        res.x[:] = np.nan
        res.fun = np.nan

    return res.x, res.fun, res


def kgs_to_Mtyr(x, inverse=False):
    """ Convert kg/s to Mt/yr. """
    factor = 1e9 / c2sm.misc.conv.SECONDS_PER_YEAR
    if inverse:
        return x * factor
    else:
        return x / factor



def gauss2d(p, x, y, u, x0=0.0, y0=0.0):
    """\
    Computes Gaussian plume (units: kg/m2).

    p: state vector (Q, K, BG)
    - Q:  emissions in kg/s
    - K:  eddy diffusion coefficient in m2/s
    - BG: background field in kg/m2

    x: distance from origin in m
    y: distance from center line in m
    u: wind speed in m/s

    x0: x-coordinate of origin
    y0: y-coordinate of origin

    TODO:
    - decay time

    """
    # state vector
    Q = float(p[0])    # emissions (kg/s)
    K = float(p[1])    # eddy diffusion coefficient (m2 / s)

    if len(p) == 3:
        BG = float(p[2])   # background field (kg/m2)
    else:
        BG = 0.0

    # dispersion along the plume
    down = x > x0
    sigma = np.sqrt(2.0 * K * (x[down] - x0) / u)

    # compute Gaussian plume
    c = Q / (np.sqrt(2.0 * np.pi) * sigma * u) * np.exp(-0.5 * (y[down]-y0)**2 / sigma**2)

    plume = np.full(x.shape, BG)
    plume[down] += c

    return plume




def find_all_roots(hh, dd, d, h_max=5000.0):
    """\
    Find all heights (hh), below hh_max where wind direction (dd) equals
    given wind direction (d).
    """

    founds = []

    for i in range(hh.size-1):
        if dd[i] < dd[i+1]:
            dmin, dmax = dd[i], dd[i+1]
            hmin, hmax = hh[i], hh[i+1]
        else:
            dmin, dmax = dd[i+1], dd[i]
            hmin, hmax = hh[i+1], hh[i]

        if hmin > h_max:
            continue

        if dmin <= d < dmax:
            d0 = np.interp(d, [dmin, dmax], [hmin, hmax])
            founds.append(d0)

    return founds




def arc_length_of_poly(f, c, a, b):

    x = np.linspace(a,b)

    s = np.sqrt( 1 + np.polyval(np.polyder(c), x)**2 )

    return scipy.integrate.simps(s,x)


def arc_length_of_2d_curve(curve, a, b):

    t = np.linspace(a,b, 5001)
    xt, yt = curve(t=t, m=1)

    v = np.sqrt( xt**2 + yt**2)

    l = scipy.integrate.simps(v,t)
    return l




def compute_arc_length_and_distance(points, curve):
    """\
    Given n points and polynomial function on [x_o, x_max], compute
    arc length along curve and distance from curve.
    """
    dxs = []
    dys = []

    t_o = curve.get_parameter(curve.x_o, curve.y_o)

    for px,py in points:
        (qx,qy), dy, res = find_min_distance([px,py], curve)

        if np.isnan(dy):
            dx = np.nan
            print([px,py])
        else:
            # tanget vector
            t = curve.get_parameter(qx, qy)
            v = curve.compute_tangent(t, norm=True)

            # sign of distance (negative left of curve)
            sign = np.sign(np.cross([px-qx,py-qy],v))
            dy *= sign

            # arc length
            dx = arc_length_of_2d_curve(curve, t_o, t)

        dxs.append(dx)
        dys.append(dy)

    return np.array(dxs), np.array(dys)



def gauss2dmean(p,xc,yc,wind,delta=100.0):
    """
    Oversample field within satellite pixel and average.
    """
    n, _ = xc.shape
    zc = np.full(n, np.nan)

    for i in range(n):

        #px = shapely.geometry.Polygon(np.transpose([xc[i],yc[i]]))
        #if not px.is_valid:
        #    print('Error: Pixel #%d is not valid.' % i)
        #    continue

        # compute lower and upper boundary of pixel (as integers)
        xmin = np.ceil(xc[i].min())
        xmax = np.floor(xc[i].max())
        m = round((xmax - xmin) / delta + 1)
        xt = np.linspace(xmin,xmax+1, m)

        start = np.argmin(xc[i])
        lower = omi.cgrate.compute_line(xt, xc[i], yc[i], start, +1)
        upper = omi.cgrate.compute_line(xt, xc[i], yc[i], start, -1)

        if np.all(lower > upper):
            lower, upper = upper, lower

        # stack coordinates
        xx = []
        yy = []

        for j in range(xt.size):
            m = round((upper[j] - lower[j]) / delta + 1)

            if m < 0:
                print('Value negative:', i, lower[j], upper[j])
                continue

            y = np.linspace(lower[j], upper[j], m)
            x = np.full(y.shape, xt[j])

            xx.append(x)
            yy.append(y)

        xx = np.concatenate(xx)
        yy = np.concatenate(yy)

        zc[i] = np.mean(gauss2d(p,xx,yy,wind))

    return zc





def flux_inversion(p0, xp, yp, zp, wind, ssp, use_mean=True):

    args = (xp, yp, wind, zp)

    if use_mean:
        def model(p):
            return (gauss2dmean(p,xp,yp,wind) - zp) / ssp
    else:
        def model(p):
            return (gauss2d(p,xp,yp,wind) - zp) / ssp


    return scipy.optimize.least_squares(model, p0, method='lm')


def compute_2D_curve(lon_or_x, lat_or_y, weights, lon_o=None, lat_o=None,
        origin=None, degree=2, epsg=cfg.EPSG):
    """\
    Fit a 2D Poly.
    """
    if epsg is None:
        x, y = lon_or_x, lat_or_y
    else:
        if origin is not None:
            lon_o, lat_o = misc.get_source_origin(origin)

        x, y = misc.xy2wgs(lon_or_x, lat_or_y, origin, epsg, inverse=True)

    x0 = -np.sum(x*weights) / np.sum(weights)
    y0 = -np.sum(y*weights) / np.sum(weights)
    curve = Poly2D(x, y, weights, degree, 0.0, 0.0, x0, y0)

    t = curve.get_parameter(x, y)
    t = np.linspace(t.min(), t.max())
    x_curve, y_curve = curve(t=t)

    return x_curve, y_curve, curve



class Poly2D:
    def __init__(self, x, y, w, degree=2, x_o=0.0, y_o=0.0,
            x0=0.0, y0=0.0, force_origin=True):
        """\
        A 2D curve fitted on the point cloud given by x and y.

        Parameters
        ----------
        x,y,w: x,y coords and weights used for fitting the data

        degree: degrees if the two polynomials x(t) and y(t)

        x_o, y_o: location of source (will be added to x,y and given
                  high weight such that curves goes through source)

        x0, y0: origin of coordinate system
        """
        self.x = x
        self.y = y
        self.w = w
        self.degree = degree
        self.force_origin = force_origin

        self.x_o = x_o
        self.y_o = y_o
        self.t_o = np.nan

        self.x0 = x0
        self.y0 = y0

        self.c = np.zeros(2 * (degree + 1))

        self._fit()

        # arc length to origin
        self.t_o = self.get_parameter(self.x_o, self.y_o)

        # 
        self.tmin = 0.0
        self.tmax = np.max(self.get_parameter(self.x, self.y))
        self.interpolator = None


    def to_file(self, filename):
        attrs = ['x', 'y', 'w', 'degree', 'x_o', 'y_o', 'x0', 'y0']
        data = dict((a, getattr(self, a)) for a in attrs)
        np.savez(filename, **data)

    @classmethod
    def from_file(cls, filename):
        data = np.load(filename)
        return cls(**data)

    def _fit(self):

        def objective(c, w, x, y):
            xt,yt = self._compute_curve(c,x,y)
            return np.concatenate([
                w * (xt - x), w * (yt - y)
            ])

        # add origin
        if self.force_origin:
            x = np.append(self.x, self.x_o)
            y = np.append(self.y, self.y_o)
            w = np.append(self.w, 1000.)

        # angle around origin (not used)
        #phi = np.arctan2(y - y0, x - x0)

        # curve fit
        res = sopt.leastsq(objective, x0=self.c, args=(w,x,y), full_output=True)
        self.c = res[0]
        self.cov_x = res[1] # TODO
        ierr = res[4]

        if ierr not in [1,2,3,4]:
            print('least square failed with error code: ', res)


    def arc2parameter(self, arc):

        if self.interpolator is None:
            ta = np.arange(self.tmin, 2 * self.tmax)
            la = [arc_length_of_2d_curve(self, self.t_o, v) for v in ta]
            self.interpolator = scipy.interpolate.interp1d(la, ta)

        return self.interpolator(arc)


    def get_parameter(self, x, y):
        return np.sqrt((x - self.x0)**2 + (y - self.y0)**2)


    def compute_tangent(self, t0, norm=False):
        v = np.array(self(t=t0, m=1))
        if norm:
            v /= np.linalg.norm(v)
        return v


    def compute_normal(self, t0, x=None, y=None, t=None):

        x0, y0 = self(t=t0)

        # tangent vector
        v = self.compute_tangent(t0, norm=True)

        # rotate 90 degree
        n = np.dot([[0,-1],[1,0]], v)

        cx = np.array([n[0], x0])
        cy = np.array([n[1], y0])

        if t is None:
            s = np.sign(v[0] * (y - y0) - v[1] * (x - x0))
            t = s * np.sqrt((x-x0)**2 + (y-y0)**2)

        xt = np.polyval(cx,t)
        yt = np.polyval(cy,t)

        return xt,yt


    def _compute_curve(self, c, x=None, y=None, t=None, m=0):

        if t is None:
            t = self.get_parameter(x,y)
        else:
            if x is not None or y is not None:
                print('Warning: `x` and `y` will be ignored as `t` was given.')

        k = c.size // 3
        cx = c[:k]
        cy = c[k:]

        if m != 0:
            cx = np.polyder(cx, m)
            cy = np.polyder(cy, m)

        xt = np.polyval(cx,t)
        yt = np.polyval(cy,t)

        return xt, yt

    def __call__(self, x=None, y=None, t=None, m=0):
        return self._compute_curve(self.c, x, y, t, m)




def create_center_line(field, weights, degree, is_plume, force_origin=True):
    # get location of plume and XCO2 values (plus wind etc.)
    x = np.asarray(field.x)[is_plume]
    y = np.asarray(field.y)[is_plume]
    w = np.asarray(weights)[is_plume]

    # origin of center line
    x0, y0 = -x.mean(), -y.mean()

    # fit 2d curve
    curve = Poly2D(x, y, w, degree, x_o=0.0, y_o=0.0, x0=x0, y0=y0,
                   force_origin=force_origin)

    return curve


def compute_xy_coords(data, origin, epsg=cfg.EPSG):
    """
    Compute xy coordinates (in meters) for pixel centers and corners using EPSG
    number.
    """
    data['x'], data['y'] = misc.xy2wgs(data.lon, data.lat, origin,
                                       epsg=epsg, inverse=True)

    data['xc'], data['yc'] = misc.xy2wgs(data.lonc, data.latc, origin,
                                         epsg=epsg, inverse=True)

    return data


def add_plume_area(data, size=10):
    area = skimage.morphology.dilation(np.array(data.detected_plume),
                                       np.ones((size,size)))

    # mask nans in different input fields
    area[np.isnan(data.z_values)] = False

    data['plume_area'] = xarray.full_like(data.detected_plume, False)
    data['plume_area'][:] = area

    return data

def setup_flux_inversion(data, origin, degree=2, epsg=cfg.EPSG, show=False,
                         area_size=10, force_origin=True):
    """
    LEVEL-2 data
    ------------
    data       :: level-2 data (xarray.Dataset):
    - co2      :: satellite observations (e.g. XCO2)
    - clouds   :: cloud fractions at ground pixels
    - detected_plume :: boolean array where plume has been detected
    - z_values :: z-values from plume detection

    FLUX INVERSION PARAMETERS
    -------------------------
    origin       :: name of plume (used to get lon_o and lat_o)
    degree       :: degree of polynomial fitted on plume

    """
    data = compute_xy_coords(data, origin)


    # DISTANCE FROM CENTER LINE
    # make plume bigger to include background
    data = add_plume_area(data, area_size)

    # DETERMINE CENTER LINE OF PLUME 
    # give some weight for pixels outside plume to avoid
    # over/under-shooting at the end of the plume
    weights = np.array(data.z_values)
    weights[data.plume_area & ~data.detected_plume] = 1.0

    # fit curve to detected plume plus surroundings
    curve = create_center_line(data, weights, degree, data.plume_area,
                               force_origin=force_origin)

    # compute for all ground pixels distance to center line
    # pixel corners
    a = np.array(data.plume_area)
    xcp = np.full(data.xc.shape, np.nan)
    ycp = np.full(data.yc.shape, np.nan)

    data['xcp'] = xarray.full_like(data.xc, np.nan)
    data['ycp'] = xarray.full_like(data.yc, np.nan)
    data.xcp.attrs['long name'] = 'along plume corner coordiates'
    data.ycp.attrs['long name'] = 'across plume corner coordinates'

    for i in range(4):
        xc_a = np.array(data.xc[:,:,i])[a]
        yc_a = np.array(data.yc[:,:,i])[a]
        points = np.transpose([xc_a, yc_a])
        xcp[a,i], ycp[a,i] = compute_arc_length_and_distance(points, curve)

    # check if pixels still valid polygons (can happen if curve is
    # over- or undershooting
    for i,j in np.ndindex(data.plume_area.shape):

        if not a[i,j]:
            continue

        coords = np.transpose([xcp[i,j],ycp[i,j]])
        px = shapely.geometry.Polygon(coords)

        if not px.is_valid or np.abs(px.area - px.convex_hull.area) > 1.0:
            print('Error: Mask invalid pixel corners (%d,%d).' % (i,j))
            xcp[i,j] = np.nan
            ycp[i,j] = np.nan


    data.xcp[:,:,:] = xcp
    data.ycp[:,:,:] = ycp

    # pixel centers
    x_a = np.array(data.x)[data.plume_area]
    y_a = np.array(data.y)[data.plume_area]
    points = np.transpose([x_a, y_a])

    xp = np.full(data.x.shape, np.nan)
    yp = np.full(data.y.shape, np.nan)

    xp[a], yp[a] = compute_arc_length_and_distance(points, curve)

    data['xp'] = xarray.full_like(data.x, np.nan)
    data['yp'] = xarray.full_like(data.y, np.nan)
    data.xp[:] = xp
    data.yp[:] = yp
    data.xp.attrs['long name'] = 'along plume coordiate'
    data.yp.attrs['long name'] = 'across plume coordinate'


    # update plume area by masking nans
    invalids = np.isnan(data.xp) | np.isnan(data.yp)
    invalids |= np.any(np.isnan(data.xcp), axis=2)
    invalids |= np.any(np.isnan(data.ycp), axis=2)

    area = np.array(data.plume_area)
    area[invalids] = False
    data['plume_area'][:] = area


    return data, curve


def estimate_wind_speed(layers, uwind, vwind, curve=None, d_sat=None,
                        hmin=0.0, hmax=1e3, show=False, use_mean=False):
    """
    Estimate wind speed at source origin by matching plume propagation
    direction and wind profiles.
    """
    if layers[0] > layers[1]:
        layers = layers[::-1]
        uwind = uwind[::-1]
        vwind = vwind[::-1]

    levels = 0.5 * (layers[1:] + layers[:-1])

    wind = np.sqrt(uwind**2 + vwind**2)
    direction = misc.wind_direction(uwind,vwind)

    # masking [hmin,hmax]
    mask = (hmin <= levels) & (levels < hmax)
    levels = levels[mask]
    wind = wind[mask]
    direction = direction[mask]

    wind_mean = np.mean(wind)

    if use_mean:
        params = {
            'mean wind (m/s)': float(wind_mean),
            'est. wind (m/s)': float(wind_mean),
            'est. wind direction (deg)': np.nan,
            'plume height (m)': np.nan,
        }
        return params



    # wind direction (satellite)
    if curve is not None:
        xt,yt = curve(curve.x_o, curve.y_o, m=1)
        d_sat = misc.wind_direction(xt,yt)

    delta = misc.anglediff(direction, d_sat, is_degree=True)

    # find roots (if any)
    roots = misc.find_all_roots(levels, delta)

    if len(roots) == 0:
        # TODO: find minimum instead of root
        print('Warning: Wind and plume direction do not match exactly '
              'and thus using mean wind.')
        plume_height = np.nan
        wind_plume = np.mean(wind)
    else:
        # TODO: check if other heights are okay too
        # TODO: consider vertical average
        plume_height = roots[0]
        wind_plume = np.interp(plume_height, levels, wind)

    params = {
        'mean wind (m/s)': float(wind_mean),
        'est. wind (m/s)': float(wind_plume),
        'est. wind direction (deg)': float(d_sat),
        'plume height (m)': float(plume_height),
    }

    if show:
        fig = plt.figure()
        ax = fig.add_subplot(121)
        ax.plot(wind, levels)
        ax.set_xlim(xmin=0)
        ax.set_ylim(hmin, hmax)
        ax.set_ylabel('height (m)')
        ax.set_xlabel('wind speed (m/s)')
        ax.grid(True)

        ax = fig.add_subplot(122)
        ax.plot(delta, levels)
        ax.set_xlim(-50,50)
        ax.set_ylim(hmin, hmax)
        #ax.set_ylabel('height (m)')
        ax.set_xlabel('difference (degrees)')
        ax.grid(True)

        plt.tight_layout()


    return params



def estimate_plume_height(levels, wind, d_sat, hmin=0, hmax=2000.0):

    mask = (hmin <= hh) & (hh < hmax)

    # brute-force match wind direction between plume and model
    k_o = np.argmin(misc.anglediff(dd, d_sat)**2)
    wind = wind[k_o]

    # fine-tuning
    hh2 = np.linspace(
        hh[max(k_o-1, 0)],
        hh[min(k_o+1, hh.size-1)],
        51
    )
    uu2 = np.interp(hh2, hh, uu)
    vv2 = np.interp(hh2, hh, vv)
    dd2 = (270.0 - np.rad2deg(np.arctan2(vv2,uu2))) % 360

    k2 = np.argmin(misc.anglediff(dd2, d_sat)**2)
    wind = np.sqrt(uu2[k2]**2 + vv2[k2]**2)

    if hh2[k2] > hmax:
        print('Warning: Plume height %d m > %d m (hmax) ' % (hh2[k2], hmax))
        wind = wind_2km

    return wind, hh2[k2], wind_2km, dd2[k2] - d_sat


def draw_situation(field_vector, plume_vector, curve, ax=None, ms=20,
                   vmin=None, vmax=None):

    t = curve.get_parameter(field_vector.x, field_vector.y)
    t = np.linspace(t.min(), t.max())

    if ax is None:
        fig = plt.figure()
        ax = fig.add_subplot(111, aspect='equal')
    else:
        fig = plt.gcf()

    ax.plot(*curve(t=t), 'k-')

    c = ax.scatter(field_vector.x, field_vector.y, c=field_vector.a,
                   vmin=vmin, vmax=vmax, s=ms)

    ax.scatter(plume_vector.x, plume_vector.y, edgecolors='#222222',
               c='none', s=ms)

    ax.grid(True)
    ax.set_xlabel('x (km)')
    ax.set_ylabel('y (km)')

    fig.subplots_adjust(right=0.8)
    cax = fig.add_axes([0.82, 0.15, 0.03, 0.7])
    cb = fig.colorbar(c, cax=cax, extend='both')
    cb.set_label('XCO$_2$ (ppm)')



def visualize(x_c, y_c, field, x, dx, dy, wind, plume, filename):

    v = round(np.mean(field.a))
    s = max(1, round(np.std(field.a)))
    vmin, vmax = v-s, v+s

    fig1 = plt.figure()
    fig2 = plt.figure()

    ax1 = fig1.add_subplot(111, aspect='equal')
    ax2 = fig2.add_subplot(111, aspect='equal')


    ax1.set_ylabel('y (km)')
    ax1.set_title('(a) Observations')
    ax2.set_title('(b) Fit')

    try:
        xmin,xmax,ymin,ymax,ms = FILENAME2LIM[filename]
    except KeyError:
        xmin=xmax=ymin=ymax=None
        ms = None

    c = ax1.scatter(field.x, field.y, c=field.a, vmin=vmin, vmax=vmax, s=ms)
    c = ax2.scatter(field.x, field.y, c=gauss2d(x,dx,dy, u=wind),
                    vmin=vmin, vmax=vmax, s=ms)
    ax1.scatter(plume.x, plume.y, edgecolors='#222222', c='none', s=ms)


    for ax in [ax1, ax2]:

        ax.set_xlim(xmin,xmax)
        ax.set_ylim(ymin,ymax)

        ax.set_xlabel('x (km)')
        ax.plot(x_c, y_c, 'k-')

        # draw arrow
        d1,d2 = x_c[-1] - x_c[-2], y_c[-1] - y_c[-2]
        d1 = d1 / np.sqrt(d1**2 + d2**2)
        d2 = d2 / np.sqrt(d1**2 + d2**2)
        ax.arrow(x_c[-1], y_c[-1], d1, d2, width=0, lw=1, head_width=2,
                head_length=2, fc='k', ec='k')

        ax.grid(True)

    #label = io.filename2label(filename)
    #text = ', '.join([
    #    'Q = %.1f Mt/y' % x[0],
    #    'K = %d m$^2$/s' % x[1],
    #    'x$_{BG}$ = %d ppm' % x[2],
    #    'u = %.1f m/s' % wind
    #])
    #fig.suptitle('Gaussian plume fit for Janschwalde: %s\n(%s)' % (label, text), fontsize=10, y=0.99)


    for fig in [fig1, fig2]:
        fig.subplots_adjust(right=0.8)
        cax = fig.add_axes([0.82, 0.15, 0.03, 0.7])
        cb = fig.colorbar(c, cax=cax, extend='both')
        cb.set_label('XCO$_2$ (ppm)')



def integrate_fields(data, tracer, curve, distances, tmax):
    fluxes = []
    fluxes_std = []

    for d in distances:
        f,u = integrate_field(data, tracer, curve, d, tmax=tmax, show=False)
        fluxes.append(f)
        fluxes_std.append(u)

    return np.array(fluxes), np.array(fluxes_std)


def integrate_field(data, tracer, curve, xp, tmax=100e3, show=False):

    if isinstance(tracer, str):
        tracer = data[tracer]

    v,l,g, = extract_along_line(data, tracer, curve, curve.t_o + xp,
                                show=show)

    # fraction of line from data
    f = 1.0 - (g*l).sum() / l.sum()

    if show:
        print(v)
        print(l)
        print(f)

    if len(v) and f > 0.50:
        integral = np.sum(v*l)

        try:
            u = tracer.attrs['noise_level'] # kg/m2
        except (AttributeError, KeyError):
            u = np.nan
        uncertainty = np.sqrt(np.sum(u**2 * l**2))

        return integral, uncertainty

    else:
        return np.nan, np.nan




def extract_along_line(data, tracer, curve, t0, show=False, use_signal=False):
    """
    Extracts pixels along line. The line is computed as perpendicular to curve
    at distance t0.
    """
    if isinstance(tracer, str):
        tracer = data[tracer]

    values = np.array(tracer)[data.plume_area]

    try:
        error = tracer.attrs['noise_level']
    except (AttributeError, KeyError):
        error = np.nan

    # gaps in detected plume
    gaps = np.isnan(np.array(data.CO2)[data.plume_area])

    xcurve,ycurve = curve.compute_normal(t0=t0, t=[-50e3, 50e3])
    line = LineString(np.transpose([xcurve,ycurve]))

    # pixel centers
    x = np.array(data.x)[data.plume_area]
    y = np.array(data.y)[data.plume_area]

    # pixels corners
    xc = np.array(data.xc)[data.plume_area]
    yc = np.array(data.yc)[data.plume_area]

    # pixels
    pixels = [Polygon(np.transpose(x)) for x in zip(xc,yc)]

    pixels_x = []
    lengths_x = []
    values_x = []
    distances_x = []
    gaps_x = []

    for i,px in enumerate(pixels):

        if px.intersects(line) and np.isfinite(values[i]):

            isect = px.intersection(line)

            # distance of isect line from origin
            xi, yi = np.array(isect.coords.xy)[:,0]
            d = np.sqrt((xi-xcurve[0])**2 + (yi-ycurve[0])**2)

            pixels_x.append(px)
            values_x.append(values[i])
            distances_x.append(d)
            lengths_x.append(isect.length)
            gaps_x.append(gaps[i])


    if show:
        x0,y0 = 1e-3 * np.array(curve.compute_normal(t0=t0, t=[0.0]))

        fig = plt.figure(figsize=(8,3.5))
        ax = fig.add_subplot(121, aspect='equal')

        for i,px in enumerate(pixels):
            label = 'satellite pixel' if i == 0 else None
            xy = 1e-3 * np.array(px.exterior.coords.xy)
            ax.plot(*xy, '-', color='silver' if gaps[i] else 'gray',
                    label=label)

        for i,px in enumerate(pixels_x):
            label = 'pixel on line' if i == 0 else None
            xy = 1e-3 * np.array(px.exterior.coords.xy)
            ax.plot(*xy, '-', color='black', label=label)


        xy = 1e-3 * np.array(line.coords.xy)
        ax.plot(*xy, 'o-', color='#1f77b4', label='cross section')


        xc, yc = c2sm.vis.corners2grid(data.xc, data.yc)
        tracer = np.array(tracer)
        #tracer[tracer < 0.05] = np.nan
        ax.pcolormesh(1e-3*xc,1e-3*yc, tracer, vmin=0, vmax=0.01, cmap='Reds')

        # center line
        xcc,ycc = 1e-3 * np.array(curve(t=curve.t_o+np.linspace(0, data.xp.max())))
        ax.plot(xcc,ycc,'-', color='#1f77b4')
        ax.plot(xcc[0],ycc[0],'D', color='#1f77b4')

        ax.grid(True)
        #ax.legend()

        r = max(np.abs(np.diff(xy[0])), np.abs(np.diff(xy[1])))

        ax.set_xlim(x0-0.55*r, x0+0.55*r)
        ax.set_ylim(y0-0.55*r, y0+0.55*r)

        ax.set_xlabel('x (km)')
        ax.set_ylabel('y (km)')
        ax.set_title('(a)')


        ax2 = fig.add_subplot(122)

        for d,l,v,g in zip(distances_x, lengths_x, values_x, gaps_x):
            t = 1e-3 * np.array([d,d+l]) - 50.0
            v = 1e3 * np.array([v,v])

            if not use_signal:
                ax2.fill_between(t, v-1e3*error, v+1e3*error, color='#1f77b4')

            if g:
                ax2.plot(t, v, '-', color='gray')
            else:
                ax2.plot(t, v, 'k-')


        ax2.set_xlabel('across plume (km)')
        ax2.set_ylabel('CO$_2$ mass (g$\,$m$^{-2}$)')

        xlim = np.max(np.abs(ax2.get_xlim()))
        ax2.set_xlim(-xlim, xlim)

        ax2.grid(True)
        ax2.set_title('(b)')

        plt.tight_layout()


    # sort values and lengths
    sorting = np.argsort(distances_x)

    values_x = np.array(values_x)[sorting]
    lengths_x = np.array(lengths_x)[sorting]
    gaps_x = np.array(gaps_x)[sorting]

    return values_x, lengths_x, gaps_x


    # sort based on distance from line origin


def create_gaussian_kernel(sigma):
    size = int(5 * sigma)
    kernel = np.zeros((size,size))
    kernel[size//2,size//2] = 1.0
    kernel = ndi.gaussian_filter(kernel, sigma=sigma)

    return kernel



def fill_gaps(values, gaps, area=Ellipsis, sigma=2.0):
    """
    Fill gaps in area using normalized convolution with a Gaussian filter.

    values: array with gaps
    gaps: boolean array (true if gap)
    area: boolean array (true where gaps shall be filled)

    """
    kernel = create_gaussian_kernel(sigma)

    values = np.array(values)

    valids = np.isfinite(values)
    values[gaps] = c2sm.misc.normalized_convolution(values, kernel,
                                                    mask=~valids)[gaps]
    #values = c2sm.misc.normalized_convolution(values, kernel,
    #                                          mask=~valids)
    # set values outside area to nan
    if area is not Ellipsis:
        values[~area] = np.nan

    if np.any(np.isnan(values[area])):
        print('Warning: Plume still has data gaps after gap filling.')

    return values


def estimate_background(data, tracer, sigma=10.0, show=False):
    """
    Estimate (smooth) CO2 background using normalized convolution.
    """

    if isinstance(tracer,str):
        tracer = data[tracer]
    c = np.array(tracer)

    area = np.asarray(data.plume_area)

    # only use pixels that are in plume area around plume without enhanced
    # tracer values
    valids = np.array(area & ~data.detected_plume & ~data.is_hit)
    valids[~np.isfinite(c)] = False

    kernel = create_gaussian_kernel(sigma)

    c[~valids] = np.nan
    bg_est = c2sm.misc.normalized_convolution(c, kernel, mask=~valids)
    bg_est[~area] = np.nan

    # true background and clouds
    if 'CO2_true_plume' in data:
        bg_true = np.array(data.CO2_noisefree - data.CO2_true_signal)
        bg_true[~area] = np.nan

        # MBV and STD
        MB = np.nanmean(bg_est - bg_true)
        SD = misc.rms(bg_est - bg_true)
    else:
        bg_true = np.full(data.CO2.shape, np.nan)
        MB = np.nan
        SD = np.nan

    clouds = np.ma.array(data.clouds > 0.01, mask=~area)


    bg_est = DataArray(bg_est, name='CO2_background', dims=tracer.dims)
    bg_est.attrs['long name'] = 'estimated CO2 background'
    bg_est.attrs['method'] = 'normalized convolution (sigma = %.1f px)' % sigma
    bg_est.attrs['bias'] = MB
    bg_est.attrs['scatter'] = SD

    bg_true = DataArray(bg_true, name='CO2_true_background', dims=tracer.dims)
    bg_true.attrs['long name'] = 'true CO2 background'

    data['CO2_background'] = bg_est
    data['CO2_true_background'] = bg_true

    if show:
        vmin, vmax = np.floor(np.nanmean(bg_est)), np.ceil(np.mean(bg_est))

        # estimated background
        fig1 = map.make_level2_map(data.lonc, data.latc, bg_est,
                                   vmin=vmin, vmax=vmax, do_zoom=True,
                                   clct=clouds, domain='smartcarb',
                                   label='XCO$_2$ background (ppm)')

        fig1.canvas.set_window_title('XCO2 estimated by normalized '\
                                     'convolution')


        if 'CO2_true_plume' in data:
            fig2 = map.make_level2_map(data.lonc, data.latc, bg_true,
                                       label='XCO$_2$ background (ppm)',
                                       clct=clouds, domain='smartcarb',
                                       do_zoom=True, vmin=vmin, vmax=vmax)

            fig2.canvas.set_window_title('True XCO2 background')
        else:
            fig2 = None

        for fig in [fig1, fig2]:
            if fig is None:
                continue

            lines = map.add_scatter(fig.axes[0], data.lon, data.lat, ms=0.7,
                            slice=data.detected_plume, label='detected plume')
            map.add_city_labels(fig.axes[0])

            if 'CO2_true_plume' in data:
                lines += c2sm.vis.draw_contours(fig.axes[0], data.lon, data.lat,
                                                data.CO2_true_plume,
                                                label='CO$_2$ plume')

            # use figure instead of geoaxes because freeze with pcolormesh
            fig.legend(lines, [line.get_label() for line in lines],
                       loc=(0.025, 0.89), numpoints=1, markerscale=1.5,
                       fontsize='small')


    return data



def draw_curves(ax, curve, along, across, color='k', epsg=cfg.EPSG, units='m'):

    scaling = {
        'm':  1.0,
        'km': 1e-3
    }[units]

    if np.ndim(across) == 0:
        across = np.full(along.shape, across)

    x,y = curve(t=curve.t_o + along)
    map.add_curve(ax, scaling*x, scaling*y, origin='Berlin',
                  color=color, epsg=epsg)

    for i,t0 in enumerate(along):
        t0 += curve.t_o
        t = np.linspace(-across[i], +across[i])
        x,y = curve.compute_normal(t0=t0, t=t)
        map.add_curve(ax, scaling*x, scaling*y, origin='Berlin',
                      color=color, ls='--', epsg=epsg)


    ax.set_xlabel('x (%s)' % units)
    ax.set_ylabel('y (%s)' % units)



def plot_plume(data, curve, tracer='CO2_signal', origin='Berlin',
               add_cross_sections=True, contour=None, vmin=None, vmax=None,
               zoom_area=None):

    distances = np.arange(0, 1e3 + data.xp.max(), 10e3)

    if isinstance(tracer, str):
        tracer = data[tracer]

    fig = map.make_level2_map(data.lonc, data.latc, tracer,
                              vmin=vmin, vmax=vmax,
                              label='XCO$_2$ signal (ppm)',
                              do_zoom=True, zoom_area=zoom_area)
    map.add_scatter(fig.axes[0], data.lon, data.lat, ms=4, marker='+',
                    slice=data.detected_plume, label='detected plume')
    map.add_scatter(fig.axes[0], data.lon, data.lat, ms=1, marker='o',
                    slice=data.plume_area & ~data.detected_plume, label='plume area')

    if "CO2_true_plume" in data:
        c2sm.vis.draw_contours(fig.axes[0], data.lon, data.lat,
                               data.CO2_true_plume, label='CO$_2$ plume')

    t = curve.t_o + np.linspace(-4e3,
                                np.nanmax(np.array(data.xp)[data.plume_area]))

    color = 'y' if contour is None else 'k'
    map.add_curve(fig.axes[0], *curve(t=t), origin=origin, color=color)

    if add_cross_sections:
        for t0 in distances: # every five km
            t0 += curve.t_o
            x,y = curve.compute_normal(t0=t0, t=np.linspace(-30e3, 30e3))
            map.add_curve(fig.axes[0], x, y, origin=origin, color='y')

    map.add_city_labels(fig.axes[0])

    if contour is not None:
        fig.axes[0].contour(data.lon, data.lat, contour, vmin=vmin, vmax=vmax,
                            levels=np.arange(vmin, vmax+0.1, 0.5),
                            transform=ccrs.PlateCarree())





def draw_filled_gaps(data):
    # XCO2 signal with filled gaps
    fig = map.make_level2_map(data.lonc, data.latc, data.CO2_signal,
                              vmin=-2, vmax=2, do_zoom=True,
                              domain='smartcarb',
                              label='XCO$_2$ signal (ppm)')

    lines = map.add_scatter(fig.axes[0], data.lon, data.lat, ms=0.7,
                    slice=data.detected_plume, label='detected plume')
    lines += c2sm.vis.draw_contours(fig.axes[0], data.lon, data.lat,
                           data.CO2_true_plume, label='CO$_2$ plume')
    map.add_city_labels(fig.axes[0])
    fig.canvas.set_window_title('XCO2 signal (gaps filled)')

    fig.legend(lines, [line.get_label() for line in lines],
               loc=(0.025, 0.89), numpoints=1, markerscale=1.5,
               fontsize='small')

    # XCO2 signal with gaps
    fig = map.make_level2_map(data.lonc, data.latc, data.CO2 - data.CO2_background,
                              vmin=-2, vmax=2, do_zoom=True,
                              domain='smartcarb',
                              label='XCO$_2$ signal (ppm)')

    lines = map.add_scatter(fig.axes[0], data.lon, data.lat, ms=0.7,
                    slice=data.detected_plume, label='detected plume')
    lines += c2sm.vis.draw_contours(fig.axes[0], data.lon, data.lat,
                           data.CO2_true_plume, label='CO$_2$ plume')
    map.add_city_labels(fig.axes[0])
    fig.canvas.set_window_title('XCO2 signal (with data gaps)')

    fig.legend(lines, [line.get_label() for line in lines],
               loc=(0.025, 0.89), numpoints=1, markerscale=1.5,
               fontsize='small')

    return lines


def all_gauss_1d(tracer, ssp, co2_ssp, origin='Berlin'):

    #filenames = sorted(fn[:-8] for fn in glob.glob('*.nc'))
    filenames = np.loadtxt('%sPlumes.dat' % origin, dtype=str)

    out_fname = '%s_CO2_estimation_%.1fppm_with_%s-%s.dat' % (origin, co2_ssp,
                                                              tracer, ssp)

    with open(out_fname, 'w') as f:
        wrote_header = False

        for fn in filenames:
            if origin == 'Berlin':
                data,res,_ = gauss_1d(fn, 'Berlin', show=False, detection_tracer=tracer,
                                      detection_ssp=ssp, co2_ssp=co2_ssp,
                                      load_results=False, save_results=True)
            else:
                data, res, _ = gauss_1d(fn, origin, show=False,
                        constant_emissions=False,
                         load_results=False, save_results=True, out_filename=None,
                         load_plume=False,

                         # curve
                         degree=5,
                         dmin=2e3, dmax=np.inf,
                         force_origin=True,

                         # plume detection
                         detection_tracer=tracer, detection_ssp=ssp,
                         co2_ssp=co2_ssp,
                         ns=1, q=0.99, min_pixels=10,

                         # background
                         bg_area=10, # 10x10 neighborhood

                         # wind speed
                         wind_use_mean=False, #
                         wind_hmin=120.0,      # Berlin: 0.0 m, Janschwalde: > 120 m
                         wind_hmax=1000.0      #       500.0 m,             < 3000 m
                )

            if res is None:
                continue

            if wrote_header == False:
                f.write('\t'.join(res.keys()))
                f.write('\n')
                wrote_header = True

            f.write('\t'.join([str(res[k]) for k in res.keys()]))
            f.write('\n')
            print(data.time, data.orbit, data.lon_eq, '%.1f, %.1f' % (
                  res['est. CO2 emissions (Mt/yr)'],
                  res['true CO2 emissions 11 UTC (Mt/yr)']
            ))



def load_1d_results(filename):
    data = xarray.open_dataset(filename + '_data.nc')
    data.attrs['time'] = datetime.strptime(data.time, '%Y-%m-%d %H:%M:%S')
    curve = Poly2D.from_file(filename + '_curve.npz')

    results = OrderedDict()
    with open(filename + '_results.dat', 'r') as f:
        for line in f:
            key, value = line.split('\t')
            if key in ['time', 'plume_detection']:
                results[key] = value.strip()
            elif key in ['orbit', 'lon_eq']:
                results[key] = int(value)
            else:
                results[key] = float(value)

    return data, results, curve



def gauss_1d(data, origin, show=False, constant_emissions=False,
             load_results=False, save_results=True, out_filename=None,
             load_plume=False,

             # curve
             degree=2,
             dmin=15e3, dmax=np.inf,
             force_origin=True,

             # plume detection
             detection_tracer='NO2', detection_ssp='high', co2_ssp=0.7,
             ns=37, q=0.99, min_pixels=100,

             # background
             bg_area=10.0, # 10x10 neighborhood

             # wind speed
             wind_use_mean=True, #
             wind_hmin=0.0,      # Berlin: 0.0 m, Janschwalde: > 120 m
             wind_hmax=500.0     #       500.0 m,             < 3000 m
            ):
    """
    CO2 emission estimates with Gauss-1D plume (cross section in plume)
    """
    filename = str(data)

    if load_results:
        data, results, curve = load_1d_results(filename)
    else:
        if load_plume:
            data = xarray.open_dataset(filename)
        else:
            cloud_thr = 0.3 if detection_tracer == 'NO2' else 0.01
            data = dplume.plume_detection_algorithm(
                filename, detection_tracer, detection_ssp, origin=origin,
                ns=ns, q=q, cloud_thr=cloud_thr,
                show=show, use_constant_emissions=constant_emissions, co2_ssp=co2_ssp
            )

        if np.sum(data.detected_plume) < min_pixels:
            return data, None, None

        results = OrderedDict()
        results['time'] = data.attrs['time']
        results['orbit'] = data.orbit
        results['lon_eq'] = data.lon_eq
        results['plume_detection'] = data.tracer # tracer used for plume detection

        # setup flux inversion
        data, curve = setup_flux_inversion(data, origin=origin, degree=degree,
                                           force_origin=force_origin,
                                           area_size=bg_area)

        # estimate XCO2 background using normalized convolution
        data = estimate_background(data, 'CO2', show=show)
        results['CO2 background bias (ppm)'] = data.CO2_background.attrs['bias']
        results['CO2 background scatter (ppm)'] = data.CO2_background.attrs['scatter']

        # compute CO2 signal and fill gaps
        CO2_signal = data.CO2 - data.CO2_background

        # fill data gaps
        gaps = data.plume_area & np.isnan(CO2_signal)
        CO2_signal = fill_gaps(CO2_signal, gaps, data.plume_area)
        data['CO2_signal'] = DataArray(CO2_signal, dims=data.CO2.dims,
                                       attrs=data.CO2.attrs)
        # convert ppm to kg/m2
        data = convert_ppm_to_mass(data, ['CO2', 'CO2_noise', 'CO2_signal',
                                          'CO2_true_signal', 'CO2_background',
                                          'CO2_true_background'])

    # wind speed
    layers,uwind,vwind = io.read_wind_profile(origin, data.attrs['time'])
    params = estimate_wind_speed(layers, uwind, vwind, curve, show=show,
                                 use_mean=wind_use_mean, hmin=wind_hmin,
                                 hmax=wind_hmax)
    for key in sorted(params):
        results[key] = params[key]

    results['plume length (km)'] = 1e-3 * np.nanmax(
        np.array(data.xp)[data.detected_plume])
    results['plume age (hours)'] = (
        np.nanmax(data.xp) / results['est. wind (m/s)'] / 3600
    )

    # integrate XCO2 signal across center lines
    distances = np.arange(0, np.nanmax(data.xp), 2e3)

    fluxes, fluxes_std = integrate_fields(data, 'CO2_signal_mass',
                                          curve, distances, tmax=100e3)

    valids = (distances > dmin) & np.isfinite(fluxes) & np.isfinite(fluxes_std)
    valids &= (distances <= dmax)

    # noise-free CO2 signal (use filled gaps)
    CO2_noise_free = np.array(data.CO2_signal_mass - data.CO2_noise_mass)
    mask = np.isfinite(data.CO2_signal_mass) & np.isnan(data.CO2_noise_mass)
    CO2_noise_free[mask] = np.array(data.CO2_signal_mass)[mask]

    fluxes_noisefree, _ = integrate_fields(data, CO2_noise_free,
                                           curve, distances, tmax=100e3)

    if origin == 'Berlin':
        fluxes_true, _ = integrate_fields(data, 'CO2_true_signal_mass',
                                          curve, distances, tmax=100e3)

        # flux error caused by errors in estiamted background
        flux_error_bg, _ = integrate_fields(data, data.CO2_background_mass -
                                         data.CO2_true_background_mass,
                                         curve, distances, tmax=100e3)

    # fit integrals to 1D gauss model
    wind_est = results['est. wind (m/s)']
    emissions = np.mean(fluxes[valids] * wind_est)
    emissions = kgs_to_Mtyr(emissions)

    # estimate uncertainty (based on instrument noise and wind speed)
    u_std = 2.0 # m/s
    emissions_std = np.sqrt(fluxes_std[valids]**2 * wind_est**2 +
                            fluxes[valids]**2 * u_std**2)
    emissions_std = np.mean(emissions_std) / np.sqrt(valids.sum())
    emissions_std = kgs_to_Mtyr(emissions_std)


    true_emissions = io.read_emissions(origin, data.attrs['time'],
                                       use_constant=constant_emissions)[0]
    true_emissions_7 = io.read_emissions(origin, data.attrs['time'].replace(hour=7),
                                         use_constant=constant_emissions)[0]

    results['est. CO2 emissions (Mt/yr)'] = emissions
    results['est. CO2 emissions uncertainty (Mt/yr)'] = emissions_std

    results['true CO2 emissions 11 UTC (Mt/yr)'] = true_emissions
    results['true CO2 emissions 07 UTC (Mt/yr)'] = true_emissions_7


    # estimate uncertainties (instrument noise, wind, background, ...)
    if show:
        plot_plume(data, curve, origin=origin)


    if show:
        x = 1e-3 * distances # m -> km
        plt.figure()
        ax = plt.subplot(111)

        # CO2 field with detection, true plume and integrals
        y = kgs_to_Mtyr(true_emissions, inverse=True) / wind_est
        ax.plot(x, np.full_like(x,y), 'k-', label='true emissions (11 UTC)')

        y = kgs_to_Mtyr(true_emissions_7, inverse=True) / wind_est
        ax.plot(x, np.full_like(x,y), 'k:', label='true emissions (7 UTC)')

        # add fluxes
        ax.plot(x, fluxes_true, '-', label='true signal', color='#1f77b4')
        ax.plot(x, fluxes_noisefree, '--', color='green', label='noise-free signal')
        ax.errorbar(x, fluxes, fluxes_std, marker='D', ms=2, ls='',
                    color='#ff7f0e', label='noisy signal')
        ax.plot(x, np.full_like(fluxes, np.mean(fluxes[valids])), '-',
                color='#ff7f0e', label='noisy signal (mean)')

        ax.fill_between(1e-3*distances[valids], *ax.get_ylim(), color='silver')

        ax.set_xlabel('along plume distance (km)')
        ax.set_ylabel('CO$_2$ flux (kg/m)')

        plt.legend(ncol=2)

    # error from estimating wind speed
    if origin == 'Berlin':
        valids2 = valids & np.isfinite(fluxes_true)
        wind_true = kgs_to_Mtyr(np.mean(true_emissions), inverse=True
                               ) / fluxes_true[valids2]
        mb = np.median(wind_est - wind_true)
        sd = misc.rms(wind_est - wind_true)
        u_mb = kgs_to_Mtyr(np.mean(fluxes_true[valids2]) * mb)
        u_sd = kgs_to_Mtyr(np.mean(fluxes_true[valids2]) * sd)

        results['true wind (m/s)'] = np.median(wind_true)
        results['wind bias (m/s)'] = mb
        results['wind scatter (m/s)'] = sd

        results['wind bias (Mt/yr)'] = u_mb
        results['wind scatter (Mt/yr)'] = u_sd

        valids2 = valids & np.isfinite(flux_error_bg)
        bg_mb = kgs_to_Mtyr(np.mean(flux_error_bg[valids2]) * np.mean(wind_true))
        bg_sd = kgs_to_Mtyr(misc.rms(flux_error_bg[valids2]) * np.mean(wind_true))

        results['CO2 background bias (Mt/yr)'] = bg_mb
        results['CO2 background scatter (Mt/yr)'] = bg_sd

    #for key in results:
    #    print('%s:\t%s' % (key, results[key]))

    if save_results:
        if out_filename is None:
            out_filename = os.path.basename(filename)
            out_filename = os.path.splitext(out_filename)[0]

            if detection_tracer == 'NO2':
                out_filename = '%s_%s-%s_and_%.1f' % (out_filename, detection_tracer,
                                             detection_ssp, co2_ssp)
            else:
                out_filename = '%s_%s-%.1f' % (out_filename, detection_tracer,
                                               detection_ssp)

        save_1d_results(out_filename, data, results, curve)

    return data, results, curve


def save_1d_results(filename, data, results, curve):

    data.attrs['time'] = str(data.attrs['time'])
    try:
        data.to_netcdf(filename + '_data.nc')
    except PermissionError:
        data.to_netcdf(filename + '_data.nc.tmp')
        data.close()
        shutil.move(filename + '_data.nc.tmp', filename + '_data.nc')

    with open(filename + '_results.dat', 'w') as f:
        for key, value in results.items():
            f.write('%s\t%s\n' % (key, str(value)))

    curve.to_file(filename + '_curve.npz')


def bg_std(filenames):
    results = []

    for filename in filenames:
        data = xarray.open_dataset(filename + '_data.nc')
        data.attrs['time'] = datetime.strptime(data.time, '%Y-%m-%d %H:%M:%S')
        print(data.attrs['time'])

        v = np.array(data.CO2)
        v[data.is_hit] = np.nan
        v[~data.plume_area] = np.nan

        r = (data.attrs['time'], data.orbit, data.lon_eq, np.nanstd(v))
        results.append(r)

    return results


def all_gauss_2d(tracer, ssp, co2_ssp, origin='Janschwalde', degree=5,
                 outpath='.', bg_area=10):

    if tracer == 'CO2':
        filename = (
            '/project/kug/smartcarb/results/Janschwalde'
            '/plume_detection/CO2_%.1f_q0.990_ns1.dat' % ssp
        )
    else:
        filename = (
            '/project/kug/smartcarb/results/Janschwalde'
            '/plume_detection/NO2_high_q0.990_ns1.dat'
        )
    d = pandas.read_csv(filename, sep='\t',
                        parse_dates=True, index_col=0)
    filenames = io.table2filename(d)[d.CO2_pixels >= 10]

    out_fname = '%s_CO2_estimation_%.1fppm_with_%s-%s_d%s.dat' % (
        origin, co2_ssp, tracer, ssp, degree)

    with open(out_fname, 'w') as f:
        wrote_header = False

        for i,fn in enumerate(filenames):
            print(i,os.path.basename(fn))
            data,res,_,_ = gauss_2d(fn, origin, show=True, detection_tracer=tracer,
                                    degree=degree,
                                    detection_ssp=ssp, co2_ssp=co2_ssp,
                                    load_results=False, save_results=True,
                                    use_mean=True, outpath=outpath,
                                    bg_area=bg_area)

            plt.close('all')

            if data is None:
                continue

            if wrote_header == False:
                f.write('\t'.join(res.keys()))
                f.write('\n')
                wrote_header = True

            f.write('\t'.join([str(res[k]) for k in res.keys()]))
            f.write('\n')
            print(i,data.time, data.orbit, data.lon_eq,
                  '%.1f' % res['est. CO2 emissions (Mt/yr)'],
                  '%.1f' % res['true CO2 emissions 11 UTC (Mt/yr)'],
            )



def gauss_2d(filename, origin, degree=5, offset=0, show=False,
             detection_tracer='NO2', detection_ssp='high', ns=1, co2_ssp=0.7,
             constant_emissions=False, pre_est_bg=True, load_results=False,
             save_results=False, use_mean=True, outpath='.', bg_area=10):

    # out filename prefix
    out_filename = os.path.basename(filename)
    out_filename = os.path.splitext(out_filename)[0]

    # detect plume
    cloud_thr = 0.3 if detection_tracer == 'NO2' else 0.01
    data = dplume.plume_detection_algorithm(
        filename, detection_tracer, detection_ssp, origin=origin,
        ns=ns, cloud_thr=cloud_thr, show=False,
        use_constant_emissions=constant_emissions, co2_ssp=co2_ssp
    )

    if np.sum(data.detected_plume) < 10:
        return None, None, None, None

    results = OrderedDict()
    results['time'] = data.attrs['time']
    results['orbit'] = data.orbit
    results['lon_eq'] = data.lon_eq
    results['plume size'] = int(np.sum(data.detected_plume))
    results['plume_detection'] = data.tracer # tracer used for plume detection

    # setup flux inversion
    data, curve = setup_flux_inversion(data, origin=origin, degree=degree,
                                       area_size=bg_area)

    if pre_est_bg:
        # estimate XCO2 background using normalized convolution
        data = estimate_background(data, 'CO2', show=False)
        CO2_signal = np.array(data.CO2 - data.CO2_background)
    else:
        CO2_signal = np.array(data.CO2)

    CO2_signal[~data.plume_area] = np.nan
    data['CO2_signal'] = DataArray(CO2_signal, dims=data.CO2.dims,
                                   attrs=data.CO2.attrs)

    # wind speed
    layers,uwind,vwind = io.read_wind_profile(origin, data.attrs['time'])
    params = estimate_wind_speed(layers, uwind, vwind, curve, show=True,
                                 use_mean=False, hmin=100.0, hmax=3e3)

    #if save_results:
    #    plt.savefig(out_filename + '_wind.png')

    for key in sorted(params):
        results[key] = params[key]

    results['plume length (km)'] = 1e-3 * np.nanmax(data.xp)
    results['plume age (hours)'] = (
        np.nanmax(data.xp) / results['est. wind (m/s)'] / 3600
    )

    # convert ppm to kg/m2
    data = convert_ppm_to_mass(data, ['CO2', 'CO2_noise', 'CO2_signal'])


    # setup inversion with 2D Gauss
    area = data.plume_area & np.isfinite(data.CO2_signal_mass)
    if use_mean:
        xp = np.array(data.xcp)[area]
        yp = np.array(data.ycp)[area]
        zp = np.array(data.CO2_signal_mass)[area]
    else:
        xp = np.array(data.xp)[area]
        yp = np.array(data.yp)[area]
        zp = np.array(data.CO2_signal_mass)[area]

    if pre_est_bg:
        p = [1500, 3000]
    else:
        p = [1500, 3000, zp.mean()]

    ssp = data.CO2_signal_mass.attrs['noise_level']
    res = flux_inversion(p, xp, yp, zp, params['est. wind (m/s)'], ssp,
                         use_mean)

    # TODO: uncertainty of wind speed
    unc = scipy.linalg.inv(np.dot(res.jac.T, res.jac))
    unc = np.sqrt(unc.diagonal())

    results['est. CO2 emissions (Mt/yr)'] = kgs_to_Mtyr(res.x[0])
    results['est. CO2 emissions uncertainty (Mt/yr)'] = kgs_to_Mtyr(unc[0])
    results['eddy diffusion coefficient (m2/s)'] = res.x[1]
    results['eddy diffusion coefficient uncertainty (m2/s)'] = unc[1]

    if not pre_est_bg:
        results['est. CO2 background (kg/m2)'] = res.x[2]
        results['est. CO2 background uncertainty (kg/m2)'] = unc[2]

    true_emissions = io.read_emissions(origin, data.attrs['time'],
                                       use_constant=constant_emissions)[0]
    results['true CO2 emissions 11 UTC (Mt/yr)'] = true_emissions
    results['message'] = res['message']

    if show:
        area = data.plume_area
        xp = np.array(data.xp)
        yp = np.array(data.yp)
        c = gauss2d(res.x, xp, yp, params['est. wind (m/s)'])
        c = ppm_to_mass(c, data.psurf, 'CO2', inverse=True)

        v = data.CO2 - np.mean(data.CO2_background)
        z = np.array(data.plume_area)
        plot_plume(data, curve, v, origin=origin, zoom_area=z,
                   add_cross_sections=False, contour=c, vmin=-2, vmax=4)
        if save_results:
            plt.savefig(os.path.join(outpath, out_filename + '_plume.png'))

    if save_results:

        if detection_tracer == 'NO2':
            out_filename = '%s_%s-%s_and_%.1f' % (out_filename, detection_tracer,
                                         detection_ssp, co2_ssp)
        else:
            out_filename = '%s_%s-%.1f' % (out_filename, detection_tracer,
                                           detection_ssp)

        save_1d_results(os.path.join(outpath, out_filename), data, results,
                        curve)

    return data, results, curve, res







