#!/usr/bin/env python
# coding: utf-8

from six import string_types
from datetime import datetime, timedelta
import glob
import itertools
import os
import sys

from xarray import DataArray, Dataset
from skimage import measure

import matplotlib.pyplot as plt
import netCDF4
import numpy as np
import scipy.ndimage as ndi
import scipy.stats as stats
import scipy.optimize as opt
import shapely
import pandas
import xarray


import c2sm
import smartcarb as sc


from . import misc, vis, ztest, io



# ignore invalid floating-point operation
np.seterr(invalid='ignore')


def estimate_background_stats(data, tracer, origin, use_s5=False,
                              mask=None):
    """
    Estimate mean and variance of background from Level-2 data
    around the origin.
    """
    if tracer == 'NO2':
        var_instr = {
            'low': 1e15,
            'high': 2e15
        }[data[tracer].noise_level]
    else:
        var_instr =  data[tracer].noise_level**2
    lon_o, lat_o, radius = sc.misc.get_source_origin(origin)

    i,j = sc.misc.nearest_index(lon_o, lat_o, data.lon, data.lat)

    v = np.array(data[tracer])

    # mask some values
    if mask is not None:
        v[mask] = np.nan

    if use_s5:
        v = v[max(0,i-13):i+13,max(0,j-13):j+13]
    else:
        v = v[max(0,i-50):i+50,max(0,j-50):j+50]

    if np.any(np.isfinite(v)):
        mean_bg = np.nanmean(v)
        var_bg = max(0, np.nanvar(v) - var_instr)
    else:
        mean_bg = np.nan
        var_bg = np.nan


    return mean_bg, var_bg



def read_background_stats(origin, tracer, time):
    """\
    Read mean and standard deviation of background field from time series
    extracted around origin.
    """
    fn = sc.config.BG_STATS_FILENAME[origin][tracer]
    d = pandas.read_csv(fn, sep='\t', parse_dates=True, index_col=0)

    mean_bg = float(d[d.index == time.strftime('%Y-%m-%d %H:%M')]['mean_bg'])
    var_bg= float(d[d.index == time.strftime('%Y-%m-%d %H:%M')]['std_bg']**2)

    return mean_bg, var_bg


def label_plumes(d, n_min=0):
    """\
    Label detected plume pixels. Regions with less than n_min are removed.
    """
    d[np.isnan(d)] = 0
    labels = measure.label(d, background=0)

    i = 1
    final = np.zeros_like(labels)

    for l in set(labels.flatten()):
        if l != 0 and np.sum(labels == l) >= n_min:
            final[labels == l] = i
            i += 1

    return final


def find_plume_by_signal(lon, lat, signal, thr, lon_o, lat_o, radius,
                         clouds=None, cloud_thr=0.01):
    """\
    Find plume that has signal above threshold and is connected to location
    given by lon_o and lat_o.
    """
    if clouds is None:
        is_plume = signal > thr
    else:
        is_plume = (signal > thr) & (clouds <= cloud_thr)

    labels = measure.label(np.array(is_plume), background=0)

    return find_plume_by_labels(lon, lat, labels, lon_o, lat_o, radius)



def extract_values_at_area(filenames, output_filename, origin='Berlin', ns=37, nl=1):
    """
    Extract CO2, NO2 and CO signals as mean of nl=10 largest values in area
    given by origin.
    """
    lon_o, lat_o, radius = sc.misc.get_source_origin(origin)

    line = '\t'.join(['time', 'orbit', 'lon_eq', 'number of pixels',
                      'CO2_signal', 'NO2_signal', 'CO_signal'])

    with open(output_filename, 'w') as out_file:
        out_file.write(line + '\n')

        for fn in filenames:
            d = sc.io.read_level2(fn, co2_noise=0.0, no2_noise=0.0,
                                  add_co_signal=True, origin=origin)
            mask = 100 * np.sqrt((d.lon - lon_o)**2 + (d.lat - lat_o)**2) <= radius
            mask &= d.CO2_true_signal > 0.05

            n_pixels = int(np.sum(mask))

            if n_pixels:
                sat, orbit, time, lon_eq = sc.io.process_filename(fn)
                line = [str(time), str(orbit), str(lon_eq), str(n_pixels)]

                for name in ['CO2', 'NO2', 'CO']:
                    values = d['%s_true_signal' % name]
                    values = ztest.local_mean(values, size=ns)[0]
                    values = np.array(values)

                    if nl == 1:
                        values = np.max(values[mask])
                    else:
                        values = np.mean(sorted(values[mask])[-nl:])

                    fmt = '%f' if name == 'CO2' else '%e'
                    line.append(fmt % values)


                line = '\t'.join(line)
                print(line)
                out_file.write(line + '\n')



def find_plume_by_labels(lon, lat, labels, lon_o, lat_o, radius):
    """\
    Find plume by using `labels` that are within `radius` (in km)  around
    (`lon_o`, `lat_o`).
    """
    lon = np.array(lon)
    lat = np.array(lat)

    area = sc.misc.get_circle(lon_o, lat_o, radius)

    numbers = []
    for i in set(labels.flatten()) - {0}:
        if misc.overlaps_with_area(lon[labels==i], lat[labels==i], area):
            numbers.append(i)

    if numbers:
        return  np.any([labels == i for i in numbers], axis=0)
    else:
        return np.zeros(labels.shape, bool)



def compute_performance(data):
    truths = np.array(data.CO2_true_plume, bool)
    detection = np.array(data.detected_plume, bool)

    res = {}

    # valid pixels in true plume (max number of detectable plumes)
    res['CO2_pixels'] = int(np.sum(np.isfinite(data.CO2) & truths))
    res['NO2_pixels'] = int(np.sum(np.isfinite(data.NO2) & truths))

    # performance
    res['P'] = np.sum(detection)
    res['TP'] = np.sum(truths & detection)
    res['TN'] = np.sum(~truths & ~detection)
    res['FP'] = np.sum(detection & ~truths)
    res['FN'] = np.sum(~detection & truths)

    res['PPV'] = res['TP'] / res['P']

    return res




def create_confusion_matrix(detection, truths):

    detection = np.array(detection)
    truths = np.array(truths)

    attrs = {
        'TP': np.sum(detection & truths),
        'TN': np.sum(~detection & ~truths),
        'FP': np.sum(detection & ~truths),
        'FN': np.sum(~detection & truths),
        'legend': ', \n'.join([
            'TP: true positives',
            'TN: true negatives',
            'FP: false positives',
            'FN: false negatives',
            'P: real positives',
            'TPR: true-positive rate (sensitivity)',
            'PPV: postive predictive value (precision)'
        ])
    }

    # the number of real positive cases in the data 
    attrs['P'] = attrs['TP'] + attrs['FN']

    # true positive rate (sensitivity)
    attrs['TPR'] = attrs['TP'] / attrs['P']

    # positive predictive value (precision)
    attrs['PPV'] = attrs['TP'] / (attrs['TP'] + attrs['FP'])

    return DataArray(None, name='confusion_matrix', attrs=attrs)



def detect(data, tracer, var_rand, var_sys, ns, q, mean_bg,
           lon_o, lat_o, radius, min_plume_size=0, max_iter=1):
    """\
    Detect plume in field at location origin.

    Parameters
    ''''''''''
    var_rand :: float
        variance of random error

    var_sys :: float
        variance of systematic error

    ns :: int
        size of neighborhood for computing local mean

    q :: float
        value for computing threshold z(q)

    mean_bg :: float
        mean of background field

    min_plume_size :: int
        minimum size of plume that get labelled

    lon_o, lat_o, radius :: floats
        location and radius (in km) of origin

    """
    # compute local mean and variance
    mean_s, n_s = ztest.local_mean(data[tracer], size=ns)

    # z-test
    n_pixels = -1
    for i in range(max_iter):
        z_values, is_hit = ztest.do_test(mean_s, mean_bg, var_rand, var_sys,
                                         n_s, q)
        # label plumes
        labels = label_plumes(is_hit, min_plume_size)

        # identify plumes intersecting with origin
        detected_plume = find_plume_by_labels(data.lon, data.lat, labels,
                                              lon_o, lat_o, radius)

        if max_iter > 1:
            print(i, mean_bg, np.sqrt(var_sys), np.sum(detected_plume))

        if np.sum(detected_plume) == n_pixels:
            break

        n_pixels = np.sum(detected_plume)
        mean_bg, var_sys = estimate_background_stats(data, tracer, 'Berlin',
                                                    False, mask=is_hit)



    # dict with some additional info used for visualizing results
    attrs = {
        'tracer': tracer,
        'probability for z-value': q,
        'size of neighborhood':    ns,
        'mean of background':      mean_bg,
        'tracer uncertainty (random)':       np.sqrt(var_rand),
        'tracer uncertainty (systematic)':   np.sqrt(var_sys),
        'plume location (lon, lat, radius)': (lon_o, lat_o, radius / 100.0) # in degrees
    }
    data.attrs.update(attrs)

    data['z_values'] = DataArray(z_values, dims=data[tracer].dims)
    data['is_hit'] = DataArray(is_hit, dims=data[tracer].dims)
    data['labels'] = DataArray(labels, dims=data[tracer].dims)
    data['detected_plume'] = DataArray(detected_plume, dims=data[tracer].dims)
    data['local_%s_mean' % tracer] = DataArray(mean_s, dims=data[tracer].dims)
    data['local_%s_pixels' % tracer] = DataArray(n_s, dims=data[tracer].dims)

    return detected_plume, data





def plume_detection_algorithm(filename, tracer, ssp, cloud_thr=0.01,
                              seed='orbit', use_epf=True,
                              q=0.990, ns=1,
                              origin='Berlin', show=False,
                              s7filename=None, use_s5=False,
                              use_constant_emissions=False,
                              min_plume_size=0, co2_ssp=0.7,
                              background='model'):
    """\
    Origin is Berlin or others (i.e. "Janschwalde" tracer)
    """
    # only for Berlin we have a ground "truth"
    has_truths = (origin == 'Berlin')

    if use_s5: # TODO
        filename, s7filename = sc.io.s7_to_s5(filename), filename

    is_sentinel_5 = 'Sentinel_5' in os.path.basename(filename)


    # lon/lat of source origin
    time = sc.io.filename2time(filename)
    lon_o, lat_o, radius = sc.misc.get_source_origin(origin)

    # time and groud pixels
    if tracer == 'CO2':
        data = sc.io.read_level2(filename, ssp, cloud_thr, 'high', 0.3,
                                 seed=seed,
                                 use_constant=use_constant_emissions)
        data7 = None
    else:
        if use_s5:
            data = sc.io.read_level2(filename, None, None, ssp, cloud_thr,
                                     use_constant=use_constant_emissions,
                                     seed=seed)
            data7 = sc.io.read_level2(s7filename, co2_ssp, 0.01, 'high', 0.30,
                                      use_constant=use_constant_emissions,
                                      seed=seed)
        else:
            data = sc.io.read_level2(filename, co2_ssp, 0.01, ssp, cloud_thr,
                                     use_constant=use_constant_emissions,
                                     seed=seed)
            data7 = None


    # check if source in swath
    if not misc.overlaps_with_origin(data, origin):
        print('Origin not in swath.')
        return data

    # cloud filter
    data['is_cloudy'] = data.clouds > cloud_thr
    data['is_cloudy'].attrs['threshold'] = cloud_thr

    # mean and variance of background from model time series
    if background == 'model':
        mean_bg, var_bg = read_background_stats(origin, tracer, time)

    elif background == 'obs':
        mean_bg, var_bg = estimate_background_stats(data, tracer, origin, use_s5)

    else:
        raise ValueError('Unknown method for computing background.')

    # set ssp based on NO2 scenario
    if tracer == 'NO2':
        if is_sentinel_5:
            ssp = 1.3e15
        elif ssp == 'low': # low noise scenario
            ssp = 1e15
        elif ssp == 'high': # high noise scenario
            ssp = 2e15

    var_rand = ssp**2
    var_sys = var_bg

    # detect plume
    detected_plume, data = detect(data, tracer, var_rand=var_rand,
                                  var_sys=var_sys, ns=ns, q=q, mean_bg=mean_bg,
                                  lon_o=lon_o, lat_o=lat_o, radius=radius,
                                  min_plume_size=min_plume_size)


    # add true plumes (if tracer available)
    if has_truths:
        if is_sentinel_5:
            data7['CO2_true_plume'] = create_true_plume(data7, 'CO2', origin,
                                                        0.05, 1.0)
            data7['NO2_true_plume'] = create_true_plume(data7, 'NO2', origin,
                                                        0.2e15, 1.0)
        else:
            data['CO2_true_plume'] = create_true_plume(data, 'CO2', origin,
                                                       0.05, 1.0)

        data['NO2_true_plume'] = create_true_plume(data, 'NO2', origin, 0.2e15,
                                                   1.0)


    if show:
        vis.visualize(data, do_zoom=True, show_clouds=True, use_noise=True,
                      is_sentinel_5=is_sentinel_5, data7=data7)



    return data


def readout_all_plumes(filenames, out_fname):

    with open(out_fname, 'w') as out:
        line = ['time (UTC)', 'orbit', 'lon_eq', '100%', '30%', '1%']
        print('\t'.join(line))
        out.write('\t'.join(line))
        out.write('\n')

        for filename in filenames:
            instrument, orbit, time, lon_eq = sc.io.process_filename(filename)
            line = [str(time), str(orbit), str(lon_eq)]
            data = sc.io.read_level2(filename, add_berlin=True)

            for thr in [2.0, 0.3, 0.01]:
                plume = create_true_plume(data, 'CO2', 'Berlin', 0.05, thr)
                n = np.sum(np.array(plume))
                line.append('%d' % n)

            print('\t'.join(line))
            out.write('\t'.join(line))
            out.write('\n')


def create_true_plume(data, tracer, origin, signal_thr=0.05, cloud_thr=0.3):
    """
    Creates a DataArray with the CO2/NO2 true plume.
    """
    lon_o, lat_o, radius = sc.misc.get_source_origin(origin)
    name = '%s_true_plume' % tracer
    units = 'ppm' if tracer == 'CO2' else 'molecules cm-2'

    shape = data['%s_true_signal' % tracer].shape
    dims = data['%s_true_signal' % tracer].dims

    d = DataArray(np.zeros(shape, bool), name=name, dims=dims)
    d[:] = find_plume_by_signal(data.lon, data.lat,
                                data['%s_true_signal' % tracer],
                                signal_thr,
                                lon_o, lat_o, radius,
                                data.clouds, cloud_thr)

    center_lat = np.mean(np.array(data.lat)[d])
    center_lon = np.mean(np.array(data.lon)[d])

    d.attrs = {
        'standard name': '%s plume based on model tracer' % tracer,
        'long name': '%s plume based on model tracer' % tracer,
        'signal threshold (%s)' % units: signal_thr,
        'cloud threshold': cloud_thr,
        'center_lon': center_lon,
        'center_lat': center_lat
    }

    return d


def project_detection(data_in, data_out, detected_plume):
    """
    Project detected pixels from data_in to data_out
    (e.g. Sentinel-5 to Sentinel-7)
    """

    shape = data_out.lon.shape

    new = np.zeros(shape, bool)

    n = int(np.sum(detected_plume))

    if n == 0:
        return new

    coords = np.transpose([
        np.array(data_in.lonc)[detected_plume],
        np.array(data_in.latc)[detected_plume]
    ])

    pixels = shapely.geometry.Polygon(coords[:,0,:])
    for i in range(1,n):
        pixels = pixels.union(shapely.geometry.Polygon(coords[:,i,:]))


    for i,j in np.ndindex(*shape):
        x,y = data_out.lon[i,j], data_out.lat[i,j]
        p = shapely.geometry.Point(x,y)
        new[i,j] =  p.intersects(pixels)

    return new




def show_berlin_plume(filename, tracer='CO2', save=True, save_pdf=False,
                      show=True):
    """
    Create figure showing Berlin with clouds for swaths overpassing over
    Berlin.
    """
    if tracer == 'CO2':
        signal_thr = 0.05
        cloud_thr = 0.01
    else:
        signal_thr = 0.2e15
        cloud_thr = 0.30

    # Berlin plume
    data = sc.io.read_level2(filename, co2_noise=0.0, no2_noise=0.0)
    name = '%s_true_plume' % tracer
    data[name] = create_true_plume(data, tracer, 'Berlin',
                                   signal_thr=signal_thr, cloud_thr=1.0)

    # make plot if cloud-free pixels overlap with Berlin area
    # i.e. only plot plumes which at least one cloud free pixel over
    # Berlin
    lon_o, lat_o, radius = sc.misc.get_source_origin('Berlin')
    area = sc.misc.get_circle(lon_o, lat_o, radius)

    lon = np.array(data.lon)[data[name] & (data.clouds <= cloud_thr)]
    lat = np.array(data.lat)[data[name] & (data.clouds <= cloud_thr)]

    does_overlap = misc.overlaps_with_area(lon, lat, area)
    is_plume = data[name]

    if np.sum(is_plume) > 0:
        _, orbit, time, lon_eq = sc.io.process_filename(filename)
        line = '%s\t%d\t%d' % (time, orbit, lon_eq)
        line = (
            '%s\t%d\t%d\t%d\t%s' % (
                line,
                np.sum(is_plume),
                np.sum(is_plume & (data.clouds <= 0.30)),
                np.sum(is_plume & (data.clouds <= 0.01)),
                str(does_overlap) # with Berlin
            )
        )
        print(line)
    else:
        line = ''

    if does_overlap:

        if show:
            label = sc.io.filename2label(filename)
            show_plume(data.lonc, data.latc, data[name], data.clouds,
                       label=label, tracer=tracer)
            if save:
                filename = os.path.basename(filename)
                filename = filename.replace('.nc', '_%s.png' % tracer)
                filename = os.path.join('true_plumes', filename)

                plt.savefig(filename)
                if save_pdf:
                    plt.savefig(filename.replace('.png', '.pdf'))

                plt.close()

    return line


def show_all_ground_truths(filename, thr=0.05, cloud_thr=0.01, output_path=None):

    # time and groud pixels
    time = sc.io.filename2time(filename)
    lon, lat, lonc, latc = sc.io.read_ground_pixels(filename)

    bv,jv,clct = sc.io.read_fields(filename, ['XCO2_BV', 'XCO2_JV', 'CLCT'],
                                time=time, correct_berlin=True)

    names = ['Janschwalde', 'Schwarze Pumpe', 'Boxberg',
            'Lippendorf', 'Turow', 'Patnow']
    offset = 2
    colors = [
        'grey',
        'white',
        '#ff7f0e', # J
        '#2ca02c', # SP
        '#d62728', # Bb
        '#9467bd', # Ld
        '#8c564b', # T
        '#e377c2', # P
        #'#7f7f7f',
        '#bcbd22',
        #'#17becf',
    ]
    labels = [
        'back-\nground',
        'clouds\n(>%d%%)' % (100*cloud_thr),
        'Jänsch-\nwalde',
        'Schwarze\nPumpe',
        'Boxberg',
        'Lippen-\ndorf',
        'Turów',
        'Patnów',
        'ambig-\nuous'
    ]

    # init mask
    mask = np.zeros(lon.shape, dtype=int)
    mask[clct > cloud_thr] = 1

    # power plant plumes
    shape = mask.shape + (6,)
    plants = np.zeros(shape, dtype=int)

    for i, name in enumerate(names):
        lon_o, lat_o, radius = sc.misc.get_source_origin(name)
        p = find_plume_by_signal(lon, lat, jv, thr, lon_o, lat_o, radius,
                                 clouds=clct, cloud_thr=cloud_thr)
        plants[p,i] = 1
        mask[p] = i+offset

    # ambiguous plunmes
    mask[np.sum(plants > 0, axis=2) > 1] = len(names)+offset

    labels2 = labels[:offset]
    for i, label in enumerate(labels[offset:]):
        label = '%s\n[%d]' % (label, np.sum(mask == i+offset))
        labels2.append(label)
    labels = labels2

    # nothing detected
    if not np.any(mask != 0):# & (clct < cloud_thr)):
        return None

    # reverse numbering, colors and labels
    mask = -1 * mask + len(names) + offset
    colors = colors[::-1]
    labels = labels[::-1]

    # plot
    fig = sc.map.make_level2_map(
        lonc, latc, mask, vmin=None, vmax=None,
        label='', is_discrete=True, cmap=colors,
        cb_labels=labels
    )

    ax, cax = fig.get_axes()
    sc.map.add_city_labels(ax, size='medium')

    text = sc.io.filename2label(filename)
    ax.text(0.0, 1.0, text, ha='left', va='top',
             bbox=dict(facecolor='white', alpha=1.0),
             transform=ax.transAxes
    )

    text = 'XCO$_2^\mathrm{thr}$ = %.2f ppm, d$_\mathrm{max}$ = %d km' % (thr,
                                                                         radius)
    ax.text(0.01, 0.0, text, ha='left', va='bottom', fontsize='x-small',
             transform=ax.transAxes
    )


    if output_path is not None:
        filename = os.path.basename(filename)
        filename = filename.replace('.nc', '_plumes.png')
        filename = os.path.join(output_path, filename)

        plt.savefig(filename)

    return True



def show_ground_truth(lonc, latc, signal, clouds, do_zoom=True, show_clouds=True,
        thr=0.1, cloud_thr=0.01, mask_clouds=True, lon_o=None, lat_o=None,
        dn=10, **kwargs):
    raise NotImplementedError('This function needs to be re-written')

    if mask_clouds:
        mask = find_plume_by_signal(signal, thr, origin,
                lonc.mean(2), latc.mean(2), dn=dn,
                clouds=clouds, cloud_thr=cloud_thr)
    else:
        mask = find_plume_by_signal(signal, thr, origin,
                lonc.mean(2), latc.mean(2), dn=dn)

    n_pixels = np.sum(mask)

    mask = (~mask).astype(int)
    if show_clouds:
        mask[clouds > cloud_thr] = 2

    fig = sc.map.make_level2_map(
        lonc, latc, mask, vmin=None, vmax=None,
        label='', clct=None, is_discrete=True, cmap='Greys_r',
        cb_labels=['plume\n[%d]' % n_pixels, 'back-\nground', 'clouds']
    )
    ax, cax = fig.get_axes()
    sc.map.add_city_labels(ax)

    if do_zoom:
        ax.set_xlim(1, 3.8)
        ax.set_ylim(3.5, 6)

    text = 'XCO$_2^\mathrm{thr}$ = %.2f ppm, d$_\mathrm{max}$ = %d px' % (thr, dn)
    ax.text(0.01, 0.0, text, ha='left', va='bottom', fontsize='x-small',
             transform=ax.transAxes
    )


def show_labels(lonc, latc, labels, clouds, do_zoom=True, **kwargs):
    # labels

    if do_zoom:
        labels = labels.copy()
        mask = (latc.mean(2) < 50.4) | (latc.mean(2) > 53.0)
        labels[mask] = 0


    fig = sc.map.make_level2_map(
        lonc, latc, labels, vmin=None, vmax=None,
        label='labels of detected plumes', clct=clouds > 0.01, is_discrete=True,
        cmap='nipy_spectral_r'
    )
    if do_zoom:
        ax, cax = fig.get_axes()
        ax.set_xlim(1, 3.8)
        ax.set_ylim(3.5, 6)
        sc.map.add_city_labels(ax)


def show_plume(lonc, latc, is_plume, clouds, do_zoom=True, label=None,
               tracer='CO2', **kwargs):


    mask = np.ones(is_plume.shape, int)
    mask[0,0] = 0
    mask[is_plume & (clouds > 0.30)] = 6
    mask[is_plume & (clouds <= 0.30)] = 5
    mask[is_plume & (clouds <= 0.01)] = 4
    mask[(clouds > 0.01) & ~is_plume] = 2
    mask[(clouds > 0.30) & ~is_plume] = 3

    n_pixels_no2 = np.sum(is_plume & (clouds <= 0.30))
    n_pixels_co2 = np.sum(is_plume & (clouds <= 0.01))
    n = np.sum(is_plume)

    tracer = {
        'CO2': 'CO$_2$',
        'NO2': 'NO$_2$'
    }[tracer]

    cmap = [
        '#a3a3a3',
        '#bcbcbc',
        '#eeeeee',
        'white',
        '#ff7f0e',
        '#ffb672',
        '#ffdfc2'
    ]
    cb_labels = [
        'outside\nswath',
        'satellite swath\n(cc$\,\leq\,$1%)',
        'satellite swath\n(1%$\,<\,$cc$\,\leq\,$30%)',
        'satellite swath\n(cc$\,>\,$30%)',
        '%s plume [%d]\n(cc$\,\leq\,$1%%)' % (tracer, np.sum(mask == 4)),
        '%s plume [%d]\n(1%%$\,<\,$cc$\,\leq\,$30%%)' % (tracer, np.sum(mask == 5)),
        '%s plume [%d]\n(cc$\,>\,$30%%)' % (tracer, np.sum(mask == 6)),
    ]


    is_cloudy = np.zeros(clouds.shape)
    is_cloudy[clouds > 0.01] = 0.5
    is_cloudy[clouds > 0.30] = 1.0

    fig = sc.map.make_level2_map(
        lonc, latc, mask, vmin=None, vmax=None, cmap=cmap, cb_labels=cb_labels,
        label='', is_discrete=True, bg_color=cmap[0], dright=0.06
    )
    ax, cax = fig.get_axes()

    if do_zoom:
        ax.set_xlim(0.96, 3.20)
        ax.set_ylim(4.63, 6.47)

    if label is not None:
        ax.text(0.0, 1.0, label, ha='left', va='top',
                 bbox=dict(facecolor='white', alpha=1.0),
                 transform=ax.transAxes
        )



def estimate_detection_limit(filename, tracer, error_std, footprint=49, p_thr=0.99, use_s5=False):
    """
    """
    if tracer == 'CO2':
        cloud_thr = 0.01
    else:
        cloud_thr = 0.30

    is_plume, info = plume_detection_algorithm(filename, tracer, use_epf=True,
            error_std=error_std, bg_type='ts', origin='Berlin', p_thr=p_thr,
            footprint=footprint, cloud_thr=cloud_thr, show=False, use_s5=use_s5)

    # read CO2 signal
    signal, clouds = sc.io.read_fields(filename, ['XCO2_BV', 'CLCT'],
            sc.io.filename2time(filename), correct_berlin=True)


    truths = info['truths']

    # TP for CO2 plume
    TP = info['C'].TP
    PPV = info['C'].PPV

    if TP < 1 or PPV < 0.80:
        return np.nan

    mask = truths.plume & (clouds < 0.01)
    signal = signal[mask]

    if TP > mask.sum():
        print('Warning: TP is larger than plume.')

    return signal[np.argsort(signal)][-TP:][0]



def create_file_with_detected_plumes(path='.'):
    products, data = io.read_results('Berlin', None, None, use_best=True,
                                  nsat=None)

    for product in products:
        d = data[product]

        # only use plume detections without large PPV rate
        # (0.8 is a good threshold based on visual inspection)
        valids = d.PPV >= 0.8

        fnames = d.filename[valids]
        n_pixels = d.TP[valids] + d.FP[valids]

        times = [sc.io.filename2time(fn) for fn in fnames]
        lon_eqs = [sc.io.process_filename(fn)[-1] for fn in fnames]

        d = pandas.DataFrame(np.transpose([lon_eqs, n_pixels]), index=times,
                             columns=['lon_eq', 'cloud-free pixels'])


        fn = os.path.join(path, '%s_plumes.dat' % product)

        d.to_csv(fn)


def extract_mc_detection(origin, tracer, ssp, q, ns, use_s5=False):

    product = io.get_product(tracer, ssp, q, ns, use_s5)

    path =  "/project/kug/smartcarb/results/%s/plume_detection_mc" % origin
    print(os.path.join(path, product))
    filenames = sorted(glob.glob(
        os.path.join(path, product, "*.nc")
    ))
    columns = ['utc', 'orbit', 'lon_eq', 'detected_pixels',
               'CO2_pixels', 'NO2_pixels', 'TP', 'FP', 'TN', 'FN', 'PPV']

    out_filename = '%s.dat' % product

    with open(out_filename, 'w') as out:

        out.write('\t'.join(columns))
        out.write('\n')

        for filename in filenames:

            sys.stdout.write('\r')
            sys.stdout.write(os.path.basename(filename))
            sys.stdout.flush()
            sat, orbit, date, lon_eq = sc.io.process_filename(filename)

            with xarray.open_dataset(filename) as data:

                if hasattr(data, 'CO2_true_plume'):
                    res = compute_performance(data)
                else:
                    res = {}

            out.write('%s\t%d\t%d\t' % (date, orbit, lon_eq))

            for key in ['P', 'CO2_pixels', 'NO2_pixels', 'TP', 'FP', 'TN', 'FN']:
                out.write('%d\t' % res.get(key, -9999))

            out.write('%f\n' % res.get('PPV', np.nan))

    print()



def extract_detection(origin, tracer, ssp, q, ns, use_s5=False):

    if use_s5:
        print('Sentinel-5 not implemented.')
        return

    path =  "/project/kug/smartcarb/plume_detection/%s/plume_detection" % origin
    product = io.get_product(tracer, ssp, q, ns, use_s5)

    filenames = sorted(glob.glob(
        os.path.join(path, product, "*.nc")
    ))

    data = []
    index = []
    columns = ['satellite', 'orbit', 'lon_eq', 'detected_pixels', 'CO2_pixels']

    for filename in filenames:
        sat, orbit, date, lon_eq = sc.io.process_filename(filename)

        index.append(date)
        line = [sat, orbit, lon_eq]

        with netCDF4.Dataset(filename) as nc:
            p = nc.variables['detected_plume'][:]
            clouds = nc.variables['clouds'][:]

            line.append(np.sum(p)) # all pixels
            line.append(np.sum(p[clouds <= 0.01]))

        data.append(line)

    out_fname = os.path.join(path, product+'.dat')

    data = pandas.DataFrame(data, index=index, columns=columns)
    data.to_csv(out_fname, sep='\t')

    return data


def show_detection(filenames):

    print(':'.join([
        'filename',
        #'d > 0',
        #'d > 10',
        #'d == nan',
        #'median(d>0)',
        'c > 0',
        'c > 10',
        'c == nan',
        'median(c>0)',
    ]))

    for filename in filenames:
        data = pandas.read_csv(filename, sep='\t', index_col=0,
                               parse_dates=True)

        d = data.detected_pixels

        # TODO: remove multiple detections without CO2 pixels
        # 0: no multiple
        # 1: yes but no CO2 pixels
        # 2: yes and more than 10 CO2 pixels
        if 'NO2' in filename:
            m = data.multiple_detections == 2
        else:
            m = data.multiple_detections == 1
        c = data.CO2_pixels

        print(os.path.basename(filename) + ':' + ':'.join([str(v) for v in [
            #np.sum((d > 0) | m),
            #np.sum((d > 10) | m),
            #sum(m),
            #int(round(np.median(d[d>0]), 0)),
            sum((c > 0) | m),
            sum((c > 9) | m),
            sum(m),
            int(round(np.median(c[c>0]), 0)),
        ]]))



def extract_table(tracer, ssp, q, ns, use_s5=False):

    path =  "/project/kug/smartcarb/plume_detection/Berlin/plume_detection"
    product = io.get_product(tracer, ssp, q, ns, use_s5)

    filenames = sorted(glob.glob(
        os.path.join(path, product, "*.nc")
    ))

    attrs = ['TP', 'TN', 'FP', 'FN', 'TPR', 'PPV']

    data = []
    index = []
    columns = ['satellite', 'orbit', 'lon_eq']
    columns += [a+'1' for a in attrs]
    columns += [a+'30' for a in attrs]

    for filename in filenames:
        sat, orbit, date, lon_eq = sc.io.process_filename(filename)

        index.append(date)
        line = [sat, orbit, lon_eq]

        with netCDF4.Dataset(filename) as nc:
            c1 = nc.variables['confusion_matrix_1']
            c30 = nc.variables['confusion_matrix_30']

            line += [c1.getncattr(a) for a in attrs]
            line += [c30.getncattr(a) for a in attrs]

        data.append(line)

    out_fname = os.path.join(path, product+'.dat')

    data = pandas.DataFrame(data, index=index, columns=columns)
    data.to_csv(out_fname, sep='\t')

    return data



def plot_number_of_pixels(ns, nsat=6, cc=30, ppv_thr=0.8):

    TP = 'TP%d' % cc
    FN = 'FN%d' % cc
    PPV = 'PPV%d' % cc

    case2label = {
        'NO2_low':  r'NO$_2$ (1$\times$10$^{15}$ cm$^{-2}$)',  # low
        'NO2_high': r'NO$_2$ (2$\times$10$^{15}$ cm$^{-2}$)',  # high
        'NO2_S5':   r'NO$_2$ Sentinel-5',
        'CO2_0.5':  r'CO$_2$ (0.5$\,$ppm)',
        'CO2_0.7':  r'CO$_2$ (0.7$\,$ppm)',
        'CO2_1.0':  r'CO$_2$ (1.0$\,$ppm)',
    }

    if not isinstance(ns, list):
        ns = [ns for _ in range(6)]
    cases = [
        ('CO2', 0.5, ns[0]),
        ('CO2', 0.7, ns[1]),
        ('CO2', 1.0, ns[2]),
        ('NO2', 'low', ns[3]),
        ('NO2', 'high', ns[4]),
        ('NO2', 'S5', ns[5])
    ]

    data = {}
    for tracer, ssp, n in cases:
        product = io.get_product(tracer, ssp, q=0.99, ns=n)
        data[product] = io.read_results(tracer, ssp, n, nsat=nsat)

    fig = plt.figure()
    ax = fig.add_subplot(111)

    bins = np.arange(0,1001)

    for i, product in enumerate(sorted(data.keys())):

        case = '_'.join(product.split('_')[:2])

        ls = {
            'NO2_low':     '--',
            'NO2_high':    '--',
            'NO2_S5': ':',
            'CO2_0.5':     '-',
            'CO2_0.7':     '-',
            'CO2_1.0':     '-',
        }[case]

        if i == 0:
            P = data[product][TP] + data[product][FN]
            n_plumes = sum(P > 0)

            values = np.array([sum(P > b) for b in bins], dtype='f4')
            #values = ndi.gaussian_filter1d(values, 5)
            ax.plot(bins, values, 'k-', label='actual plumes', zorder=5)
            print('real plume\t', '\t'.join('%d' % v for v in [
                n_plumes,
                sum(P > 100),
                100 * sum(P > 100) / n_plumes,
                np.mean(P),
                np.median(P),
            ]))
        v = data[product]
        values = np.array([sum((v[TP] > b) & (v['PPV30'] >= ppv_thr)) for b in bins],
                          dtype='f4')
        #values = ndi.gaussian_filter1d(values, 5)
        ax.plot(bins, values, ls=ls, label=case2label[case])


        values = v[TP][v['PPV30'] >= ppv_thr]

        print('%s\t' % product, '\t'.join('%d' % v for v in [
            len(values),
            sum(values > 100),
            100 * sum(values > 100) / n_plumes,
            np.mean(values),
            np.median(values),
        ]))


    ax.set_xlim(0,600)
    ax.set_ylim(0,nsat*10)

    ax.set_xlabel('number of pixels (N)')
    ax.set_ylabel('number of plumes with more\nthan N detected XCO$_2$ pixels')

    plt.xticks(np.arange(0,601,50))
    plt.yticks(np.arange(0,nsat*10+1,5))
    plt.grid(True)

    plt.legend(fontsize='small', loc=1, ncol=1, framealpha=0.8)
    plt.tight_layout()

    return data


def plot_power_plant_boxplot(origin):
    """
    Plot boxplot for power plants
    """
    # TODO: redo
    p_thr = 0.99
    footprint = 5

    # indices of the 9 Level-2 data where only Janschwalde was detected for
    # all instrument scenarios
    if origin == 'Janschwalde':
        fnames = np.loadtxt('/project/smartcarb/plume_detection/Level-2_Janschwalde.dat', dtype=bytes).astype(str)

        # excl. the following orbits, because plumes overlap or detection fails
        excl_orbits = [641, 656, 846, 1010, 1230, 2132, 2296, 2322,
              2501, 3280, 3485, 3593, 3608, 4018, 4469, 4551, 5084]

        ind = [i for i,fn in enumerate(fnames) if sc.io.filename2orbit(fn) not in excl_orbits]
        xmax = 350

    else:
        fnames = np.loadtxt('/project/smartcarb/plume_detection/Level-2_Lippendorf.dat', dtype=bytes).astype(str)

        # excl. the following orbits, because plumes overlap or detection fails
        excl_orbits = [1148, 1614, 1640, 2311, 2501, 2665, 2706, 2721, 3157, 3485,
                3500, 4059, 4074, 4238, 4525]

        ind = [i for i,fn in enumerate(fnames) if sc.io.filename2orbit(fn) not in excl_orbits]
        xmax = 200

    print(len(fnames), len(ind))

    # read data
    products, data = io.read_results(origin, p_thr, footprint)

    # extract positive predictions
    X = np.transpose([[data[p].TP[i] + data[p].FP[i] for i in ind] for p in products if 'low' not in p])

    labels = []
    for p in products:

        if 'low' in p:
            continue

        n = np.sum(np.array([data[p].TP[i] + data[p].FP[i] for i in ind]) > 10)
        print(p,n)

        labels.append({
            'NO2_high_cc30': r'NO$_2$ high$_{30}$ (%d)',  # high
            'NO2_high_cc1':  r'NO$_2$ high$_1$ (%d)',     # high
            'CO2_10': r'CO$_2$ high (%d)',
            'CO2_07': r'CO$_2$ mid (%d)',
            'CO2_05': r'CO$_2$ low (%d)'}[p] % n
        )


    fig = plt.figure()
    ax = fig.add_subplot(111)

    ax.boxplot(X, whis='range', vert=False)
    ax.set_xlabel('size of detected plume')
    ax.set_xlim(-5,xmax)

    ax.set_xticks(np.arange(0,xmax+1,50))
    ax.grid(True)

    ax.set_yticklabels(labels)

    plt.tight_layout()

    return data



def plot_number_of_plumes(tracer, ssp, nsat=6, ppv_thr=0.8, cc=30,
                          use_s5=False):
    PPV = 'PPV%d' % cc
    TP = 'TP%d' % cc

    if use_s5:
        ns = [1,5,9,13,25]
    else:
        ns = [1,5,9,13,25,37,49,81,121]

    data = {}
    for n in ns:
        data[n] = io.read_results(tracer, ssp, ns=n, q=0.99, nsat=nsat,
                               use_s5=use_s5)

    n_swaths = len(data[ns[0]])
    print('number of swaths:', n_swaths)


    n_ppv = np.array([
        sum(data[n][PPV] >= 0.8) for n in ns
    ]) / n_swaths

    fig = plt.figure()
    ax = fig.add_subplot(111)

    markers = 'osv^d><'

    for i,tp_min in enumerate([0,100,250,500]):
        n_both = np.array([
            sum((data[n][PPV] >= ppv_thr ) & (data[n][TP] > tp_min)) for n in ns
        ]) / n_swaths

        if tp_min == 100:
            print('max(50):', ns[np.argmax(n_both)])

        ax.plot(ns, n_both, ls='-', marker=markers[i],
                label='TP $>$ %d and PPV $\geq$ %.1f' % (tp_min, ppv_thr))


    ax.set_xlim(0, ns[-1]+2)
    ax.set_xlabel('size of neighborhood')

    ax.set_xticks(ns)
    ax.set_ylabel('number of plumes (%)')

    ax.set_ylim(0,1.02)
    ax.grid(True)

    plt.legend()
    plt.tight_layout()



def plot_performance_per_footprint(tracer, ssp, q=0.99, cc=30):

    use_s5 = ssp == 'S5'

    if use_s5:
        ns = [25,13,9,5,1]
        ns = [13,9,5,1]
    else:
        ns = [121,81,49,37,25,13,9,5,1]

    rates = ['TP%d' % cc, 'FP%d' % cc, 'TPR%d' % cc, 'PPV%d' % cc]

    rate2long_name = {
        'TP%d' % cc: 'true positive',
        'FP%d' % cc: 'false positive',
        'TPR%d' % cc: 'true-positive rate',
        'PPV%d' % cc: 'positive predictive value',
    }

    data = {}
    for n in ns:
        data[n] = io.read_results(tracer, ssp, ns=n, q=0.99, nsat=6, use_s5=use_s5)

    fig = plt.figure(figsize=(7,5))

    for i,rate in enumerate(rates):
        ax = fig.add_subplot(2,2,i+1)

        # remove nans (e.g. if TP == 0)
        values = [data[n][rate] for n in ns]
        values = [v[np.isfinite(v)] for v in values]

        # add 0.80 PPV line (useful threshold for removing bad detections)
        if rate.startswith('PPV'):
            ax.vlines([0.80], -1, len(ns)+1, linestyles='--', colors='k')

        ax.boxplot(values, whis='range', sym='', vert=False)
        ax.set_title('(%s) %s' % ('abcd'[i], rate2long_name[rate]))

        if rate in ['TPR%d' % cc, 'PPV%d' % cc]:
            ax.set_xlim(-0.01,1.01)
        else:
            ax.set_xlim(0,1000)

        if i in [0,2]:
            ax.set_yticklabels(ns, rotation=0, fontsize='small')
            ax.set_ylabel('size of neighborhood')
        else:
            ax.set_yticklabels([])

    plt.tight_layout()

    return data







def plot_performance(origin, p_thr, footprint, use_rates=True):


    product2label = {
        'NO2_low_cc30':  r'NO$_2$ (1$\times$10$^{15}$, 30%)',  # low
        'NO2_high_cc30': r'NO$_2$ (2$\times$10$^{15}$, 30%)', # high
        'NO2_low_cc1':   r'NO$_2$ (1$\times$10$^{15}$, 1%)',  # low
        'NO2_high_cc1':  r'NO$_2$ (2$\times$10$^{15}$, 1%)',  # high
        'NO2_S5_cc30':   r'NO$_2$ Sentinel-5',
        'CO2_05':        r'CO$_2$ (0.5$\,$ppm)',
        'CO2_07':        r'CO$_2$ (0.7$\,$ppm)',
        'CO2_10':        r'CO$_2$ (1.0$\,$ppm)',
    }


    if use_rates:
        rates = ['TPR', 'PPV']
    else:
        rates = ['TP', 'FP']

    rates = ['TP', 'FP', 'TPR', 'PPV']

    rate2long_name = {
        'TP': 'true positive (TP)',
        'FP': 'false positive (FP)',
        'TPR': 'true-positive rate (TPR)',
        'PPV': 'positive predictive value (PPV)',
        'F1': 'F1 score'
    }

    products, data = read_results(origin, p_thr, footprint, use_best=p_thr=='best')
    labels = [product2label[p] for p in products]

    text = '\n'.join([
        '$\mathbf{Cases}:$',
        'CO$_2$ low: $\sigma_{ref}$ = 0.5 ppm',
        'CO$_2$ mid: $\sigma_{ref}$ = 0.7 ppm',
        'CO$_2$ high: $\sigma_{ref}$ = 1.0 ppm',
        'NO$_2$ S5: Sentinel-5',
        'NO$_2$ high$_1$: $\sigma_{ref}$ = 2$\\times$10$^{15}$ cm$^{-2}$, cc < 1$\%$',
        'NO$_2$ low$_1$: $\sigma_{ref}$ = 1$\\times$10$^{15}$ cm$^{-2}$, cc < 1$\%$',
        'NO$_2$ high$_{30}$: $\sigma_{ref}$ = 2$\\times$10$^{15}$ cm$^{-2}$, cc < 30$\%$',
        'NO$_2$ low$_{30}$: $\sigma_{ref}$ = 1$\\times$10$^{15}$ cm$^{-2}$, cc < 30$\%$',
    ])


    fig = plt.figure(figsize=(8,6))
    #fig.text(0.69,0.47, text, va='top', fontsize='small')



    valids = np.all([np.isfinite(data[p][r])
                     for r, p in itertools.product(rates, products)],
                    axis=0)

    for p in products:
        n = sum((data[p].TP >= 50) & (data[p].PPV >= 0.8))
        print(p, n, 100 * n / len(data[p]))

    for i,rate in enumerate(rates):

        ax = fig.add_subplot(2,2,i+1)

        # remove nans (e.g. if TP == 0)
        if rate in ['TPR', 'PPV', 'F1']:
            values = [data[p][rate][valids] for p in products]
        else:
            values = [data[p][rate] for p in products]


        ax.boxplot(values, whis='range', vert=False)
        #ax.set_title('(%s) %s' % ('abc'[i], rate2long_name[rate]))
        ax.set_xlabel(rate2long_name[rate])

        if rate in ['TPR', 'PPV', 'F1']:
            ax.set_xlim(-0.01,1.01)
        else:
            if footprint == 1:
                ax.set_xlim(-10,600)
            else:
                ax.set_xlim(-10,1000)

        if i in [0,2]:
            ax.set_yticklabels(labels, rotation=0, fontsize='small')
        else:
            ax.set_yticklabels([])


    plt.tight_layout()

    return data


def get_level2_filenames(origin, nsat=None):

    return sorted(glob.glob(
        '/project/smartcarb/data1/level2/Sentinel_7_CO2/*.nc'
    ))

    path = "/project/kug/smartcarb/plume_detection/"
    filename = os.path.join(path, origin, "%sPlume.dat" % origin)
    data = pandas.read_csv(filename, sep='\t', index_col=2, parse_dates=True)

    if nsat is not None:
        data = sc.misc.filter_dataset(data, nsat=nsat)

    if origin == 'Berlin':
        nmin = 99
        level2_filenames = [
            sc.io.create_filename(data.index[i], None, 'level2', 'Sentinel_7_CO2',
                               data.orbit[i], data.lon_eq[i])
            for i in range(data.shape[0])
            if data['pixels (<1%)'][i] > nmin
        ]
    else:
        level2_filenames = [
            sc.io.create_filename(data.index[i], None, 'level2', 'Sentinel_7_CO2',
                               data.orbit[i], data.lon_eq[i])
            for i in range(data.shape[0])
        ]


    return level2_filenames



def main(filename, tracer, ssp, origin='Berlin', q=0.99, ns=37, cloud_thr=0.01,
         use_s5=False, make_image=False, nsamples=1, seed=None):

    product = io.get_product(tracer, ssp, q, ns, use_s5=use_s5)
    output_path = '/project/kug/smartcarb/results'

    if nsamples == 1:
        output_path = os.path.join(output_path, origin, 'plume_detection',
                                   product)
    else:
        output_path = os.path.join(output_path, origin, 'plume_detection_mc',
                                   product)

    os.makedirs(output_path, exist_ok=True)

    print(os.path.basename(filename), product)

    for n in range(nsamples):

        data = plume_detection_algorithm(filename, tracer, ssp=ssp,
                                         cloud_thr=cloud_thr, seed=seed,
                                         ns=ns, q=q, origin=origin,
                                         use_s5=use_s5, show=make_image,
                                         use_constant_emissions=False)

        # skip if detected plume not available (e.g. no overpass over origin)
        # stop if no detection (currenlty only happens if origin not in swath)
        if 'detected_plume' not in data:
            break

        if nsamples == 1:
            suffix = product
        else:
            suffix = '%s_n%02d' % (product, n)

        # output filename
        output_fname = os.path.basename(filename).replace('.nc', '_%s.nc' % suffix)
        output_fname = os.path.join(output_path, output_fname)

        # set compression level and save to netcdf
        data.attrs['time'] = str(data.attrs['time'])
        encoding = {var: {'zlib': True, 'complevel': 9} for var in data.data_vars}
        data.to_netcdf(output_fname, encoding=encoding)

        print(os.path.join(output_path, output_fname))

        # savefig and close
        if make_image:
            img_filename = os.path.join(
                output_path,
                os.path.basename(filename).replace('.nc', '_%s.png' % suffix)
            )
            plt.savefig(img_filename)
            plt.close()



if __name__ == '__name__':
    pass





