"""Constrained propagation fit of the frozen CHIME/FRB Catalog 1 cutout of FRB 20180725A.

Three models share one burst shape (Gaussian of width sigma, one-sided scattering tail with
tau(nu) = tau_600 (nu / 600 MHz)^-4, residual dispersion delta_DM):
  H0           one burst;
  H_chromatic  one burst plus a copy r times as bright whose delay scales as nu^-2, as a plasma-lens
               echo does when the lens dispersion term dominates;
  H_achromatic one burst plus a copy r times as bright at a fixed delay.
Each channel has its own amplitude and baseline; the copy ratio r and the delay are shared by all
channels, which is the propagation constraint. The burst shape is held at the H0 fit while the copy
is scanned. The null distribution of the echo statistic comes from synthetic white noise with the
off-pulse scatter of each channel, so it is only as good as that noise model.

Units: time in ms, frequency in MHz, dispersion measure in pc cm^-3. Requires numpy and scipy.
"""
import json
from pathlib import Path

import numpy as np
from scipy.optimize import minimize
from scipy.special import erfcx

HERE = Path(__file__).resolve().parent
CUTOUT_NAME = 'chime-frb20180725a-intensity.json'
REPOSITORY_COPY = HERE.parents[2] / 'labs/interstellar-propagation/data' / CUTOUT_NAME
DISPERSION_CONSTANT_MS = 4.148808  # ms GHz^2 per pc cm^-3
REFERENCE_MHZ = 600.0
OVERSAMPLING = 16
OFF_PULSE_BINS = np.r_[0:8, 22:38]
CHROMATIC_DELAYS_MS = np.round(np.arange(0.5, 9.51, 0.1), 3)   # the 400 MHz copy stays inside the window
ACHROMATIC_DELAYS_MS = np.round(np.arange(0.5, 20.01, 0.1), 3)
COPY_RATIOS = np.round(np.arange(0.05, 1.501, 0.05), 3)


def load_cutout(path=None):
    path = Path(path) if path else (HERE / CUTOUT_NAME if (HERE / CUTOUT_NAME).exists() else REPOSITORY_COPY)
    record = json.loads(path.read_text())
    intensity = np.array(record['intensity'], dtype=float)
    frequency = np.array(record['frequency_mhz'], dtype=float)
    time = np.array(record['time_from_window_start_ms'], dtype=float)
    good = ~np.isnan(intensity).any(axis=1)
    return {'path': path, 'intensity': intensity[good], 'frequency': frequency[good], 'time': time,
            'sample_ms': float(np.median(np.diff(time))), 'masked_channels': int((~good).sum())}


def channel_noise(intensity):
    """Robust off-pulse scatter per channel (median absolute deviation scaled to a Gaussian sigma)."""
    off = intensity[:, OFF_PULSE_BINS]
    return 1.4826 * np.median(np.abs(off - np.median(off, axis=1, keepdims=True)), axis=1)


def pulse_templates(time, sample_ms, arrival, sigma, tau):
    """Unit-area exponentially modified Gaussian, averaged over each sample. arrival and tau are per channel."""
    offsets = (np.arange(OVERSAMPLING) + 0.5) / OVERSAMPLING - 0.5
    fine = time[None, :, None] + offsets[None, None, :] * sample_ms
    lag = fine - arrival[:, None, None]
    tau = np.maximum(tau, 1e-4)[:, None, None] * np.ones_like(lag)
    argument = (sigma / tau - lag / sigma) / np.sqrt(2.0)
    shape = np.empty_like(lag)
    regular = argument > -5.0
    shape[regular] = np.exp(-0.5 * (lag[regular] / sigma) ** 2) * erfcx(argument[regular]) / (2.0 * tau[regular])
    # Far into the tail erfc -> 2, so the shape is the pure exponential; this avoids inf * 0.
    tail = ~regular
    shape[tail] = np.exp(0.5 * (sigma / tau[tail]) ** 2 - lag[tail] / tau[tail]) / tau[tail]
    return shape.mean(axis=2)


def burst_arrival(frequency, arrival_ms, residual_dm):
    return arrival_ms + DISPERSION_CONSTANT_MS * residual_dm * ((frequency / 1000.0) ** -2 - (REFERENCE_MHZ / 1000.0) ** -2)


def scattering_time(frequency, tau_600):
    return tau_600 * (frequency / REFERENCE_MHZ) ** -4


def weighted_projection(design, noise):
    """Per-channel weighted least-squares operator: coefficients = P @ data. design has shape (C, T, K)."""
    weighted = design / noise[:, None, None]
    normal = np.einsum('ctk,ctl->ckl', weighted, weighted)
    return np.einsum('ckl,ctl->ckt', np.linalg.pinv(normal), weighted) / noise[:, None, None]


def chi_square(design, data, noise):
    projection = weighted_projection(design, noise)
    coefficients = np.einsum('ckt,...ct->...ck', projection, data)
    model = np.einsum('ctk,...ck->...ct', design, coefficients)
    return (((data - model) / noise[:, None]) ** 2).sum(axis=(-1, -2)), model


def burst_templates(cutout, parameters, delays=None):
    arrival_ms, sigma, log_tau, residual_dm = parameters
    frequency, time, sample = cutout['frequency'], cutout['time'], cutout['sample_ms']
    arrival = burst_arrival(frequency, arrival_ms, residual_dm)
    tau = scattering_time(frequency, np.exp(log_tau))
    return pulse_templates(time, sample, arrival if delays is None else arrival + delays, sigma, tau)


def with_baseline(template):
    return np.stack([template, np.ones_like(template)], axis=2)


def fit_single_burst(cutout, noise, data=None):
    data = cutout['intensity'] if data is None else data
    def objective(parameters):
        if not (0.15 < parameters[1] < 6 and -9 < parameters[2] < 2.5):
            return 1e12
        return float(chi_square(with_baseline(burst_templates(cutout, parameters)), data, noise)[0])
    starts = [[13.0, 0.8, np.log(0.3), 0.0], [13.5, 0.5, np.log(1.0), 0.5], [12.8, 1.2, np.log(0.05), -0.5]]
    best = min((minimize(objective, start, method='Nelder-Mead', options={'xatol': 1e-4, 'fatol': 1e-4, 'maxiter': 4000}) for start in starts), key=lambda result: result.fun)
    return best.x, float(best.fun)


def copy_delays(cutout, delay, chromatic):
    return delay * (cutout['frequency'] / REFERENCE_MHZ) ** -2 if chromatic else np.full(cutout['frequency'].shape, delay)


def echo_scan(cutout, noise, parameters, data, chromatic):
    """Largest drop in chi-square from adding one copy, over the delay and ratio grids, per data set.

    data may carry leading simulation axes. Returns (statistic, best delay, best ratio) arrays.
    """
    main = burst_templates(cutout, parameters)
    base, _ = chi_square(with_baseline(main), data, noise)
    delays = CHROMATIC_DELAYS_MS if chromatic else ACHROMATIC_DELAYS_MS
    best = np.full(base.shape, -np.inf)
    best_delay = np.zeros(base.shape)
    best_ratio = np.zeros(base.shape)
    for delay in delays:
        copy = burst_templates(cutout, parameters, copy_delays(cutout, delay, chromatic))
        for ratio in COPY_RATIOS:
            trial, _ = chi_square(with_baseline(main + ratio * copy), data, noise)
            improvement = base - trial
            better = improvement > best
            best = np.where(better, improvement, best)
            best_delay = np.where(better, delay, best_delay)
            best_ratio = np.where(better, ratio, best_ratio)
    return best, best_delay, best_ratio


def best_fit_model(cutout, noise, parameters, data):
    return chi_square(with_baseline(burst_templates(cutout, parameters)), data, noise)[1]


def inject_echo(cutout, noise, parameters, delay_ms, ratio, seed):
    """H0 best fit plus a chromatic copy, plus synthetic noise; then the full procedure as for the data."""
    generator = np.random.default_rng(seed)
    main = burst_templates(cutout, parameters)
    projection = weighted_projection(with_baseline(main), noise)
    coefficients = np.einsum('ckt,ct->ck', projection, cutout['intensity'])
    copy = burst_templates(cutout, parameters, copy_delays(cutout, delay_ms, True))
    signal = coefficients[:, :1] * (main + ratio * copy) + coefficients[:, 1:]
    data = signal + generator.normal(size=signal.shape) * noise[:, None]
    refit, _ = fit_single_burst(cutout, noise, data)
    statistic, best_delay, best_ratio = echo_scan(cutout, noise, refit, data, chromatic=True)
    return {'injected_delay_ms': delay_ms, 'injected_ratio': ratio, 'seed': seed,
            'refit_sigma_ms': round(float(refit[1]), 4), 'refit_tau_600_ms': round(float(np.exp(refit[2])), 4),
            'max_delta_chi_square': round(float(statistic), 2), 'recovered_delay_ms': round(float(best_delay), 2),
            'recovered_ratio': round(float(best_ratio), 3)}


def run(trials=300, seed=20261114):
    cutout = load_cutout()
    noise = channel_noise(cutout['intensity'])
    parameters, chi_square_h0 = fit_single_burst(cutout, noise)
    degrees = cutout['intensity'].size - 4 - 2 * cutout['intensity'].shape[0]
    reduced = chi_square_h0 / degrees
    generator = np.random.default_rng(seed)
    model = best_fit_model(cutout, noise, parameters, cutout['intensity'])
    simulations = model[None] + generator.normal(size=(trials,) + model.shape) * noise[None, :, None]
    searches = {}
    for name, chromatic in (('chromatic_copy', True), ('achromatic_copy', False)):
        observed, delay, ratio = echo_scan(cutout, noise, parameters, cutout['intensity'], chromatic)
        null, _, _ = echo_scan(cutout, noise, parameters, simulations, chromatic)
        observed = float(observed)
        searches[name] = {
            'max_delta_chi_square': round(observed, 2), 'best_delay_ms_at_600MHz': round(float(delay), 2), 'best_ratio': round(float(ratio), 3),
            'null_95th_percentile': round(float(np.percentile(null, 95)), 2), 'null_99th_percentile': round(float(np.percentile(null, 99)), 2),
            'false_alarm_probability_white_noise': round(float((null >= observed).mean()), 4),
            # If the excess chi-square of H0 is unmodelled noise, every statistic shrinks by the reduced chi-square.
            'max_delta_chi_square_noise_rescaled': round(observed / reduced, 2),
            'false_alarm_probability_noise_rescaled': round(float((null >= observed / reduced).mean()), 4),
        }
    return {
        'input': {'file': cutout['path'].name, 'channels_used': int(cutout['intensity'].shape[0]),
                  'channels_fully_masked': cutout['masked_channels'], 'time_bins': int(cutout['time'].size),
                  'sample_ms': round(cutout['sample_ms'], 6), 'off_pulse_bins': '0-7 and 22-37'},
        'single_burst_fit': {'arrival_ms_at_600MHz': round(float(parameters[0]), 4), 'sigma_ms': round(float(parameters[1]), 4),
                             'tau_600_ms': round(float(np.exp(parameters[2])), 4), 'residual_dm_pc_cm3': round(float(parameters[3]), 4),
                             'chi_square': round(chi_square_h0, 2), 'degrees_of_freedom': int(degrees), 'reduced_chi_square': round(reduced, 4)},
        'echo_search': {'chromatic_delay_grid_ms': [float(CHROMATIC_DELAYS_MS[0]), float(CHROMATIC_DELAYS_MS[-1]), 0.1],
                        'achromatic_delay_grid_ms': [float(ACHROMATIC_DELAYS_MS[0]), float(ACHROMATIC_DELAYS_MS[-1]), 0.1],
                        'ratio_grid': [float(COPY_RATIOS[0]), float(COPY_RATIOS[-1]), 0.05],
                        'null_trials': trials, 'seed': seed, **searches},
        'injections': [inject_echo(cutout, noise, parameters, 4.0, 0.5, seed + 1),
                       inject_echo(cutout, noise, parameters, 4.0, 0.15, seed + 2),
                       inject_echo(cutout, noise, parameters, 0.5, 0.5, seed + 3)],
    }


if __name__ == '__main__':
    print(json.dumps(run(), indent=1))
