"""Measure the galaxy correlation function of a small DESI DR1 BGS patch and try a BAO fit.

Inputs (same directory unless given on the command line):
  desi-dr1-bgs-sgc-patch-galaxies.csv   15 699 BGS_BRIGHT-21.5 galaxies, DESI DR1 LSS v1.5, SGC patch
  desi-dr1-bgs-sgc-patch-randoms.csv    47 097 randoms from random file 0, subsampled to 3x the galaxies
  bao-template-desi-fiducial.csv        damped linear xi(r) from CLASS at z = 0.295 (offline reference)

Steps:
  1. redshift -> comoving distance in the DESI DR1 fiducial flat LCDM, in h^-1 Mpc
  2. weighted pair counts DD, DR, RR in 10 h^-1 Mpc bins up to 200 h^-1 Mpc (scipy cKDTree)
  3. Landy-Szalay monopole xi(s) and a jackknife covariance from the occupied cells of an 8 x 4 sky grid
  4. a restricted isotropic BAO fit, B xi_t(alpha s) + a0 + a1/s + a2/s^2 over 50-150 h^-1 Mpc
  5. four failure cases that change one step at a time

Usage: python measure_desi_patch_correlation.py [--input DIR] [--output result.json] [--skip-failures]
Needs numpy and scipy (versions in requirements.txt). Takes a few minutes on a laptop.

Data: DESI Collaboration et al., "Data Release 1 of the Dark Energy Spectroscopic Instrument",
arXiv:2503.14745. CC BY 4.0. The patch is a modified subset; see desi-patch-provenance.json.
"""
import argparse
import json
from pathlib import Path

import numpy as np
from scipy.spatial import cKDTree

SPEED_OF_LIGHT_OVER_100 = 2997.92458  # c / (100 km/s/Mpc) in Mpc, so distances come out in h^-1 Mpc
FIDUCIAL = {"omega_b": 0.02237, "omega_cdm": 0.1200, "omega_nu": 0.00064420, "h": 0.6736}
OMEGA_MATTER = (FIDUCIAL["omega_b"] + FIDUCIAL["omega_cdm"] + FIDUCIAL["omega_nu"]) / FIDUCIAL["h"] ** 2
EDGES = np.arange(0.0, 200.1, 10.0)
CENTERS = 0.5 * (EDGES[1:] + EDGES[:-1])
FIT_MIN, FIT_MAX = 50.0, 150.0
ALPHAS = np.round(np.arange(0.80, 1.2001, 0.002), 4)
RA_REGIONS, DEC_REGIONS = 8, 4
EFFECTIVE_REDSHIFT = 0.295
FIDUCIAL_RS_DRAG = 99.0846  # h^-1 Mpc, CLASS for the fiducial cosmology (147.097 Mpc x 0.6736)


def load_catalog(path):
    table = np.loadtxt(path, delimiter=",", skiprows=1)
    return {"ra": table[:, 0], "dec": table[:, 1], "z": table[:, 2], "weight": table[:, 3], "weight_fkp": table[:, 4]}


def comoving_distance(redshift, omega_matter=OMEGA_MATTER):
    """Flat LCDM with radiation neglected (below 0.01 per cent at z < 0.4); the massive neutrino counts as matter."""
    grid = np.linspace(0.0, 0.5, 5001)
    inverse_e = 1.0 / np.sqrt(omega_matter * (1 + grid) ** 3 + 1 - omega_matter)
    cumulative = np.concatenate([[0.0], np.cumsum(0.5 * (inverse_e[1:] + inverse_e[:-1]) * np.diff(grid))])
    return SPEED_OF_LIGHT_OVER_100 * np.interp(redshift, grid, cumulative)


def cartesian(catalog, distance_scale=1.0):
    distance = comoving_distance(catalog["z"]) * distance_scale
    ra, dec = np.radians(catalog["ra"]), np.radians(catalog["dec"])
    return np.column_stack([distance * np.cos(dec) * np.cos(ra), distance * np.cos(dec) * np.sin(ra), distance * np.sin(dec)])


def jackknife_regions(catalog):
    ra = np.where(catalog["ra"] >= 180, catalog["ra"] - 360, catalog["ra"])
    column = np.clip(((ra + 20.0) / 40.0 * RA_REGIONS).astype(int), 0, RA_REGIONS - 1)
    row = np.clip(((catalog["dec"] + 10.0) / 20.0 * DEC_REGIONS).astype(int), 0, DEC_REGIONS - 1)
    return row * RA_REGIONS + column


def binned(tree_a, tree_b, weights_a, weights_b):
    """Weighted ordered-pair counts in each separation bin; pairs at zero separation (self-pairs) drop out."""
    if len(weights_a) == 0 or len(weights_b) == 0:
        return np.zeros(len(EDGES) - 1)
    cumulative = tree_a.count_neighbors(tree_b, EDGES, weights=(weights_a, weights_b))
    return np.diff(cumulative)


def pair_counts(data_positions, data_weights, data_regions, random_positions, random_weights, random_regions):
    """Total counts and, for each jackknife region k, the counts with region k removed."""
    data_tree, random_tree = cKDTree(data_positions), cKDTree(random_positions)
    totals = {"DD": binned(data_tree, data_tree, data_weights, data_weights),
              "DR": binned(data_tree, random_tree, data_weights, random_weights),
              "RR": binned(random_tree, random_tree, random_weights, random_weights)}
    norms = {"DD": data_weights.sum() ** 2 - (data_weights ** 2).sum(), "DR": data_weights.sum() * random_weights.sum(),
             "RR": random_weights.sum() ** 2 - (random_weights ** 2).sum()}
    # Regions with no randoms lie outside the footprint (two of the 32 here); leaving them out would
    # repeat the full sample, so the jackknife uses only occupied regions.
    without = []
    for region in np.unique(random_regions):
        data_in, random_in = data_regions == region, random_regions == region
        d_tree, r_tree = cKDTree(data_positions[data_in]), cKDTree(random_positions[random_in])
        wd, wr = data_weights[data_in], random_weights[random_in]
        dd_k_all, dd_kk = binned(d_tree, data_tree, wd, data_weights), binned(d_tree, d_tree, wd, wd)
        rr_k_all, rr_kk = binned(r_tree, random_tree, wr, random_weights), binned(r_tree, r_tree, wr, wr)
        dr_k_all, rd_k_all, dr_kk = binned(d_tree, random_tree, wd, random_weights), binned(r_tree, data_tree, wr, data_weights), binned(d_tree, r_tree, wd, wr)
        counts = {"DD": totals["DD"] - 2 * dd_k_all + dd_kk, "RR": totals["RR"] - 2 * rr_k_all + rr_kk, "DR": totals["DR"] - dr_k_all - rd_k_all + dr_kk}
        wd_out, wr_out = data_weights[~data_in], random_weights[~random_in]
        local_norms = {"DD": wd_out.sum() ** 2 - (wd_out ** 2).sum(), "DR": wd_out.sum() * wr_out.sum(), "RR": wr_out.sum() ** 2 - (wr_out ** 2).sum()}
        without.append((counts, local_norms))
    return totals, norms, without


def landy_szalay(counts, norms, normalise=True):
    if not normalise:
        return (counts["DD"] - 2 * counts["DR"] + counts["RR"]) / counts["RR"]
    dd, dr, rr = counts["DD"] / norms["DD"], counts["DR"] / norms["DR"], counts["RR"] / norms["RR"]
    return (dd - 2 * dr + rr) / rr


def measure(galaxies, randoms, use_weights=True, distance_scale=1.0, random_fraction=1.0, normalise=True):
    random_keep = np.ones(len(randoms["z"]), dtype=bool)
    if random_fraction < 1:
        random_keep = np.random.default_rng(65).random(len(randoms["z"])) < random_fraction
    subset = {key: value[random_keep] for key, value in randoms.items()}
    weight = lambda catalog: catalog["weight"] * catalog["weight_fkp"] if use_weights else np.ones(len(catalog["z"]))
    totals, norms, without = pair_counts(cartesian(galaxies, distance_scale), weight(galaxies), jackknife_regions(galaxies),
                                         cartesian(subset, distance_scale), weight(subset), jackknife_regions(subset))
    xi = landy_szalay(totals, norms, normalise)
    samples = np.array([landy_szalay(counts, local, normalise) for counts, local in without])
    count = len(samples)
    deviation = samples - samples.mean(axis=0)
    covariance = (count - 1) / count * np.einsum("ki,kj->ij", deviation, deviation)
    return {"xi": xi, "covariance": covariance, "pairs": totals, "regions": count}


def load_template(path):
    table = np.loadtxt(path, delimiter=",", skiprows=1)
    return table[:, 0], table[:, 1]


def fit_bao(xi, covariance, regions, template_r, template_xi):
    """Grid over alpha; the amplitude B >= 0 and three broadband terms are solved by generalised least squares."""
    use = (CENTERS > FIT_MIN) & (CENTERS < FIT_MAX)
    s, data = CENTERS[use], xi[use]
    bins = use.sum()
    hartlap = (regions - bins - 2) / (regions - 1)
    inverse = hartlap * np.linalg.inv(covariance[np.ix_(use, use)])
    broadband = np.column_stack([np.ones_like(s), 100 / s, (100 / s) ** 2])

    def solve(design):
        normal = design.T @ inverse @ design
        parameters = np.linalg.solve(normal, design.T @ inverse @ data)
        residual = data - design @ parameters
        return float(residual @ inverse @ residual), parameters

    chi_without, _ = solve(broadband)
    chi = []
    for alpha in ALPHAS:
        value, parameters = solve(np.column_stack([np.interp(alpha * s, template_r, template_xi), broadband]))
        chi.append(value if parameters[0] > 0 else np.inf)
    chi = np.array(chi)
    best = int(np.argmin(chi))
    inside = ALPHAS[chi <= chi[best] + 1]
    return {"bins": int(bins), "jackknifeRegions": int(regions), "hartlapFactor": round(float(hartlap), 4),
            "alpha": float(ALPHAS[best]), "alphaLow": float(inside.min()), "alphaHigh": float(inside.max()),
            "intervalTouchesGridEdge": bool(inside.min() <= ALPHAS[0] or inside.max() >= ALPHAS[-1]),
            "chiSquared": round(float(chi[best]), 3), "chiSquaredWithoutTemplate": round(chi_without, 3), "degreesOfFreedom": int(bins - 5),
            "peakSignificance": round(float(np.sqrt(max(0.0, chi_without - chi[best]))), 3),
            "profile": [{"alpha": float(a), "deltaChiSquared": (round(float(c - chi[best]), 3) if np.isfinite(c) else None)} for a, c in zip(ALPHAS[::10], chi[::10])]}


def desi_expected_alpha():
    """alpha_iso implied by the published DESI DR1 BGS result D_V / r_d = 7.93 +- 0.15 at z_eff = 0.30 (Table 18)."""
    z = EFFECTIVE_REDSHIFT
    transverse = comoving_distance(np.array([z]))[0]
    hubble = SPEED_OF_LIGHT_OVER_100 / np.sqrt(OMEGA_MATTER * (1 + z) ** 3 + 1 - OMEGA_MATTER)
    volume = (z * transverse ** 2 * hubble) ** (1 / 3)
    fiducial_ratio = volume / FIDUCIAL_RS_DRAG
    return {"fiducialDVOverRd": round(float(fiducial_ratio), 4), "publishedDVOverRd": 7.93, "publishedError": 0.15,
            "alphaIso": round(7.93 / float(fiducial_ratio), 4), "alphaIsoError": round(0.15 / float(fiducial_ratio), 4)}


def summary(result):
    xi, error = result["xi"], np.sqrt(np.diag(result["covariance"]))
    return [{"s": float(s), "xi": round(float(x), 6), "error": round(float(e), 6), "s2xi": round(float(s * s * x), 3), "s2xiError": round(float(s * s * e), 3)} for s, x, e in zip(CENTERS, xi, error)]


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--input", default=str(Path(__file__).resolve().parent))
    parser.add_argument("--output", default="desi-patch-result.json")
    parser.add_argument("--skip-failures", action="store_true")
    arguments = parser.parse_args()
    folder = Path(arguments.input)
    galaxies, randoms = load_catalog(folder / "desi-dr1-bgs-sgc-patch-galaxies.csv"), load_catalog(folder / "desi-dr1-bgs-sgc-patch-randoms.csv")
    template_r, template_xi = load_template(folder / "bao-template-desi-fiducial.csv")
    print(f"galaxies {len(galaxies['z'])}, randoms {len(randoms['z'])}, Omega_m {OMEGA_MATTER:.5f}")
    main_result = measure(galaxies, randoms)
    fit = fit_bao(main_result["xi"], main_result["covariance"], main_result["regions"], template_r, template_xi)
    print(f"alpha = {fit['alpha']:.3f}, Delta chi^2 = 1 range {fit['alphaLow']:.3f}-{fit['alphaHigh']:.3f}, peak significance {fit['peakSignificance']:.2f}")
    output = {
        "version": "ac65-desi-patch-result-1",
        "provenance": "real DESI DR1 data (small patch); template is an offline CLASS reference",
        "units": "separations in h^-1 Mpc, fiducial flat LCDM Omega_m = %.5f" % OMEGA_MATTER,
        "counts": {"galaxies": int(len(galaxies["z"])), "randoms": int(len(randoms["z"]))},
        "correlation": summary(main_result),
        "correlationMatrixFitRange": np.round(np.corrcoef(main_result["covariance"])[np.ix_((CENTERS > FIT_MIN) & (CENTERS < FIT_MAX), (CENTERS > FIT_MIN) & (CENTERS < FIT_MAX))], 3).tolist(),
        "fit": fit,
        "publishedComparison": desi_expected_alpha(),
    }
    if not arguments.skip_failures:
        failures = {}
        cases = {
            "no-weights": {"use_weights": False},
            "one-third-of-the-randoms": {"random_fraction": 1 / 3},
            "unnormalised-pair-counts": {"normalise": False},
            "distances-in-mpc-not-hinv-mpc": {"distance_scale": 1 / FIDUCIAL["h"]},
        }
        for name, options in cases.items():
            result = measure(galaxies, randoms, **options)
            failures[name] = {"options": {key: (round(value, 6) if isinstance(value, float) else value) for key, value in options.items()},
                              "correlation": summary(result), "fit": {key: value for key, value in fit_bao(result["xi"], result["covariance"], result["regions"], template_r, template_xi).items() if key != "profile"}}
            print(name, "alpha", failures[name]["fit"]["alpha"], "significance", failures[name]["fit"]["peakSignificance"])
        output["failureCases"] = failures
    Path(arguments.output).write_text(json.dumps(output, indent=2) + "\n", encoding="ascii")


if __name__ == "__main__":
    main()
