"""
QQQM vs QQQ: fee, tracking, liquidity and what switching costs (FinLab research, 2026-11).

QQQM listed on 2020-10-13. Both funds track the Nasdaq-100.

Test (M1): buy QQQM instead of QQQ and hold. Nine settings, one for each pair of
  start 2020-10-30 / 2021-04-30 / 2021-10-29 and end 2025-10-31 / 2026-04-30 / 2026-10-30.
The original is the longest window, 2020-10-30 to 2026-10-30. QQQM passes only if
  1. at least 7 of the 9 windows have a higher daily Sharpe (no risk-free rate) than QQQ,
  2. it beats QQQ in both halves of the original window,
  3. a QQQ holder who pays 0.1% per side to switch to QQQM (0.2% in all) still beats QQQ:
     buy QQQ at the first close after the start, switch at the close after that (2020-11-03).
     The switch is repeated on the next six closes to show how much the fill date matters.

Run:  python research.py      writes metrics.json and data.csv next to this file
"""
import json
from datetime import date
from pathlib import Path

import finlab
import numpy as np
import pandas as pd
from finlab import data
from finlab.backtest import sim

STARTS = ("2020-10-30", "2021-04-30", "2021-10-29")
ENDS = ("2025-10-31", "2026-04-30", "2026-10-30")
START, END = STARTS[0], ENDS[-1]
LISTED = "2020-10-13"
CONVERSION = "2025-12-22"      # QQQ became an open-end ETF and its fee fell from 0.20% to 0.18%
COST_PER_SIDE = 0.001
FEE_GAP_NOW = 0.0003           # 0.18% - 0.15%
WINDOWS = {"1y": 252, "3y": 756, "5y": 1260}
TRADE_SIZES = (10_000, 100_000, 1_000_000, 10_000_000)
OUT = Path(__file__).resolve().parent
FUNDS = ["QQQ", "QQQM"]

data.truncate_end = END
data.set_market("us")


def table(key):
    frame = data.get(f"us_fund_price:{key}")[FUNDS]
    return pd.DataFrame(frame.loc[frame["QQQ"].notna()].loc[LISTED:])


PRICES = table("adj_close")
DAYS = PRICES.index


def daily_sharpe(returns):
    return returns.mean() / returns.std() * 252 ** 0.5


def cagr(curve):
    years = (curve.index[-1] - curve.index[0]).days / 365.25
    return (curve.iloc[-1] / curve.iloc[0]) ** (1 / years) - 1


def compare(start, end):
    window = PRICES.loc[start:end]
    returns = window.pct_change().dropna()
    sharpe = {t: daily_sharpe(returns[t]) for t in FUNDS}
    growth = {t: cagr(window[t]) for t in FUNDS}
    return {
        "start": str(window.index[0].date()), "end": str(window.index[-1].date()),
        "qqqm_sharpe": round(sharpe["QQQM"], 6), "qqq_sharpe": round(sharpe["QQQ"], 6),
        "qqqm_cagr": round(growth["QQQM"], 6), "qqq_cagr": round(growth["QQQ"], 6),
        "cagr_gap": round(growth["QQQM"] - growth["QQQ"], 6),
        "qqqm_total": round(window["QQQM"].iloc[-1] / window["QQQM"].iloc[0] - 1, 6),
        "qqq_total": round(window["QQQ"].iloc[-1] / window["QQQ"].iloc[0] - 1, 6),
        "qqqm_mdd": round((window["QQQM"] / window["QQQM"].cummax() - 1).min(), 6),
        "qqq_mdd": round((window["QQQ"] / window["QQQ"].cummax() - 1).min(), 6),
        "beats_qqq": bool(sharpe["QQQM"] > sharpe["QQQ"]),
    }


def run(position):
    curve = sim(position, trade_at_price="close", fee_ratio=COST_PER_SIDE, market="us_fund", upload=False).creturn
    moved = (curve != curve.iloc[0]).to_numpy().nonzero()[0]
    return curve.iloc[max(moved[0] - 1, 0):]


def switching(fill):
    """Buy QQQ at the start, switch to QQQM at the close of days[fill], against staying in QQQ."""
    days = DAYS[DAYS >= START]
    stay = pd.DataFrame({"QQQ": 1.0}, index=days)
    move = pd.DataFrame({"QQQ": 1.0, "QQQM": 0.0}, index=days)
    move.iloc[fill - 1:] = [0.0, 1.0]          # a signal fills at the next close
    stay_curve, move_curve = run(stay), run(move)
    common = stay_curve.index.intersection(move_curve.index)
    stay_curve, move_curve = stay_curve.loc[common], move_curve.loc[common]
    sharpe = {k: daily_sharpe(c.pct_change().dropna()) for k, c in (("stay", stay_curve), ("move", move_curve))}
    stay_multiple, move_multiple = stay_curve.iloc[-1] / stay_curve.iloc[0], move_curve.iloc[-1] / move_curve.iloc[0]
    return {"filled": str(days[fill].date()), "stay_sharpe": round(sharpe["stay"], 6), "move_sharpe": round(sharpe["move"], 6),
            "stay_multiple": round(stay_multiple, 6), "move_multiple": round(move_multiple, 6),
            "lead": round(move_multiple / stay_multiple - 1, 6), "beats_qqq": bool(sharpe["move"] > sharpe["stay"])}


def evaluate():
    grid = [compare(s, e) for s in STARTS for e in ENDS]
    base = grid[2]
    days = PRICES.loc[START:END].index
    middle = str(days[len(days) // 2].date())
    halves = [compare(START, middle), compare(middle, END)]
    switch = switching(2)
    wins = sum(g["beats_qqq"] for g in grid)
    checks = {"grid": wins >= 7, "halves": all(h["beats_qqq"] for h in halves), "cost": switch["beats_qqq"]}
    return {"grid": grid, "grid_wins": wins, "original": base, "halves": halves, "switching": switch,
            "switching_later_fills": [switching(fill) for fill in range(3, 9)],
            "checks": checks, "passes": all(checks.values())}


def tracking():
    window = PRICES.loc[START:END]
    returns = window.pct_change().dropna()
    gap = returns["QQQM"] - returns["QQQ"]
    curve = window / window.iloc[0]
    ends = window.resample("YE").last()
    yearly = ends / ends.shift(1).fillna(window.iloc[0]) - 1
    monthly = window.resample("ME").last().pct_change().dropna()
    lead = curve["QQQM"] / curve["QQQ"]
    before, after = compare(START, CONVERSION), compare(CONVERSION, END)
    return {
        "before_conversion": before, "after_conversion": after,
        "yearly": {str(d.year): {"qqq": round(row["QQQ"], 6), "qqqm": round(row["QQQM"], 6),
                                 "gap": round(row["QQQM"] - row["QQQ"], 6)} for d, row in yearly.iterrows()},
        "share_of_months_qqqm_ahead": round(((monthly["QQQM"] - monthly["QQQ"]) > 0).mean(), 4),
        "share_of_days_qqqm_ahead": round((gap > 0).mean(), 4),
        "correlation": round(returns.corr().iloc[0, 1], 6),
        "tracking_error": round(gap.std() * 252 ** 0.5, 6),
        "largest_daily_gap": round(gap.abs().max(), 6), "largest_daily_gap_date": str(gap.abs().idxmax().date()),
        "median_abs_daily_gap": round(gap.abs().median(), 6),
        "end_lead": round(lead.iloc[-1] - 1, 6),
        "lead_since_2021_04_30": round(lead.iloc[-1] / lead.loc["2021-04-30"] - 1, 6),
    }


def holding_windows():
    prices = PRICES.loc[START:END]
    out = {}
    for label, days in WINDOWS.items():
        returns = (prices / prices.shift(days) - 1).dropna()
        gap = (returns["QQQM"] - returns["QQQ"]).set_axis(prices.index[:-days])
        worst = gap.nsmallest(3)
        out[label] = {"windows": len(gap), "qqqm_beats": round((gap > 0).mean(), 4),
                      "median_gap": round(gap.median(), 6), "worst_gap": round(gap.min(), 6),
                      "worst_starts": {str(d.date()): round(v, 6) for d, v in worst.items()}}
    return out


def liquidity():
    close, volume = table("close").loc[START:END], table("volume").loc[START:END]
    high, low = table("high").loc[START:END], table("low").loc[START:END]
    value = close * volume
    latest = value.iloc[-252:]
    returns = PRICES.loc[START:END].pct_change().iloc[-252:]
    median = latest.median()
    yearly = value.groupby(value.index.year).median()
    return {
        "yearly_median_dollar_volume": {str(y): {k: round(v) for k, v in row.items()} for y, row in yearly.iterrows()},
        "yearly_ratio": {str(y): round(row["QQQ"] / row["QQQM"], 2) for y, row in yearly.iterrows()},
        "last_year": {
            "median_dollar_volume": {k: round(v) for k, v in median.items()},
            "median_shares": {k: round(v) for k, v in volume.iloc[-252:].median().items()},
            "lowest_dollar_volume": {k: round(v) for k, v in latest.min().items()},
            "price_move_bp_per_million": {k: round(v, 5) for k, v in ((returns.abs() / latest).median() * 1e6 * 1e4).items()},
            "median_high_low_range": {k: round(v, 6) for k, v in ((high - low) / close).iloc[-252:].median().items()},
        },
        "price": {k: round(v, 2) for k, v in close.iloc[-1].items()},
        "one_cent_in_bp": {k: round(0.01 / v * 1e4, 3) for k, v in close.iloc[-1].items()},
        "trade_as_share_of_median_day": {str(size): {k: round(size / v, 8) for k, v in median.items()} for size in TRADE_SIZES},
    }


def distributions():
    rows = pd.DataFrame(data.get("us_fund_corporate_actions"))
    rows = rows[rows["stock_id"].isin(FUNDS) & (rows["date"] >= START) & (rows["date"] <= END)].copy()
    rows["lag"] = (pd.to_datetime(rows["payment_date"]) - pd.to_datetime(rows["date"])).dt.days
    rows["era"] = np.where(rows["date"] <= CONVERSION, "before", "after")
    close = table("close").loc[START:END]
    last_year = rows[rows["date"] > pd.Timestamp(END) - pd.DateOffset(years=1)]
    return {
        "days_ex_date_to_payment": {f"{k[0]}_{k[1]}": round(v, 1) for k, v in rows.groupby(["stock_id", "era"])["lag"].median().items()},
        "trailing_yield": {t: round(last_year.loc[last_year["stock_id"] == t, "dividend"].sum() / close[t].iloc[-1], 6) for t in FUNDS},
        "same_ex_dates": set(rows.loc[rows["stock_id"] == "QQQ", "date"].dt.date.astype(str))
                         == set(rows.loc[rows["stock_id"] == "QQQM", "date"].dt.date.astype(str)),
    }


def break_even():
    switch_cost = COST_PER_SIDE * 2
    return {"switch_cost": switch_cost, "fee_gap_now": FEE_GAP_NOW, "years_at_fee_gap": round(switch_cost / FEE_GAP_NOW, 1)}


def main():
    out = {"window": {"listed": LISTED, "start": START, "end": END, "conversion": CONVERSION},
           "run_date": str(date.today()), "finlab_version": finlab.__version__, "pandas_version": pd.__version__,
           "test": evaluate(), "tracking": tracking(), "windows": holding_windows(),
           "liquidity": liquidity(), "distributions": distributions()}
    out["break_even"] = break_even()
    print("QQQM", "PASS" if out["test"]["passes"] else "FAIL", out["test"]["grid_wins"], out["test"]["checks"])
    window = PRICES.loc[START:END]
    curve = window / window.iloc[0]
    frame = curve.rename(columns={"QQQ": "QQQ total return", "QQQM": "QQQM total return"})
    frame["QQQM lead over QQQ"] = curve["QQQM"] / curve["QQQ"] - 1
    value = (table("close") * table("volume")).loc[START:END]
    frame["QQQ dollar volume"] = value["QQQ"].round()
    frame["QQQM dollar volume"] = value["QQQM"].round()
    frame.round(6).to_csv(OUT / "data.csv", index_label="date")
    (OUT / "metrics.json").write_text(json.dumps(out, indent=1, ensure_ascii=False))


if __name__ == "__main__":
    main()
