options-pricer/backend/src/lib/datafetch.ts

import YahooFinance from "yahoo-finance2";
import type { OptionsResult, CallOrPut } from "yahoo-finance2/modules/options";
import { bsPrice, bsGreeks, impliedVol } from "./blackscholes.js";
import type { OptionQuote, ChainSnapshot } from "./analytics.js";
import { computeSkewMetrics } from "./analytics.js";
import { saveSnapshot, getAverageAtmIv } from "../db/snapshots.js";
import {
  fmpEnabled,
  fmpExpirations,
  fmpOptionsChain,
  fmpQuote,
} from "./fmp.js";

/** Spike threshold — current ATM IV must be at least this much above the baseline. */
const SPIKE_JUMP_PCT = 0.30;       // +30%
const SPIKE_BIG_MOVE_PCT = 3;      // |today Δ| ≥ 3%

const yf = new YahooFinance({ suppressNotices: ["yahooSurvey"] });

const RISK_FREE_RATE = 0.05;

/**
 * Compute 30-day annualized realized volatility from daily closing prices.
 * Used as the ATM IV baseline when options markets are closed / bid-ask are stale.
 */
async function fetchHistoricalVol(symbol: string): Promise<number> {
  try {
    const end = new Date();
    const start = new Date();
    start.setDate(start.getDate() - 45); // fetch 45 days to ensure 30 trading days

    const rows = await yf.historical(symbol, {
      period1: start,
      period2: end,
      interval: "1d",
    });

    const closes = rows
      .map((r) => r.adjClose ?? r.close)
      .filter((v): v is number => v != null && v > 0);

    if (closes.length < 5) return 0;

    const logReturns: number[] = [];
    for (let i = 1; i < closes.length; i++) {
      logReturns.push(Math.log(closes[i] / closes[i - 1]));
    }

    const mean = logReturns.reduce((a, b) => a + b, 0) / logReturns.length;
    const variance =
      logReturns.reduce((sum, r) => sum + (r - mean) ** 2, 0) /
      (logReturns.length - 1);

    return Math.sqrt(variance * 252); // annualize
  } catch {
    return 0;
  }
}

function timeToExpiry(expiryDateStr: string): number {
  const daysRemaining = (new Date(expiryDateStr).getTime() - Date.now()) / (1000 * 60 * 60 * 24);
  return Math.max(daysRemaining, 0) / 365;
}

function toExpiryString(val: Date | number | string): string {
  if (typeof val === "string") return val;
  const d = val instanceof Date ? val : new Date((val as number) * 1000);
  return d.toISOString().split("T")[0];
}

function enrichOption(
  raw: CallOrPut,
  type: "call" | "put",
  expiry: string,
  spot: number
): OptionQuote | null {
  const strike = raw.strike ?? 0;
  const bid = raw.bid ?? 0;
  const ask = raw.ask ?? 0;
  const lastPrice = parseFloat(String(raw.lastPrice ?? 0));
  const volume = raw.volume ?? 0;
  const openInterest = raw.openInterest ?? 0;

  if (strike <= 0) return null;

  // During market hours: use mid/ask. After hours: fall back to lastPrice (last traded price).
  const marketPrice =
    bid > 0 && ask > 0 ? (bid + ask) / 2 :
    ask > 0 ? ask :
    lastPrice;

  if (marketPrice <= 0) return null;

  const midPrice = ask > 0 ? (bid > 0 ? (bid + ask) / 2 : ask) : lastPrice;
  const T = timeToExpiry(expiry);
  const r = RISK_FREE_RATE;

  let iv: number;
  if (marketPrice > 0 && T > 0 && spot > 0) {
    const calculatedIV = impliedVol(spot, strike, T, r, marketPrice, type) ?? null;
    if (calculatedIV !== null && calculatedIV > 0.01 && calculatedIV < 5 && isFinite(calculatedIV)) {
      iv = calculatedIV;
    } else {
      // Newton-Raphson failed — try Yahoo's per-option IV as fallback, but only if plausible
      const yahooIV = raw.impliedVolatility;
      if (yahooIV > 0.01 && yahooIV < 5 && isFinite(yahooIV)) {
        iv = yahooIV;
      } else {
        return null;
      }
    }
  } else {
    return null;
  }

  const theoreticalPrice = T > 0 && spot > 0 ? bsPrice(spot, strike, T, r, iv, type) : 0;
  const greeks =
    T > 0 && spot > 0
      ? bsGreeks(spot, strike, T, r, iv, type)
      : { delta: 0, gamma: 0, theta: 0, vega: 0, rho: 0 };

  return {
    strike,
    expiry,
    type,
    bid,
    ask,
    iv,
    delta: greeks.delta,
    gamma: greeks.gamma,
    theta: greeks.theta,
    vega: greeks.vega,
    volume,
    openInterest,
    bsPrice: theoreticalPrice,
    midPrice,
  };
}

// ---------------------------------------------------------------------------
// FMP path — converts FmpOption[] into ChainSnapshot
// ---------------------------------------------------------------------------

async function fetchViaFmp(symbol: string, expiry: string): Promise<ChainSnapshot | null> {
  try {
    const [fmpOptions, quote] = await Promise.all([
      fmpOptionsChain(symbol, expiry),
      fmpQuote(symbol),
    ]);

    if (fmpOptions.length === 0) return null;

    const spot = quote?.price ?? 0;
    const T = timeToExpiry(expiry);
    const r = RISK_FREE_RATE;

    const chain: OptionQuote[] = fmpOptions.map((o) => {
      const theoreticalPrice =
        T > 0 && spot > 0
          ? bsPrice(spot, o.strike, T, r, o.impliedVolatility, o.type)
          : 0;

      return {
        strike: o.strike,
        expiry,
        type: o.type,
        bid: o.bid,
        ask: o.ask,
        iv: o.impliedVolatility,
        delta: o.delta,
        gamma: o.gamma,
        theta: o.theta,
        vega: o.vega,
        volume: o.volume,
        openInterest: o.openInterest,
        bsPrice: theoreticalPrice,
        midPrice: o.mid,
      };
    });

    const spotIv = quote?.impliedVolatility ?? 0;

    return {
      symbol,
      expiry,
      spot,
      spotIv,
      timestamp: new Date().toISOString(),
      chain,
    };
  } catch (err) {
    console.warn(`[datafetch] FMP failed for ${symbol} ${expiry}:`, (err as Error).message);
    return null;
  }
}

// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------

export async function fetchExpirations(symbol: string): Promise<string[]> {
  if (fmpEnabled()) {
    try {
      const dates = await fmpExpirations(symbol);
      if (dates.length > 0) {
        console.log(`[datafetch] FMP expirations for ${symbol}: ${dates.length} dates`);
        return dates;
      }
    } catch (err) {
      console.warn(`[datafetch] FMP expirations failed, falling back to Yahoo:`, (err as Error).message);
    }
  }

  const result: OptionsResult = await yf.options(symbol);
  const dates = result.expirationDates ?? [];
  return dates.map((d) => toExpiryString(d)).sort();
}

export async function fetchOptionsChain(
  symbol: string,
  expiry?: string
): Promise<ChainSnapshot[]> {
  // --- Determine which expiries to fetch ---
  let expiriesToFetch: string[];

  if (expiry) {
    expiriesToFetch = [expiry];
  } else {
    const all = await fetchExpirations(symbol);
    const now = Date.now();
    expiriesToFetch = all.filter((e) => new Date(e).getTime() > now).slice(0, 3);
  }

  if (expiriesToFetch.length === 0) {
    throw new Error(`No valid expiration dates found for ${symbol}`);
  }

  // --- FMP path ---
  if (fmpEnabled()) {
    const snapshots: ChainSnapshot[] = [];
    for (const exp of expiriesToFetch) {
      const snap = await fetchViaFmp(symbol, exp);
      if (snap) {
        snapshots.push(snap);
      } else {
        console.warn(`[datafetch] FMP returned no data for ${symbol} ${exp}`);
      }
    }
    if (snapshots.length > 0) {
      console.log(`[datafetch] FMP: fetched ${snapshots.length} snapshots for ${symbol}`);
      return snapshots;
    }
    console.warn(`[datafetch] FMP returned nothing for ${symbol}, falling back to Yahoo`);
  }

  // --- Yahoo Finance fallback ---
  const historicalVol = await fetchHistoricalVol(symbol);
  const snapshots: ChainSnapshot[] = [];

  for (const expiryDate of expiriesToFetch) {
    try {
      const result: OptionsResult = await yf.options(symbol, {
        date: new Date(expiryDate),
      });

      const spot: number = result.quote?.regularMarketPrice ?? 0;

      const optionExpiry = result.options?.[0];
      const rawCalls: CallOrPut[] = optionExpiry?.calls ?? [];
      const rawPuts: CallOrPut[] = optionExpiry?.puts ?? [];

      const chain: OptionQuote[] = [
        ...rawCalls.map((r) => enrichOption(r, "call", expiryDate, spot)),
        ...rawPuts.map((r) => enrichOption(r, "put", expiryDate, spot)),
      ]
        .filter((q): q is OptionQuote => q !== null)
        .sort((a, b) => a.strike - b.strike);

      snapshots.push({ symbol, expiry: expiryDate, spot, spotIv: historicalVol, timestamp: new Date().toISOString(), chain });
    } catch (err) {
      console.error(`[datafetch] Failed for ${symbol} expiry ${expiryDate}: ${err instanceof Error ? err.message : err}`);
    }
  }

  if (snapshots.length === 0) {
    throw new Error(`Failed to fetch any options data for ${symbol}`);
  }

  return snapshots;
}

// ---------------------------------------------------------------------------
// Lightweight per-symbol scan — for the IV-spike scanner
// ---------------------------------------------------------------------------

export type ScanResult = {
  symbol: string;
  spot: number;
  change: number;
  changePct: number;
  atmIv: number;
  hv30: number;
  baselineIv: number;     // recent-30d avg ATM IV from snapshots, else HV30
  baselineSrc: "history" | "hv30" | "none";
  baselineN: number;      // # of snapshot rows feeding the baseline (0 = HV30 fallback)
  ivJumpPct: number;      // (atmIv - baselineIv) / baselineIv
  ivHv: number;           // atmIv / hv30 (informational; pre-baseline metric)
  spike: boolean;         // ivJumpPct ≥ 30%
  bigMove: boolean;       // |today Δ| ≥ 3%
  expiry: string;
  error?: string;
};

/** Fetch just the front-expiry chain + HV30 for a single symbol — cheap enough to parallelize. */
export async function scanSymbol(symbol: string): Promise<ScanResult> {
  // 1) list of expirations
  const exps: string[] = await fetchExpirations(symbol);
  const now = Date.now();
  const upcoming = exps.filter((e) => new Date(e).getTime() > now);
  if (upcoming.length === 0) throw new Error("No upcoming expirations");
  const expiry = upcoming[0];

  // 2) the front-expiry chain + spot
  let spot = 0, prevClose = 0;
  let chain: OptionQuote[] = [];

  if (fmpEnabled()) {
    try {
      const [fmpChain, q] = await Promise.all([fmpOptionsChain(symbol, expiry), fmpQuote(symbol)]);
      if (q?.price) spot = q.price;
      const T = Math.max(0, (new Date(expiry).getTime() - Date.now()) / (1000 * 60 * 60 * 24)) / 365;
      chain = fmpChain.map((o) => ({
        strike: o.strike, expiry, type: o.type,
        bid: o.bid, ask: o.ask,
        iv: o.impliedVolatility, delta: o.delta, gamma: o.gamma, theta: o.theta, vega: o.vega,
        volume: o.volume, openInterest: o.openInterest,
        bsPrice: spot > 0 ? bsPrice(spot, o.strike, T, RISK_FREE_RATE, o.impliedVolatility, o.type) : 0,
        midPrice: o.mid,
      }));
    } catch { /* fall through to yahoo */ }
  }

  if (chain.length === 0) {
    const result: OptionsResult = await yf.options(symbol, { date: new Date(expiry) });
    spot = result.quote?.regularMarketPrice ?? 0;
    prevClose = result.quote?.regularMarketPreviousClose ?? spot;
    const optExp = result.options?.[0];
    const rawCalls = optExp?.calls ?? [];
    const rawPuts = optExp?.puts ?? [];
    chain = [
      ...rawCalls.map((r) => enrichOption(r, "call", expiry, spot)),
      ...rawPuts.map((r) => enrichOption(r, "put", expiry, spot)),
    ].filter((q): q is OptionQuote => q !== null);
  } else if (!prevClose) {
    prevClose = spot;
  }

  const change = spot - prevClose;
  const changePct = prevClose > 0 ? (change / prevClose) * 100 : 0;

  // ATM IV — closest-strike option's IV (pick the lower-IV side to avoid wing inflation)
  let atmIv = 0;
  if (chain.length > 0 && spot > 0) {
    const closest = chain.reduce((b, q) =>
      Math.abs(q.strike - spot) < Math.abs(b.strike - spot) ? q : b
    );
    const sameStrike = chain.filter((q) => q.strike === closest.strike && q.iv > 0);
    atmIv = sameStrike.length > 1
      ? sameStrike.reduce((s, q) => s + q.iv, 0) / sameStrike.length
      : (closest.iv || 0);
  }

  const hv30 = await fetchHistoricalVol(symbol);
  const ivHv = hv30 > 0 ? atmIv / hv30 : 0;

  // Save this scan as a snapshot so the baseline grows over time.
  // Use the snapshot's chain so per-strike IV is recorded too.
  try {
    if (atmIv > 0 && spot > 0) {
      const snap = {
        symbol, expiry, spot, spotIv: hv30,
        timestamp: new Date().toISOString(),
        chain,
      };
      saveSnapshot(symbol, expiry, spot, computeSkewMetrics(snap), chain);
    }
  } catch { /* non-fatal */ }

  // Baseline IV: prefer 30-day avg from snapshots; fall back to HV30.
  const hist = getAverageAtmIv(symbol, 30);
  let baselineIv = 0, baselineSrc: ScanResult["baselineSrc"] = "none", baselineN = 0;
  if (hist && hist.avg > 0) { baselineIv = hist.avg; baselineSrc = "history"; baselineN = hist.n; }
  else if (hv30 > 0)        { baselineIv = hv30;     baselineSrc = "hv30";    baselineN = 0; }

  const ivJumpPct = baselineIv > 0 ? (atmIv - baselineIv) / baselineIv : 0;
  const spike = ivJumpPct >= SPIKE_JUMP_PCT;
  const bigMove = Math.abs(changePct) >= SPIKE_BIG_MOVE_PCT;

  return {
    symbol, spot, change, changePct, atmIv, hv30,
    baselineIv, baselineSrc, baselineN,
    ivJumpPct, ivHv, spike, bigMove, expiry,
  };
}