#!/usr/bin/env python3
"""FlashMemory wafer-level statistics: independent wafers, nested sites. Training data only."""
import csv
import json
import math
import statistics
from collections import defaultdict
from pathlib import Path

file = Path(__file__).with_name("wafer-thickness-synthetic.csv")
rows = list(csv.DictReader(file.open(encoding="utf-8", newline="")))
assert len(rows) == 36
sites = defaultdict(list)
for r in rows:
    if r["period"] not in ("before","after") or r["site"] not in ("1","2","3"):
        raise ValueError("Invalid synthetic category")
    key = (r["period"],r["wafer_id"])
    sites[key].append(float(r["thickness_nm"]))
assert len(sites) == 12
assert all(len(vals) == 3 for vals in sites.values())
wafer_means = defaultdict(list)
for (period, wafer), values in sites.items():
    wafer_means[period].append(statistics.mean(values))

# Two-sided 95% Student-t critical value at df=5, from standard t tables.
# Applies only to this pedagogical n=6 normal/independent-wafer example.
T_975_DF5 = 2.5705818366
result = {"dataset":"SYNTHETIC_TRAINING_ONLY","raw_site_rows":len(rows),
          "independent_wafer_units":len(sites),"periods":{}}
for period in ("before","after"):
    values=wafer_means[period]
    n=len(values)
    mean=statistics.mean(values)
    sample_sd=statistics.stdev(values)
    se=sample_sd/math.sqrt(n)
    margin=T_975_DF5*se
    all_sites=[v for (p,_),arr in sites.items() if p==period for v in arr]
    result["periods"][period]={
        "wafer_n":n, "site_rows":len(all_sites),
        "wafer_level_mean_nm":round(mean,4),
        "wafer_level_sample_sd_nm":round(sample_sd,4),
        "wafer_level_se_nm":round(se,4),
        "mean_95pct_t_ci_nm":[round(mean-margin,4),round(mean+margin,4)],
        "incorrect_if_sites_treated_as_independent_se_nm":round(statistics.stdev(all_sites)/math.sqrt(len(all_sites)),4)
    }
result["descriptive_mean_difference_after_minus_before_nm"]=round(
 result["periods"]["after"]["wafer_level_mean_nm"]-
 result["periods"]["before"]["wafer_level_mean_nm"],4)
# Unlike comparing two separate mean CIs, calculate the independent-group
# difference CI directly. Welch-Satterthwaite df is 10 for this symmetric synthetic
# example (n=6 per period, equal observed sample variances).
n_before=len(wafer_means["before"])
n_after=len(wafer_means["after"])
var_before=statistics.variance(wafer_means["before"])
var_after=statistics.variance(wafer_means["after"])
part_before=var_before/n_before
part_after=var_after/n_after
difference_se=math.sqrt(part_before+part_after)
df=(part_before+part_after)**2/(part_before**2/(n_before-1)+part_after**2/(n_after-1))
assert abs(df-10)<1e-10
T_975_DF10=2.2281388519649385  # Standard Student-t table, df=10 (this example only).
diff=result["descriptive_mean_difference_after_minus_before_nm"]
margin=T_975_DF10*difference_se
result["independent_group_mean_difference_95pct_welch_ci_nm"]={
    "estimate_nm":diff,
    "standard_error_nm":round(difference_se,4),
    "approx_df":round(df,4),
    "ci_nm":[round(diff-margin,4),round(diff+margin,4)]
}
result["interpretation"]="Each group CI estimates its own mean, while the separate Welch CI estimates after-before. This assumes independent representative wafers and approximately valid t inference; no tolerance/SPC limits or process causality follow."
assert result["independent_group_mean_difference_95pct_welch_ci_nm"]["ci_nm"]==[-0.8193,2.8193]
assert result["periods"]["before"]["wafer_level_mean_nm"] == 50
assert result["periods"]["after"]["wafer_level_mean_nm"] == 51
assert all(abs(result["periods"][p]["wafer_level_sample_sd_nm"]-1.4142)<.0001 for p in ("before","after"))
assert result["descriptive_mean_difference_after_minus_before_nm"] == 1
print(json.dumps(result,ensure_ascii=False,indent=2))