import json, re, collections, statistics

path_4b = 'benchmark/100_queries/results_qwen3-4b_100.jsonl'
path_8b = 'benchmark/100_queries/results_qwen3-8b_100.jsonl'
path_e4b = 'benchmark/100_queries/results_gemma4-e4b_100.jsonl'

SOP_SECTIONS = [
    (r'gejala', 'Gejala'),
    (r'verifikasi', 'Verifikasi'),
    (r'akar masalah', 'Akar Masalah'),
    (r'dampak', 'Dampak'),
    (r'rekomendasi|fdt', 'FDT'),
]

DOMAIN_TERMS = ['HHWL','BFP','DCV','DCA','HPH','RCFA','ECT','DCS','PLC','TTD','PAUT','NDT','PID','FDT','NWL','SSC','MOV','FMEA']
TAG_PATTERN = re.compile(r'3FW-H0[567]0')

def load(path):
    data = []
    with open(path, encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            if line:
                data.append(json.loads(line))
    return data

data4b = load(path_4b)
data8b = load(path_8b)
data_e4b = load(path_e4b)

print("=== qwen3:4b SOP FAILURES ===")
for i, item in enumerate(data4b):
    resp = item.get('response','') or item.get('generated_response','')
    resp_lower = resp.lower()
    missing = [name for pattern, name in SOP_SECTIONS if not re.search(pattern, resp_lower, re.IGNORECASE)]
    if missing:
        tag = item.get('tag','')
        risk = item.get('risk','')
        print(f"  Case {i+1} tag={tag} risk={risk} missing={missing}")

print()
print("=== RISK DISTRIBUTION ===")
for data, model in [(data4b,'qwen3:4b'), (data8b,'qwen3:8b'), (data_e4b,'gemma4:e4b')]:
    rc = collections.Counter(item.get('risk','') for item in data)
    print(f"{model}: {dict(rc)}")

print()
print("=== TAG INTEGRITY ===")
for data, model in [(data4b,'qwen3:4b'), (data8b,'qwen3:8b'), (data_e4b,'gemma4:e4b')]:
    mismatches = []
    for i, item in enumerate(data):
        expected = item.get('tag','')
        resp = item.get('response','') or item.get('generated_response','')
        found_tags = set(TAG_PATTERN.findall(resp))
        found_tags.discard(expected)
        if found_tags:
            risk = item.get('risk','')
            mismatches.append((i+1, expected, found_tags, risk))
    print(f"{model}: {len(mismatches)} mismatches")
    for m in mismatches:
        print(f"  Case {m[0]}: expected={m[1]} found={m[2]} risk={m[3]}")

print()
print("=== TERMINOLOGY DENSITY ===")
for data, model in [(data4b,'qwen3:4b'), (data8b,'qwen3:8b'), (data_e4b,'gemma4:e4b')]:
    counts = []
    for item in data:
        resp = item.get('response','') or item.get('generated_response','')
        counts.append(sum(1 for t in DOMAIN_TERMS if t in resp))
    print(f"{model}: avg={statistics.mean(counts):.2f} min={min(counts)} max={max(counts)}")