import json, statistics, re

files = {
    'qwen3:4b':   r'root_cause/user_conv_log/results_qwen3-4b_test.jsonl',
    'qwen3:8b':   r'root_cause/user_conv_log/results_qwen3-8b_test.jsonl',
    'gemma4:e2b': r'root_cause/user_conv_log/results_gemma4-e2b_test.jsonl',
    'gemma4:e4b': r'root_cause/user_conv_log/results_gemma4-e4b_test.jsonl',
}

SOP_KEYWORDS = [
    ('Gejala',       re.compile(r'gejala|symptom', re.I)),
    ('Verifikasi',   re.compile(r'verifikasi|verif.*lapangan|field.*verif', re.I)),
    ('Akar Masalah', re.compile(r'akar\s*masalah|root\s*cause', re.I)),
    ('Dampak',       re.compile(r'dampak|system\s*impact', re.I)),
    ('FDT',          re.compile(r'rekomendasi|FDT|recommended\s*action', re.I)),
]

DOMAIN_RE = re.compile(r'\b(HHWL|BFP|DCV|DCA|HPH|RCFA|ECT|DCS|PLC|TTD|PAUT|NDT|PID|FDT|NWL|SSC|MOV)\b')
TAG_RE = re.compile(r'\b3[A-Z]{2}-H\d{3}\b')

results = {}
for model, path in files.items():
    rows = []
    try:
        with open(path, encoding='utf-8') as f:
            for line in f:
                line = line.strip()
                if line:
                    rows.append(json.loads(line))
    except Exception as e:
        print('ERROR reading {}: {}'.format(path, e))
        continue

    speeds, times, tokens, sop_scores, term_counts = [], [], [], [], []
    tag_mismatches = 0
    sop_bd = {k: 0 for k, _ in SOP_KEYWORDS}

    for row in rows:
        s = row.get('stats', {})
        resp = row.get('generated_response', '')
        tag  = row.get('tag', '')

        speeds.append(s.get('tokens_per_sec', 0))
        times.append(s.get('total_time', 0))
        tokens.append(s.get('eval_count', 0))

        score = 0
        for kname, kr in SOP_KEYWORDS:
            if kr.search(resp):
                score += 1
                sop_bd[kname] += 1
        sop_scores.append(score)

        term_counts.append(len(DOMAIN_RE.findall(resp)))

        found_tags = set(TAG_RE.findall(resp))
        if found_tags and not all(t == tag for t in found_tags):
            tag_mismatches += 1

    n = len(rows)
    results[model] = {
        'n': n,
        'avg_speed': statistics.mean(speeds),
        'min_speed': min(speeds),
        'max_speed': max(speeds),
        'avg_time':  statistics.mean(times),
        'total_tokens': sum(tokens),
        'avg_tokens': statistics.mean(tokens),
        'median_tokens': statistics.median(tokens),
        'sop_perfect': sum(1 for s in sop_scores if s == 5),
        'sop_avg': statistics.mean(sop_scores),
        'sop_bd': sop_bd,
        'avg_domain_terms': statistics.mean(term_counts),
        'tag_mismatches': tag_mismatches,
        'risk_dist': {
            'High':   sum(1 for r in rows if r.get('risk') == 'High'),
            'Medium': sum(1 for r in rows if r.get('risk') == 'Medium'),
            'Low':    sum(1 for r in rows if r.get('risk') == 'Low'),
        }
    }

for model, d in results.items():
    print('=== {} (n={}) ==='.format(model, d['n']))
    print('  Speed: avg={:.1f} t/s, min={:.1f}, max={:.1f}'.format(d['avg_speed'], d['min_speed'], d['max_speed']))
    print('  Time:  avg={:.1f}s | Tokens: avg={:.0f}, median={:.0f}, total={}'.format(
          d['avg_time'], d['avg_tokens'], d['median_tokens'], d['total_tokens']))
    print('  SOP Perfect (5/5): {}/{} | SOP avg: {:.2f}/5'.format(d['sop_perfect'], d['n'], d['sop_avg']))
    print('  SOP Breakdown: {}'.format(d['sop_bd']))
    print('  Avg Domain Terms: {:.1f}'.format(d['avg_domain_terms']))
    print('  Tag Mismatches: {}'.format(d['tag_mismatches']))
    print('  Risk Dist: {}'.format(d['risk_dist']))
    print()