You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
93 lines
3.5 KiB
Python
93 lines
3.5 KiB
Python
import json, statistics, re
|
|
|
|
files = {
|
|
'qwen3:4b': r'root_cause/user_conv_log/results_qwen3-4b_test.jsonl',
|
|
'qwen3:8b': r'root_cause/user_conv_log/results_qwen3-8b_test.jsonl',
|
|
'gemma4:e2b': r'root_cause/user_conv_log/results_gemma4-e2b_test.jsonl',
|
|
'gemma4:e4b': r'root_cause/user_conv_log/results_gemma4-e4b_test.jsonl',
|
|
}
|
|
|
|
SOP_KEYWORDS = [
|
|
('Gejala', re.compile(r'gejala|symptom', re.I)),
|
|
('Verifikasi', re.compile(r'verifikasi|verif.*lapangan|field.*verif', re.I)),
|
|
('Akar Masalah', re.compile(r'akar\s*masalah|root\s*cause', re.I)),
|
|
('Dampak', re.compile(r'dampak|system\s*impact', re.I)),
|
|
('FDT', re.compile(r'rekomendasi|FDT|recommended\s*action', re.I)),
|
|
]
|
|
|
|
DOMAIN_RE = re.compile(r'\b(HHWL|BFP|DCV|DCA|HPH|RCFA|ECT|DCS|PLC|TTD|PAUT|NDT|PID|FDT|NWL|SSC|MOV)\b')
|
|
TAG_RE = re.compile(r'\b3[A-Z]{2}-H\d{3}\b')
|
|
|
|
results = {}
|
|
for model, path in files.items():
|
|
rows = []
|
|
try:
|
|
with open(path, encoding='utf-8') as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
rows.append(json.loads(line))
|
|
except Exception as e:
|
|
print('ERROR reading {}: {}'.format(path, e))
|
|
continue
|
|
|
|
speeds, times, tokens, sop_scores, term_counts = [], [], [], [], []
|
|
tag_mismatches = 0
|
|
sop_bd = {k: 0 for k, _ in SOP_KEYWORDS}
|
|
|
|
for row in rows:
|
|
s = row.get('stats', {})
|
|
resp = row.get('generated_response', '')
|
|
tag = row.get('tag', '')
|
|
|
|
speeds.append(s.get('tokens_per_sec', 0))
|
|
times.append(s.get('total_time', 0))
|
|
tokens.append(s.get('eval_count', 0))
|
|
|
|
score = 0
|
|
for kname, kr in SOP_KEYWORDS:
|
|
if kr.search(resp):
|
|
score += 1
|
|
sop_bd[kname] += 1
|
|
sop_scores.append(score)
|
|
|
|
term_counts.append(len(DOMAIN_RE.findall(resp)))
|
|
|
|
found_tags = set(TAG_RE.findall(resp))
|
|
if found_tags and not all(t == tag for t in found_tags):
|
|
tag_mismatches += 1
|
|
|
|
n = len(rows)
|
|
results[model] = {
|
|
'n': n,
|
|
'avg_speed': statistics.mean(speeds),
|
|
'min_speed': min(speeds),
|
|
'max_speed': max(speeds),
|
|
'avg_time': statistics.mean(times),
|
|
'total_tokens': sum(tokens),
|
|
'avg_tokens': statistics.mean(tokens),
|
|
'median_tokens': statistics.median(tokens),
|
|
'sop_perfect': sum(1 for s in sop_scores if s == 5),
|
|
'sop_avg': statistics.mean(sop_scores),
|
|
'sop_bd': sop_bd,
|
|
'avg_domain_terms': statistics.mean(term_counts),
|
|
'tag_mismatches': tag_mismatches,
|
|
'risk_dist': {
|
|
'High': sum(1 for r in rows if r.get('risk') == 'High'),
|
|
'Medium': sum(1 for r in rows if r.get('risk') == 'Medium'),
|
|
'Low': sum(1 for r in rows if r.get('risk') == 'Low'),
|
|
}
|
|
}
|
|
|
|
for model, d in results.items():
|
|
print('=== {} (n={}) ==='.format(model, d['n']))
|
|
print(' Speed: avg={:.1f} t/s, min={:.1f}, max={:.1f}'.format(d['avg_speed'], d['min_speed'], d['max_speed']))
|
|
print(' Time: avg={:.1f}s | Tokens: avg={:.0f}, median={:.0f}, total={}'.format(
|
|
d['avg_time'], d['avg_tokens'], d['median_tokens'], d['total_tokens']))
|
|
print(' SOP Perfect (5/5): {}/{} | SOP avg: {:.2f}/5'.format(d['sop_perfect'], d['n'], d['sop_avg']))
|
|
print(' SOP Breakdown: {}'.format(d['sop_bd']))
|
|
print(' Avg Domain Terms: {:.1f}'.format(d['avg_domain_terms']))
|
|
print(' Tag Mismatches: {}'.format(d['tag_mismatches']))
|
|
print(' Risk Dist: {}'.format(d['risk_dist']))
|
|
print()
|