You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
71 lines
2.5 KiB
Python
71 lines
2.5 KiB
Python
import json, re, collections, statistics
|
|
|
|
path_4b = 'benchmark/100_queries/results_qwen3-4b_100.jsonl'
|
|
path_8b = 'benchmark/100_queries/results_qwen3-8b_100.jsonl'
|
|
path_e4b = 'benchmark/100_queries/results_gemma4-e4b_100.jsonl'
|
|
|
|
SOP_SECTIONS = [
|
|
(r'gejala', 'Gejala'),
|
|
(r'verifikasi', 'Verifikasi'),
|
|
(r'akar masalah', 'Akar Masalah'),
|
|
(r'dampak', 'Dampak'),
|
|
(r'rekomendasi|fdt', 'FDT'),
|
|
]
|
|
|
|
DOMAIN_TERMS = ['HHWL','BFP','DCV','DCA','HPH','RCFA','ECT','DCS','PLC','TTD','PAUT','NDT','PID','FDT','NWL','SSC','MOV','FMEA']
|
|
TAG_PATTERN = re.compile(r'3FW-H0[567]0')
|
|
|
|
def load(path):
|
|
data = []
|
|
with open(path, encoding='utf-8') as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
data.append(json.loads(line))
|
|
return data
|
|
|
|
data4b = load(path_4b)
|
|
data8b = load(path_8b)
|
|
data_e4b = load(path_e4b)
|
|
|
|
print("=== qwen3:4b SOP FAILURES ===")
|
|
for i, item in enumerate(data4b):
|
|
resp = item.get('response','') or item.get('generated_response','')
|
|
resp_lower = resp.lower()
|
|
missing = [name for pattern, name in SOP_SECTIONS if not re.search(pattern, resp_lower, re.IGNORECASE)]
|
|
if missing:
|
|
tag = item.get('tag','')
|
|
risk = item.get('risk','')
|
|
print(f" Case {i+1} tag={tag} risk={risk} missing={missing}")
|
|
|
|
print()
|
|
print("=== RISK DISTRIBUTION ===")
|
|
for data, model in [(data4b,'qwen3:4b'), (data8b,'qwen3:8b'), (data_e4b,'gemma4:e4b')]:
|
|
rc = collections.Counter(item.get('risk','') for item in data)
|
|
print(f"{model}: {dict(rc)}")
|
|
|
|
print()
|
|
print("=== TAG INTEGRITY ===")
|
|
for data, model in [(data4b,'qwen3:4b'), (data8b,'qwen3:8b'), (data_e4b,'gemma4:e4b')]:
|
|
mismatches = []
|
|
for i, item in enumerate(data):
|
|
expected = item.get('tag','')
|
|
resp = item.get('response','') or item.get('generated_response','')
|
|
found_tags = set(TAG_PATTERN.findall(resp))
|
|
found_tags.discard(expected)
|
|
if found_tags:
|
|
risk = item.get('risk','')
|
|
mismatches.append((i+1, expected, found_tags, risk))
|
|
print(f"{model}: {len(mismatches)} mismatches")
|
|
for m in mismatches:
|
|
print(f" Case {m[0]}: expected={m[1]} found={m[2]} risk={m[3]}")
|
|
|
|
print()
|
|
print("=== TERMINOLOGY DENSITY ===")
|
|
for data, model in [(data4b,'qwen3:4b'), (data8b,'qwen3:8b'), (data_e4b,'gemma4:e4b')]:
|
|
counts = []
|
|
for item in data:
|
|
resp = item.get('response','') or item.get('generated_response','')
|
|
counts.append(sum(1 for t in DOMAIN_TERMS if t in resp))
|
|
print(f"{model}: avg={statistics.mean(counts):.2f} min={min(counts)} max={max(counts)}")
|