feat(test): add 'acceptable' match type for similarity >= 60%
Add a new match category 'acceptable' for institution name matches with similarity between 60% and 85%, providing more nuanced matching results. Changes: 1. Add ACCEPTABLE_THRESHOLD = 60.0 constant 2. Update classify_match() to include 'acceptable' category 3. Add blue color (#2196f3) for acceptable matches in reports 4. Update all statistics to count acceptable matches separately 5. Modify HTML summary to show 5 columns instead of 4 6. Update JSON output to include acceptable count 7. Add [ACCEPTABLE] symbol in result tables Match levels (from highest to lowest): - exact: 100% similarity → green - partial: >= 85% similarity → orange - acceptable: >= 60% similarity → blue ← NEW - no_match: < 60% similarity → red This improves the granularity of match reporting, especially for cases where OCR artifacts or minor variations cause similarity to drop below the 85% partial threshold but are still reasonably accurate. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
parent
f5981fdf72
commit
22773f3cc8
|
|
@ -127,6 +127,7 @@ RESULTS_JSON = Path(r"src/test/resources/data/results.json")
|
|||
OUTPUT_DIR = Path("test_reports_full")
|
||||
BATCH_SIZE = 20
|
||||
SIMILARITY_THRESHOLD = 85.0
|
||||
ACCEPTABLE_THRESHOLD = 60.0 # 相似度阈值，用于判断"acceptable"级别的匹配
|
||||
|
||||
# OCR Model Configuration
|
||||
# Options: "ppocr_v5" (default), "paddleocr_vl"
|
||||
|
|
@ -1594,6 +1595,8 @@ def classify_match(extracted: Optional[str], expected: str, field_type: str = 'd
|
|||
match_type = 'exact'
|
||||
elif similarity >= SIMILARITY_THRESHOLD:
|
||||
match_type = 'partial'
|
||||
elif similarity >= ACCEPTABLE_THRESHOLD:
|
||||
match_type = 'acceptable'
|
||||
else:
|
||||
match_type = 'no_match'
|
||||
|
||||
|
|
@ -1883,8 +1886,8 @@ def generate_individual_report(result: Dict[str, Any], output_dir: Path):
|
|||
total_time = result['performance']['total_time']
|
||||
|
||||
# Colors
|
||||
cma_color = '#4caf50' if cma_match == 'exact' else '#ff9800' if cma_match == 'partial' else '#f44336'
|
||||
inst_color = '#4caf50' if inst_match == 'exact' else '#ff9800' if inst_match == 'partial' else '#f44336'
|
||||
cma_color = '#4caf50' if cma_match == 'exact' else '#ff9800' if cma_match == 'partial' else '#2196f3' if cma_match == 'acceptable' else '#f44336'
|
||||
inst_color = '#4caf50' if inst_match == 'exact' else '#ff9800' if inst_match == 'partial' else '#2196f3' if inst_match == 'acceptable' else '#f44336'
|
||||
|
||||
# Build seals HTML
|
||||
seals_html = ""
|
||||
|
|
@ -2025,11 +2028,13 @@ def generate_summary_report(all_results: List[Dict[str, Any]], output_dir: Path)
|
|||
|
||||
cma_exact = sum(1 for r in valid_cma if r['comparison']['cma'].get('match_type') == 'exact')
|
||||
cma_partial = sum(1 for r in valid_cma if r['comparison']['cma'].get('match_type') == 'partial')
|
||||
cma_no = len(valid_cma) - cma_exact - cma_partial
|
||||
cma_acceptable = sum(1 for r in valid_cma if r['comparison']['cma'].get('match_type') == 'acceptable')
|
||||
cma_no = len(valid_cma) - cma_exact - cma_partial - cma_acceptable
|
||||
|
||||
inst_exact = sum(1 for r in valid_inst if r['comparison']['institution'].get('match_type') == 'exact')
|
||||
inst_partial = sum(1 for r in valid_inst if r['comparison']['institution'].get('match_type') == 'partial')
|
||||
inst_no = len(valid_inst) - inst_exact - inst_partial
|
||||
inst_acceptable = sum(1 for r in valid_inst if r['comparison']['institution'].get('match_type') == 'acceptable')
|
||||
inst_no = len(valid_inst) - inst_exact - inst_partial - inst_acceptable
|
||||
|
||||
cma_acc = (cma_exact / len(valid_cma) * 100) if valid_cma else 0
|
||||
inst_acc = (inst_exact / len(valid_inst) * 100) if valid_inst else 0
|
||||
|
|
@ -2045,7 +2050,7 @@ def generate_summary_report(all_results: List[Dict[str, Any]], output_dir: Path)
|
|||
body {{ font-family: 'Segoe UI', sans-serif; margin: 0; padding: 20px; background: #f5f5f5; }}
|
||||
.container {{ max-width: 1400px; margin: 0 auto; background: white; padding: 30px; border-radius: 8px; }}
|
||||
h1 {{ color: #333; }}
|
||||
.summary {{ display: grid; grid-template-columns: repeat(4, 1fr); gap: 20px; margin: 20px 0; }}
|
||||
.summary {{ display: grid; grid-template-columns: repeat(5, 1fr); gap: 15px; margin: 20px 0; }}
|
||||
.summary-card {{ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); padding: 20px; border-radius: 8px; color: white; text-align: center; }}
|
||||
.summary-card .label {{ font-size: 14px; opacity: 0.9; }}
|
||||
.summary-card .value {{ font-size: 32px; font-weight: bold; }}
|
||||
|
|
@ -2069,11 +2074,15 @@ def generate_summary_report(all_results: List[Dict[str, Any]], output_dir: Path)
|
|||
<div class="label">Partial Match</div>
|
||||
<div class="value">{cma_partial}/{len(valid_cma)}</div>
|
||||
</div>
|
||||
<div class="summary-card" style="background: linear-gradient(135deg, #2196f3 0%, #1976d2 100%);">
|
||||
<div class="label">Acceptable</div>
|
||||
<div class="value">{cma_acceptable}/{len(valid_cma)}</div>
|
||||
</div>
|
||||
<div class="summary-card" style="background: linear-gradient(135deg, #f44336 0%, #d32f2f 100%);">
|
||||
<div class="label">No Match</div>
|
||||
<div class="value">{cma_no}/{len(valid_cma)}</div>
|
||||
</div>
|
||||
<div class="summary-card" style="background: linear-gradient(135deg, #2196F3 0%, #1976D2 100%);">
|
||||
<div class="summary-card" style="background: linear-gradient(135deg, #9C27B0 0%, #7B1FA2 100%);">
|
||||
<div class="label">Accuracy</div>
|
||||
<div class="value">{cma_acc:.1f}%</div>
|
||||
</div>
|
||||
|
|
@ -2089,11 +2098,15 @@ def generate_summary_report(all_results: List[Dict[str, Any]], output_dir: Path)
|
|||
<div class="label">Partial Match</div>
|
||||
<div class="value">{inst_partial}/{len(valid_inst)}</div>
|
||||
</div>
|
||||
<div class="summary-card" style="background: linear-gradient(135deg, #2196f3 0%, #1976d2 100%);">
|
||||
<div class="label">Acceptable</div>
|
||||
<div class="value">{inst_acceptable}/{len(valid_inst)}</div>
|
||||
</div>
|
||||
<div class="summary-card" style="background: linear-gradient(135deg, #f44336 0%, #d32f2f 100%);">
|
||||
<div class="label">No Match</div>
|
||||
<div class="value">{inst_no}/{len(valid_inst)}</div>
|
||||
</div>
|
||||
<div class="summary-card" style="background: linear-gradient(135deg, #2196F3 0%, #1976D2 100%);">
|
||||
<div class="summary-card" style="background: linear-gradient(135deg, #9C27B0 0%, #7B1FA2 100%);">
|
||||
<div class="label">Accuracy</div>
|
||||
<div class="value">{inst_acc:.1f}%</div>
|
||||
</div>
|
||||
|
|
@ -2120,8 +2133,8 @@ def generate_summary_report(all_results: List[Dict[str, Any]], output_dir: Path)
|
|||
<tbody>"""
|
||||
|
||||
for r in all_results:
|
||||
cma_symbol = {'exact': '[OK]', 'partial': '[PARTIAL]', 'no_match': '[FAIL]'}.get(r['comparison'].get('cma', {}).get('match_type', 'no_match'), '[?]')
|
||||
inst_symbol = {'exact': '[OK]', 'partial': '[PARTIAL]', 'no_match': '[FAIL]'}.get(r['comparison'].get('institution', {}).get('match_type', 'no_match'), '[?]')
|
||||
cma_symbol = {'exact': '[OK]', 'partial': '[PARTIAL]', 'acceptable': '[ACCEPTABLE]', 'no_match': '[FAIL]'}.get(r['comparison'].get('cma', {}).get('match_type', 'no_match'), '[?]')
|
||||
inst_symbol = {'exact': '[OK]', 'partial': '[PARTIAL]', 'acceptable': '[ACCEPTABLE]', 'no_match': '[FAIL]'}.get(r['comparison'].get('institution', {}).get('match_type', 'no_match'), '[?]')
|
||||
seals_count = len(r['seal_results'])
|
||||
|
||||
html += f"""
|
||||
|
|
@ -2360,13 +2373,15 @@ def main():
|
|||
valid_cma = [r for r in all_results if r['expected']['cma'] not in ['无', None]]
|
||||
cma_exact = sum(1 for r in valid_cma if r['comparison']['cma'].get('match_type') == 'exact')
|
||||
cma_partial = sum(1 for r in valid_cma if r['comparison']['cma'].get('match_type') == 'partial')
|
||||
cma_no = len(valid_cma) - cma_exact - cma_partial
|
||||
cma_acceptable = sum(1 for r in valid_cma if r['comparison']['cma'].get('match_type') == 'acceptable')
|
||||
cma_no = len(valid_cma) - cma_exact - cma_partial - cma_acceptable
|
||||
cma_acc = (cma_exact / len(valid_cma) * 100) if valid_cma else 0
|
||||
|
||||
valid_inst = [r for r in all_results if r['expected']['institution'] not in ['无', None] and r['extracted']['institution']]
|
||||
inst_exact = sum(1 for r in valid_inst if r['comparison']['institution'].get('match_type') == 'exact')
|
||||
inst_partial = sum(1 for r in valid_inst if r['comparison']['institution'].get('match_type') == 'partial')
|
||||
inst_no = len(valid_inst) - inst_exact - inst_partial
|
||||
inst_acceptable = sum(1 for r in valid_inst if r['comparison']['institution'].get('match_type') == 'acceptable')
|
||||
inst_no = len(valid_inst) - inst_exact - inst_partial - inst_acceptable
|
||||
inst_acc = (inst_exact / len(valid_inst) * 100) if valid_inst else 0
|
||||
|
||||
# Generate summary report
|
||||
|
|
@ -2380,12 +2395,14 @@ def main():
|
|||
'cma': {
|
||||
'exact': cma_exact,
|
||||
'partial': cma_partial,
|
||||
'acceptable': cma_acceptable,
|
||||
'no_match': cma_no,
|
||||
'accuracy': cma_acc / 100
|
||||
},
|
||||
'institution': {
|
||||
'exact': inst_exact,
|
||||
'partial': inst_partial,
|
||||
'acceptable': inst_acceptable,
|
||||
'no_match': inst_no,
|
||||
'accuracy': inst_acc / 100
|
||||
},
|
||||
|
|
@ -2406,12 +2423,14 @@ def main():
|
|||
print("CMA Code Results:")
|
||||
print(f" Exact Match: {cma_exact}/{len(valid_cma)} ({cma_exact/len(valid_cma)*100:.1f}%)")
|
||||
print(f" Partial Match: {cma_partial}/{len(valid_cma)} ({cma_partial/len(valid_cma)*100:.1f}%)")
|
||||
print(f" Acceptable Match: {cma_acceptable}/{len(valid_cma)} ({cma_acceptable/len(valid_cma)*100:.1f}%)")
|
||||
print(f" No Match: {cma_no}/{len(valid_cma)} ({cma_no/len(valid_cma)*100:.1f}%)")
|
||||
print(f" ** CMA Accuracy: {cma_acc:.1f}% **")
|
||||
print()
|
||||
print("Institution Name Results:")
|
||||
print(f" Exact Match: {inst_exact}/{len(valid_inst)} ({inst_exact/len(valid_inst)*100:.1f}%)")
|
||||
print(f" Partial Match: {inst_partial}/{len(valid_inst)} ({inst_partial/len(valid_inst)*100:.1f}%)")
|
||||
print(f" Acceptable Match: {inst_acceptable}/{len(valid_inst)} ({inst_acceptable/len(valid_inst)*100:.1f}%)")
|
||||
print(f" No Match: {inst_no}/{len(valid_inst)} ({inst_no/len(valid_inst)*100:.1f}%)")
|
||||
print(f" ** Institution Accuracy: {inst_acc:.1f}% **")
|
||||
print()
|
||||
|
|
|
|||
Loading…
Reference in New Issue