语料质检

import json

def check_corpus(json_path, ratio_low=0.15, ratio_high=4.0):
    """对语料 JSON 做自动化质检，返回可疑句对列表。"""
    with open(json_path, "r", encoding="utf-8") as f:
        data = json.load(f)

    pairs = data["content"]
    issues = []

    seen = set()  # 重复检测

    for i, pair in enumerate(pairs):
        en = pair.get("sentence_english", "").strip()
        zh = pair.get("sentence_chinese", "").strip()
        idx = f"#{i + 1}"

        # ① 空句对
        if not en or not zh:
            issues.append({"index": idx, "type": "empty", "en": en, "zh": zh})
            continue

        # ② 英中长度比异常（字符数）
        ratio = len(zh) / len(en) if len(en) > 0 else 0
        if ratio < ratio_low or ratio > ratio_high:
            issues.append({
                "index": idx, "type": "length_ratio",
                "ratio": round(ratio, 2), "en": en[:60], "zh": zh[:60]
            })

        # ③ 完全重复
        key = (en, zh)
        if key in seen:
            issues.append({"index": idx, "type": "duplicate", "en": en[:60]})
        seen.add(key)

    return issues

# 使用示例
issues = check_corpus("corpus/china-english-language-education.json")
print(f"发现 {len(issues)} 条可疑记录")
for item in issues[:5]:
    print(item)