quick_verify_crossref.py

#!/usr/bin/env python3
"""
快速验证Crossref API的is-referenced-by-count字段准确性
"""

import requests
import json
import time

def quick_verify_crossref(query="machine learning", limit=5):
    """快速验证Crossref引用量准确性"""
    base_url = "https://api.crossref.org/works"
    headers = {
        'User-Agent': 'Academic-Reviewer-System/1.0 (mailto:test@example.com)'
    }
    
    print(f"=== 快速验证Crossref引用量 ===")
    print(f"查询: {query}")
    print(f"验证数量: {limit}")
    print("=" * 60)
    
    # 1. 搜索并按引用量排序
    print("\n--- 步骤1: 搜索并按引用量排序 ---")
    params = {
        "query": query,
        "rows": min(limit * 2, 20),  # 获取更多结果
        "sort": "relevance",
        "order": "desc",
        "select": "DOI,title,author,container-title,published-print,published-online,is-referenced-by-count"
    }
    
    try:
        response = requests.get(base_url, params=params, headers=headers, timeout=30)
        response.raise_for_status()
        data = response.json()
        
        items = data.get("message", {}).get("items", [])
        print(f"获取到 {len(items)} 个搜索结果")
        
        # 按引用量排序
        sorted_items = sorted(items, key=lambda x: x.get('is-referenced-by-count', 0), reverse=True)
        
        # 显示前几个结果
        print(f"\n按引用量排序的前{limit}个结果:")
        for i, item in enumerate(sorted_items[:limit], 1):
            title = item.get('title', ['N/A'])[0] if item.get('title') else 'N/A'
            if len(title) > 50:
                title = title[:50] + "..."
            
            cited_count = item.get('is-referenced-by-count', 0)
            doi = item.get('DOI', 'N/A')
            
            print(f"{i}. {title}")
            print(f"   DOI: {doi}")
            print(f"   搜索中的引用量: {cited_count}")
        
        # 2. 通过DOI验证引用量
        print(f"\n--- 步骤2: 通过DOI验证引用量 ---")
        verification_results = []
        
        for i, item in enumerate(sorted_items[:limit], 1):
            doi = item.get('DOI')
            if not doi:
                print(f"{i}. 无DOI，跳过验证")
                continue
            
            print(f"{i}. 验证DOI: {doi}")
            
            # 获取DOI详细信息
            try:
                doi_url = f"https://api.crossref.org/works/{doi}"
                doi_response = requests.get(doi_url, headers=headers, timeout=30)
                doi_response.raise_for_status()
                doi_data = doi_response.json()
                
                doi_details = doi_data.get("message", {})
                
                if doi_details:
                    search_citation = item.get('is-referenced-by-count', 0)
                    doi_citation = doi_details.get('is-referenced-by-count', 0)
                    
                    title = doi_details.get('title', ['N/A'])[0] if doi_details.get('title') else 'N/A'
                    if len(title) > 40:
                        title = title[:40] + "..."
                    
                    print(f"   标题: {title}")
                    print(f"   搜索中的引用量: {search_citation}")
                    print(f"   DOI查询的引用量: {doi_citation}")
                    print(f"   匹配状态: {'✅ 匹配' if search_citation == doi_citation else '❌ 不匹配'}")
                    
                    verification_results.append({
                        'doi': doi,
                        'search_citation': search_citation,
                        'doi_citation': doi_citation,
                        'match': search_citation == doi_citation
                    })
                else:
                    print(f"   ❌ 无法获取DOI详细信息")
                    verification_results.append({
                        'doi': doi,
                        'search_citation': item.get('is-referenced-by-count', 0),
                        'doi_citation': -1,
                        'match': False
                    })
                
            except Exception as e:
                print(f"   ❌ DOI查询失败: {str(e)}")
                verification_results.append({
                    'doi': doi,
                    'search_citation': item.get('is-referenced-by-count', 0),
                    'doi_citation': -1,
                    'match': False
                })
            
            print()
            time.sleep(0.5)  # 避免请求过快
        
        # 3. 统计结果
        print(f"\n--- 验证结果统计 ---")
        total_verified = len(verification_results)
        matched = sum(1 for r in verification_results if r['match'])
        mismatched = total_verified - matched
        
        print(f"总验证数: {total_verified}")
        print(f"匹配数: {matched}")
        print(f"不匹配数: {mismatched}")
        print(f"匹配率: {matched/total_verified*100:.1f}%" if total_verified > 0 else "匹配率: 0%")
        
        # 显示不匹配的详情
        if mismatched > 0:
            print(f"\n--- 不匹配详情 ---")
            for result in verification_results:
                if not result['match']:
                    print(f"DOI: {result['doi']}")
                    print(f"  搜索引用量: {result['search_citation']}")
                    print(f"  DOI引用量: {result['doi_citation']}")
                    if result['doi_citation'] != -1:
                        print(f"  差异: {abs(result['search_citation'] - result['doi_citation'])}")
                    print()
        
        return verification_results
        
    except Exception as e:
        print(f"搜索失败: {str(e)}")
        return []

def test_multiple_queries():
    """测试多个查询"""
    test_queries = [
        "machine learning",
        "CRISPR", 
        "cryo-electron microscopy"
    ]
    
    print(f"\n{'='*80}")
    print("多查询验证测试")
    print(f"{'='*80}")
    
    all_results = []
    
    for query in test_queries:
        print(f"\n--- 测试查询: {query} ---")
        results = quick_verify_crossref(query, limit=3)
        all_results.extend(results)
        time.sleep(1)
    
    # 总体统计
    print(f"\n{'='*80}")
    print("总体验证统计")
    print(f"{'='*80}")
    
    total_verified = len(all_results)
    matched = sum(1 for r in all_results if r['match'])
    mismatched = total_verified - matched
    
    print(f"总验证数: {total_verified}")
    print(f"匹配数: {matched}")
    print(f"不匹配数: {mismatched}")
    print(f"总体匹配率: {matched/total_verified*100:.1f}%" if total_verified > 0 else "总体匹配率: 0%")
    
    # 引用量分布
    search_citations = [r['search_citation'] for r in all_results if r['search_citation'] > 0]
    doi_citations = [r['doi_citation'] for r in all_results if r['doi_citation'] > 0]
    
    if search_citations:
        print(f"\n搜索引用量分布:")
        print(f"  非零数量: {len(search_citations)}/{total_verified}")
        print(f"  平均: {sum(search_citations) / len(search_citations):.2f}")
        print(f"  最大: {max(search_citations)}")
    
    if doi_citations:
        print(f"\nDOI引用量分布:")
        print(f"  非零数量: {len(doi_citations)}/{total_verified}")
        print(f"  平均: {sum(doi_citations) / len(doi_citations):.2f}")
        print(f"  最大: {max(doi_citations)}")

if __name__ == "__main__":
    print("Crossref引用量快速验证工具")
    print("=" * 60)
    
    # 快速测试
    quick_verify_crossref("machine learning", 5)
    
    # 多查询测试
    test_multiple_queries()