#!/usr/bin/env python3
"""
数据清洗器
清理干扰信息、标准化字段、验证数据质量
"""

import re
import json
from typing import Dict, List, Any, Set, Tuple
from dataclasses import dataclass
from datetime import datetime
from collections import Counter


@dataclass
class CleaningConfig:
    """清洗配置"""
    remove_html_tags: bool = True           # 移除HTML标签
    remove_wiki_noise: bool = True          # 移除Wikipedia噪声
    standardize_date: bool = True           # 标准化日期
    deduplicate: bool = True                # 去重
    validate_urls: bool = False             # 验证URL
    language: str = "en"                    # 语言


class DataCleaner:
    """数据清洗器"""
    
    # Wikipedia噪声模式
    WIKI_NOISE_PATTERNS = {
        'en': [
            r'\[edit\]',
            r'\[note \d+\]',
            r'\[source\?\]',
            r'\[citation needed\]',
            r'\[ref\].*?\[/ref\]',
            r'class="[^"]*"',
            r'title="[^"]*"',
            r'href="[^"]*"',
            r'References.*?(?:==|$)',
            r'See also.*?(?:==|$)',
            r'Further reading.*?(?:==|$)',
            r'External links.*?(?:==|$)',
            r'Infobox.*?(?:==|$)',
            r'Born.*?Died',  # 日期范围误识别
        ],
        'zh': [
            r'\[编辑\]',
            r'\[注释\]',
            r'\[来源请求\]',
            r'参考文献.*?(?:==|$)',
            r'另见.*?(?:==|$)',
            r'外部链接.*?(?:==|$)',
            r'Infobox.*?(?:==|$)',
        ],
        'ja': [
            r'\[編集\]',
            r'\[注釈\]',
            r'参考文献.*?(?:==|$)',
            r'関連項目.*?(?:==|$)',
            r'外部リンク.*?(?:==|$)',
        ]
    }
    
    # 日期模式
    DATE_PATTERNS = [
        r'(\d{4})年(\d{1,2})月(\d{1,2})日',
        r'(\d{4})-(\d{1,2})-(\d{1,2})',
        r'(\w+ \d{1,2},? \d{4})',
        r'(\d{1,2} \w+ \d{4})',
    ]
    
    # 需要清理的无效值
    INVALID_VALUES = {
        'en': ['unknown', 'unknown unknown', 'n/a', 'none', 'null', 'undefined', '[citation needed]'],
        'zh': ['未知', '不详', '无', '无资料', '暂无数据'],
        'ja': ['不明', '不明', 'なし', 'データなし'],
    }
    
    # 干扰信息关键词
    NOISE_KEYWORDS = {
        'en': [' Allegiance', ' Died', ' Service years', ' References', ' Notes', 
               ' External links', ' See also', ' v ', ' t ', ' d ', ' edit ',
               ' class=', ' style=', ' rowspan=', ' colspan='],
        'zh': ['效力', '逝世', '服役时间', '参考', '注释', '另见', '外部链接', 
               ' colspan=', ' rowspan=', ' class=', ' style='],
    }
    
    def __init__(self, config: CleaningConfig = None):
        self.config = config or CleaningConfig()
        self.lang = self.config.language
        self.stats = {
            'records_processed': 0,
            'records_cleaned': 0,
            'fields_cleaned': 0,
            'invalid_removed': 0,
            'noise_removed': 0,
        }
        
    def remove_wiki_noise(self, text: str) -> str:
        """移除Wikipedia噪声"""
        if not text:
            return ""
            
        result = text
        
        # 移除噪声模式
        patterns = self.WIKI_NOISE_PATTERNS.get(self.lang, self.WIKI_NOISE_PATTERNS['en'])
        for pattern in patterns:
            result = re.sub(pattern, '', result, flags=re.IGNORECASE)
            
        # 移除链接残片
        result = re.sub(r'\]', '', result)
        result = re.sub(r'\[+', '', result)
        
        return result
    
    def remove_noise_keywords(self, text: str) -> Tuple[str, int]:
        """移除干扰关键词"""
        if not text:
            return "", 0
            
        result = text
        count = 0
        
        keywords = self.NOISE_KEYWORDS.get(self.lang, self.NOISE_KEYWORDS['en'])
        for keyword in keywords:
            if keyword in result:
                result = result.replace(keyword, '')
                count += 1
                
        return result, count
    
    def clean_field(self, field_name: str, value: Any) -> Any:
        """清洗单个字段"""
        if value is None:
            return None
            
        if isinstance(value, str):
            # 检查是否为空值
            lower_val = value.lower().strip()
            invalid = self.INVALID_VALUES.get(self.lang, self.INVALID_VALUES['en'])
            if lower_val in invalid or not lower_val:
                self.stats['invalid_removed'] += 1
                return None
                
            # 移除HTML标签
            if self.config.remove_html_tags:
                value = re.sub(r'<[^>]+>', '', value)
                
            # 移除Wikipedia噪声
            if self.config.remove_wiki_noise:
                value = self.remove_wiki_noise(value)
                
            # 移除干扰关键词
            if '出生地' in field_name or 'birthplace' in field_name.lower():
                value, count = self.remove_noise_keywords(value)
                if count > 0:
                    self.stats['noise_removed'] += count
                    
            # 清理多余空白
            value = re.sub(r'\s+', ' ', value).strip()
            
            if value != str(value).strip():
                self.stats['fields_cleaned'] += 1
                
            return value if value else None
            
        return value
    
    def standardize_date(self, date_str: str) -> str:
        """标准化日期格式"""
        if not date_str:
            return ""
            
        # 尝试匹配各种日期格式
        for pattern in self.DATE_PATTERNS:
            match = re.search(pattern, date_str)
            if match:
                groups = match.groups()
                if len(groups) == 3:
                    try:
                        year, month, day = int(groups[0]), int(groups[1]), int(groups[2])
                        return f"{year:04d}-{month:02d}-{day:02d}"
                    except:
                        pass
                        
        return date_str
    
    def process_record(self, record: Dict) -> Dict:
        """处理单条记录"""
        self.stats['records_processed'] += 1
        cleaned_record = {}
        
        for field, value in record.items():
            cleaned_value = self.clean_field(field, value)
            if cleaned_value is not None:
                cleaned_record[field] = cleaned_value
                
        # 标准化日期字段
        date_fields = ['出生日期', 'birth_date', '去世日期', 'death_date', 'collection_time']
        for field in date_fields:
            if field in cleaned_record:
                cleaned_record[field] = self.standardize_date(cleaned_record[field])
                
        if cleaned_record != record:
            self.stats['records_cleaned'] += 1
            
        return cleaned_record
    
    def process_batch(self, records: List[Dict]) -> List[Dict]:
        """批量处理记录"""
        cleaned_records = []
        
        for record in records:
            cleaned = self.process_record(record)
            if cleaned:
                cleaned_records.append(cleaned)
                
        return cleaned_records
    
    def get_stats(self) -> Dict:
        """获取清洗统计"""
        return self.stats.copy()


def clean_records(records: List[Dict], config: CleaningConfig = None) -> List[Dict]:
    """
    清洗记录
    
    Args:
        records: 原始记录列表
        config: 清洗配置
        
    Returns:
        清洗后的记录
    """
    cleaner = DataCleaner(config)
    cleaned = cleaner.process_batch(records)
    
    print(f"清洗统计: {cleaner.get_stats()}")
    
    return cleaned


if __name__ == "__main__":
    print("Data Cleaner Module")
    print("用法: from cleaner import DataCleaner, clean_records")