#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
海外中国投资项目分析简报 - 优化版采集脚本
整合 RSS + SearXNG + 手动 curated 数据源
"""

import argparse
import feedparser
import json
from datetime import datetime, timedelta
from pathlib import Path
import re
import time
import random
import urllib.parse
import urllib.request


# ==================== 配置 ====================

# 项目配置
PROJECTS = {
    '尼贝石油管道': {
        'countries': ['尼日尔', '贝宁', '西非'],
        'keywords_zh': ['尼贝', '石油管道', '原油管道', '输油管道'],
        'keywords_en': ['Niger Benin pipeline', 'oil pipeline', 'crude pipeline'],
        'companies': ['中石油', '中海油', 'CNPC', 'CNOOC'],
        'risk_level': 'high'
    },
    '莱比塘铜矿': {
        'countries': ['缅甸'],
        'keywords_zh': ['莱比塘', '铜矿', '铜业'],
        'keywords_en': ['Letpadaung', 'copper mine', 'copper project'],
        'companies': ['北方工业', '万宝矿产', 'Norinco'],
        'risk_level': 'medium'
    },
    '德崇扶南运河': {
        'countries': ['柬埔寨'],
        'keywords_zh': ['德崇', '扶南', '运河', '水道'],
        'keywords_en': ['Funan Techo', 'canal', 'waterway'],
        'companies': ['中国路桥', '中国交建', 'CRBC', 'CCCC'],
        'risk_level': 'medium'
    },
    '中缅石油管道': {
        'countries': ['缅甸', '中国'],
        'keywords_zh': ['中缅', '石油管道', '原油管道', '皎漂'],
        'keywords_en': ['China Myanmar pipeline', 'oil pipeline', 'Kyaukpyu'],
        'companies': ['中石油', 'CNPC', '东南亚管道'],
        'risk_level': 'high'
    }
}

# 高质量媒体源（经过验证）
PREMIUM_FEEDS = [
    {'name': '新华社', 'url': 'http://www.xinhuanet.com/world/rss.xml', 'region': '大陆', 'trust': 'A'},
    {'name': '人民网', 'url': 'http://en.people.cn/rss/102786.xml', 'region': '大陆', 'trust': 'A'},
    {'name': '中国日报', 'url': 'http://www.chinadaily.com.cn/china/rss.xml', 'region': '大陆', 'trust': 'A'},
    {'name': '观察者网', 'url': 'https://www.guancha.cn/rss.xml', 'region': '大陆', 'trust': 'A'},
    {'name': '环球时报', 'url': 'https://www.globaltimes.cn/rss/world.xml', 'region': '大陆', 'trust': 'A'},
    {'name': '路透社', 'url': 'https://www.reuters.com/rss/world/africa', 'region': '国际', 'trust': 'A'},
    {'name': '彭博社', 'url': 'https://www.bloomberg.com/feed/podcast/etf-report.xml', 'region': '国际', 'trust': 'A'},
    {'name': 'Control Risks', 'url': 'https://www.controlrisks.com/en/our-thinking/rss', 'region': '专业', 'trust': 'A'},
]

# 黑名单媒体
BLOCKLIST = {'大纪元', '新唐人', '美国之音', 'VOA', 'RFA', '自由亚洲', '自由欧洲'}


# ==================== 工具函数 ====================

def clean_html(text):
    """清理 HTML 标签"""
    if not text:
        return ''
    text = re.sub(r'<[^>]+>', '', text)
    text = re.sub(r'\s+', ' ', text)
    return text.strip()[:500]


def is_blocked(title, summary):
    """检查是否在黑名单中"""
    text = f"{title} {summary}".lower()
    for blocked in BLOCKLIST:
        if blocked.lower() in text:
            return True
    return False


def match_project(title, summary):
    """匹配项目"""
    text = f"{title} {summary}"
    
    for project, config in PROJECTS.items():
        # 检查中文关键词
        for kw in config['keywords_zh']:
            if kw in text:
                return project, config
        
        # 检查英文关键词
        for kw in config['keywords_en']:
            if kw.lower() in text.lower():
                return project, config
        
        # 检查公司名称
        for co in config['companies']:
            if co in text:
                return project, config
        
        # 检查国家/地区
        for country in config['countries']:
            if country in text and any(kw in text for kw in ['投资', '项目', 'pipeline', 'mine', 'canal']):
                return project, config
    
    return None, None


def classify_risk(title, summary):
    """分类风险类型"""
    text = f"{title} {summary}".lower()
    
    risk_types = []
    
    # 政治风险
    if any(kw in text for kw in ['政', '党', '选', '政府', '政治', 'election', 'government', 'policy']):
        risk_types.append('政治风险')
    
    # 安全风险
    if any(kw in text for kw in ['袭击', '恐怖', '武装', '冲突', '安全', 'attack', 'terror', 'conflict', 'security']):
        risk_types.append('安全风险')
    
    # 社会风险
    if any(kw in text for kw in ['抗议', '罢工', '社区', '民众', '示威', 'protest', 'strike', 'community']):
        risk_types.append('社会风险')
    
    # 生态风险
    if any(kw in text for kw in ['环保', '生态', '环境', '污染', '环评', 'environment', 'ecological', 'pollution']):
        risk_types.append('生态风险')
    
    # 经济风险
    if any(kw in text for kw in ['经济', '投资', '资金', '汇率', '制裁', 'economic', 'investment', 'sanction']):
        risk_types.append('经济风险')
    
    # 运营风险
    if any(kw in text for kw in ['停工', '检修', '运营', '生产', 'operat', 'maintenance', 'production']):
        risk_types.append('运营风险')
    
    return risk_types if risk_types else ['其他']


def severity_level(title, summary, risk_types):
    """判定风险等级"""
    text = f"{title} {summary}".lower()
    
    # 高危
    high_terms = ['伤亡', '死亡', '袭击', '政变', '内战', '撤离', '征收', '恐怖', '武装冲突',
                  'casualty', 'death', 'attack', 'coup', 'evacuation', 'nationalization']
    if any(t in text for t in high_terms):
        return 'high'
    
    # 中危
    medium_terms = ['抗议', '罢工', '停工', '制裁', '调查', '反对', '大规模',
                    'protest', 'strike', 'suspension', 'sanction', 'investigation', 'opposition']
    if any(t in text for t in medium_terms):
        return 'medium'
    
    # 低危
    return 'low'


# ==================== 数据采集 ====================

def fetch_rss(feed_url, hours=72):
    """抓取 RSS 源"""
    entries = []
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
        feed = feedparser.parse(feed_url, request_headers=headers)
        
        if not feed.entries:
            return entries
        
        cutoff = datetime.now() - timedelta(hours=hours)
        
        for entry in feed.entries[:20]:
            published = None
            if hasattr(entry, 'published_parsed') and entry.published_parsed:
                try:
                    published = datetime(*entry.published_parsed[:6])
                except:
                    pass
            
            title = entry.get('title', '')
            summary = clean_html(entry.get('summary', '') or entry.get('description', ''))
            link = entry.get('link', '')
            
            # 检查黑名单
            if is_blocked(title, summary):
                continue
            
            entries.append({
                'title': title,
                'link': link,
                'published': published.isoformat() if published else None,
                'summary': summary,
                'source': feed.feed.get('title', 'Unknown'),
                'region': 'RSS'
            })
    except Exception as e:
        pass
    
    return entries


def searxng_search(query, searxng_url='http://localhost:8080'):
    """SearXNG 搜索"""
    results = []
    try:
        encoded = urllib.parse.quote(query)
        url = f'{searxng_url}/search?q={encoded}&format=json&language=zh-CN&language=en'
        
        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read().decode('utf-8'))
            
            for result in data.get('results', [])[:15]:
                title = result.get('title', '')
                summary = clean_html(result.get('content', ''))
                link = result.get('url', result.get('link', ''))
                
                if is_blocked(title, summary):
                    continue
                
                results.append({
                    'title': title,
                    'link': link,
                    'published': result.get('publishedDate', None),
                    'summary': summary,
                    'source': result.get('source', 'Unknown'),
                    'region': 'SearXNG'
                })
    except Exception as e:
        pass
    
    return results


# ==================== 主流程 ====================

def collect_all(searxng_url='http://localhost:8080', hours=72):
    """执行完整采集"""
    all_entries = []
    stats = {'rss': 0, 'searxng': 0, 'by_project': {}}
    
    print("=" * 60)
    print("海外中国投资项目 - 数据采集")
    print("=" * 60)
    
    # 1. RSS 采集
    print("\n【1】RSS 采集...")
    for feed in PREMIUM_FEEDS:
        print(f"  采集：{feed['name']}...", end=' ')
        entries = fetch_rss(feed['url'], hours)
        # 过滤项目相关
        project_entries = []
        for e in entries:
            project, _ = match_project(e['title'], e['summary'])
            if project:
                e['project'] = project
                project_entries.append(e)
        
        if project_entries:
            print(f"✓ {len(project_entries)} 条")
            all_entries.extend(project_entries)
            stats['rss'] += len(project_entries)
        else:
            print("○")
        time.sleep(random.uniform(0.3, 0.8))
    
    # 2. SearXNG 搜索
    print("\n【2】SearXNG 搜索...")
    for project, config in PROJECTS.items():
        print(f"  搜索：{project}...", end=' ')
        
        # 构建搜索查询
        queries = []
        queries.extend([f"{kw} 中国 投资" for kw in config['keywords_zh'][:2]])
        queries.extend([f"{kw} China" for kw in config['keywords_en'][:2]])
        
        results = []
        for q in queries[:3]:
            results.extend(searxng_search(q, searxng_url))
            time.sleep(0.5)
        
        # 过滤和标记
        project_results = []
        for r in results:
            matched_project, _ = match_project(r['title'], r['summary'])
            if matched_project == project:
                r['project'] = project
                r['risk_types'] = classify_risk(r['title'], r['summary'])
                r['severity'] = severity_level(r['title'], r['summary'], r['risk_types'])
                project_results.append(r)
        
        # 去重
        seen = set()
        unique = []
        for r in project_results:
            if r['link'] not in seen:
                seen.add(r['link'])
                unique.append(r)
        
        print(f"✓ {len(unique)} 条")
        all_entries.extend(unique[:10])
        stats['searxng'] += len(unique[:10])
        
        if project not in stats['by_project']:
            stats['by_project'][project] = 0
        stats['by_project'][project] += len(unique[:10])
    
    # 3. 去重
    print("\n【3】去重处理...")
    seen = set()
    unique_entries = []
    for e in all_entries:
        if e['link'] not in seen:
            seen.add(e['link'])
            unique_entries.append(e)
    
    print(f"  原始：{len(all_entries)} 条 → 去重：{len(unique_entries)} 条")
    
    # 4. 分类统计
    print("\n【4】分类统计...")
    for e in unique_entries:
        if 'project' not in e:
            project, config = match_project(e['title'], e['summary'])
            if project:
                e['project'] = project
                e['risk_types'] = classify_risk(e['title'], e['summary'])
                e['severity'] = severity_level(e['title'], e['summary'], e['risk_types'])
    
    # 按项目分组统计
    project_stats = {}
    severity_stats = {'high': 0, 'medium': 0, 'low': 0}
    
    for e in unique_entries:
        project = e.get('project', '其他')
        if project not in project_stats:
            project_stats[project] = 0
        project_stats[project] += 1
        
        sev = e.get('severity', 'low')
        severity_stats[sev] += 1
    
    print(f"  按项目：{project_stats}")
    print(f"  按风险：高={severity_stats['high']}, 中={severity_stats['medium']}, 低={severity_stats['low']}")
    
    return unique_entries, stats, project_stats, severity_stats


def format_output(entries, project_stats, severity_stats):
    """格式化输出"""
    output = []
    output.append("# 原始采集素材（优化版）\n")
    output.append(f"**采集时间：** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    output.append(f"**总条数：** {len(entries)}")
    output.append(f"**高危信息：** {severity_stats.get('high', 0)} 条")
    output.append(f"**中危信息：** {severity_stats.get('medium', 0)} 条\n")
    output.append("---\n")
    
    # 按项目分组
    by_project = {}
    for e in entries:
        project = e.get('project', '其他')
        if project not in by_project:
            by_project[project] = []
        by_project[project].append(e)
    
    for project, project_entries in by_project.items():
        output.append(f"## {project} ({len(project_entries)} 条)\n")
        
        # 按风险等级排序
        sorted_entries = sorted(project_entries, 
                               key=lambda x: {'high': 0, 'medium': 1, 'low': 2}.get(x.get('severity', 'low'), 2))
        
        for i, e in enumerate(sorted_entries[:15], 1):
            severity_icon = {'high': '🔴', 'medium': '🟠', 'low': '🟢'}.get(e.get('severity', 'low'), '⚪')
            risk_tags = ', '.join(e.get('risk_types', ['其他']))
            
            output.append(f"### {i}. {severity_icon} {e['title']}\n")
            output.append(f"**来源：** {e.get('source', '未知')} | **风险：** {risk_tags}")
            if e.get('published'):
                pub_date = str(e['published'])[:10]
                output.append(f" | **时间：** {pub_date}")
            output.append(f"\n**链接：** {e['link']}\n")
            if e.get('summary'):
                output.append(f"\n**摘要：** {e['summary']}\n")
            output.append("\n---\n")
    
    return '\n'.join(output)


def main():
    parser = argparse.ArgumentParser(description='优化版采集脚本')
    parser.add_argument('--searxng-url', default='http://localhost:8080')
    parser.add_argument('--hours', type=int, default=72)
    parser.add_argument('--output', required=True)
    
    args = parser.parse_args()
    
    # 执行采集
    entries, stats, project_stats, severity_stats = collect_all(args.searxng_url, args.hours)
    
    # 格式化输出
    output = format_output(entries, project_stats, severity_stats)
    
    # 保存
    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
    with open(args.output, 'w', encoding='utf-8') as f:
        f.write(output)
    
    # 保存 JSON 数据（供简报生成使用）
    json_path = args.output.replace('.md', '.json')
    with open(json_path, 'w', encoding='utf-8') as f:
        json.dump({
            'entries': entries,
            'stats': stats,
            'project_stats': project_stats,
            'severity_stats': severity_stats,
            'timestamp': datetime.now().isoformat()
        }, f, ensure_ascii=False, indent=2)
    
    print(f"\n✓ 输出：{args.output}")
    print(f"✓ JSON: {json_path}")


if __name__ == '__main__':
    main()