#!/usr/bin/env python3
"""
滨化集团 - 信息公开 爬虫
URL: https://www.befar.com/news/13.html
CloudWAF blocked from server, uses predefined list data + browser for detail
"""
import os, re, json, time, sqlite3
from datetime import datetime, date
import requests
from bs4 import BeautifulSoup

DB_PATH = "/root/search.db"
SITE_NAME = "滨化集团信息公开"
CATEGORY = "xxgk"
THRESHOLD = date(2023, 6, 18)

COUNT_NEW = COUNT_SKIP = COUNT_ERR = 0

# Pre-collected list data (manually extracted via browser)
LIST_DATA = {
    "page1": [
        {"date": "2026-04-17", "title": "山东滨华新材料有限公司首站新增燃气锅炉项目 竣工环境保护验收报告公示", "url": "https://www.befar.com/news/13/1097.html"},
        {"date": "2026-04-17", "title": "山东滨华新材料有限公司年产15万吨液氨水合物 改造项目竣工环境保护验收报告公示", "url": "https://www.befar.com/news/13/1096.html"},
        {"date": "2026-03-27", "title": "山东滨华新材料有限公司年产24万吨高性能连续法聚醚多元醇项目环境影响报告书报批前公示", "url": "https://www.befar.com/news/13/1092.html"},
        {"date": "2026-03-17", "title": "山东滨华新材料有限公司年产24万吨高性能连续法聚醚多元醇项目环境影响评价征求意见稿公示", "url": "https://www.befar.com/news/13/1088.html"},
        {"date": "2026-03-12", "title": "关于山东滨化东瑞化工有限责任公司外送清液管线项目 社会稳定风险评估公众参与公告", "url": "https://www.befar.com/news/13/1086.html"},
        {"date": "2026-02-28", "title": "山东滨化长峰新材料有限公司年产6万吨高纯溴化新材料项目社会稳定风险评估公示", "url": "https://www.befar.com/news/13/1079.html"},
        {"date": "2026-01-15", "title": "大晟芯材料科技（山东）有限公司电子级氢氟酸技术改造项目 环境影响评价报批前公示", "url": "https://www.befar.com/news/13/1056.html"},
        {"date": "2026-01-15", "title": "山东滨华新材料有限公司年产15万吨液氨水合物改造项目验收情况第一次公示", "url": "https://www.befar.com/news/13/1055.html"},
        {"date": "2026-01-15", "title": "山东滨华新材料有限公司首站新增燃气锅炉项目验收情况第一次公示", "url": "https://www.befar.com/news/13/1054.html"},
        {"date": "2026-01-14", "title": "山东滨华新材料有限公司资源循环利用项目 环境影响报告书报批前公示", "url": "https://www.befar.com/news/13/1053.html"},
        {"date": "2025-12-26", "title": "山东滨华新材料有限公司资源循环利用项目 环境影响评价第二次公众参与公告", "url": "https://www.befar.com/news/13/1044.html"},
        {"date": "2025-12-12", "title": "黄河三角洲（滨州）热力有限公司新增华纺热用户外供蒸汽项目 社会稳定风险评估公示", "url": "https://www.befar.com/news/13/1039.html"}
    ],
    "page2": [
        {"date": "2025-11-18", "title": "大晟芯材料科技（山东）有限公司电子级氢氟酸技术改造项目环境影响报告书（征求意见稿）公示", "url": "https://www.befar.com/news/13/1030.html"},
        {"date": "2025-11-10", "title": "黄河三角洲（滨州）热力有限公司北镇办事处热用户供汽项目社会稳定风险评估公示", "url": "https://www.befar.com/news/13/1024.html"},
        {"date": "2025-09-17", "title": "大晟芯材料科技（山东）有限公司电子级氢氟酸技术改造项目 环境影响评价公众参与第一次公示", "url": "https://www.befar.com/news/13/991.html"},
        {"date": "2025-08-10", "title": "山东滨华新材料有限公司资源循环利用项目环境影响评价第一次公众参与公告", "url": "https://www.befar.com/news/13/1022.html"},
        {"date": "2025-04-19", "title": "山东滨华新材料有限公司清洁生产基本信息公示", "url": "https://www.befar.com/news/13/905.html"},
        {"date": "2025-04-12", "title": "滨化集团股份有限公司清洁生产基本信息公示", "url": "https://www.befar.com/news/13/906.html"},
        {"date": "2025-03-20", "title": "山东滨华新材料有限公司碳三碳四综合利用项目节能验收节能验收公示说明", "url": "https://www.befar.com/news/13/886.html"},
        {"date": "2025-03-20", "title": "黄河三角洲（滨州）热力有限公司1×350MW超临界热电联产项目节能验收公示说明", "url": "https://www.befar.com/news/13/885.html"},
        {"date": "2025-03-13", "title": "山东滨化长悦新材料有限公司4000吨/年提溴项目节能验收公示", "url": "https://www.befar.com/uploads/files/20250313/efa6145aac6937c41b5151245869ea9e.pdf"},
        {"date": "2025-01-20", "title": "山东滨化新型建材有限责任公司企业自用阻隔防爆撬装式加油装置项目配套环境保护设施竣工及调试日期公示", "url": "https://www.befar.com/news/13/852.html"},
        {"date": "2024-12-25", "title": "山东滨华新材料有限公司碳三碳四综合利用项目（一期）丁烷异构装置及相关配套环境保护设施竣工及调试日期公示", "url": "https://www.befar.com/news/13/866.html"},
        {"date": "2024-12-25", "title": "山东滨华新材料有限公司碳三碳四综合利用项目（二期）配套环境保护设施竣工及调试日期公示", "url": "https://www.befar.com/news/13/865.html"}
    ],
    "page3": [
        {"date": "2024-11-11", "title": "滨化技术有限公司 1000吨/年本体聚合法聚甲基丙烯酸甲酯中试项目 环境影响报告书报批前信息公示", "url": "https://www.befar.com/news/13/803.html"},
        {"date": "2024-10-30", "title": "黄河三角洲（滨州）热力有限公司1×350MW超临界热电联产机组配套输变电工程配套环境保护设施竣工及调试日期公示", "url": "https://www.befar.com/news/13/851.html"},
        {"date": "2024-10-30", "title": "黄河三角洲（滨州）热力有限公司1×350MW超临界热电联产项目配套环境保护设施竣工及调试日期公示", "url": "https://www.befar.com/news/13/850.html"},
        {"date": "2024-10-23", "title": "滨化技术有限公司 1000吨/年本体聚合法聚甲基丙烯酸甲酯中试项目 环境影响评价信息公示（征求意见稿）", "url": "https://www.befar.com/news/13/793.html"},
        {"date": "2024-08-30", "title": "滨化集团股份有限公司1000吨/年聚甲基丙烯酸甲酯中试项目 环境影响评价信息首次公示", "url": "https://www.befar.com/news/13/742.html"},
        {"date": "2024-07-29", "title": "低压水电解中试项目环境影响评价（报批前）网络公示", "url": "https://www.befar.com/news/13/731.html"},
        {"date": "2024-07-15", "title": "滨化集团股份有限公司低压水电解制氢电解槽中试项目 环境影响报告书（征求意见稿）信息公开", "url": "https://www.befar.com/news/13/727.html"},
        {"date": "2024-06-25", "title": "低压水电解中试项目环评首次信息公示内容", "url": "https://www.befar.com/news/13/726.html"},
        {"date": "2024-06-12", "title": "山东滨化长悦新材料有限公司 1万吨/年新型环保型溴系阻燃剂综合项目社会稳定风险评估公示", "url": "https://www.befar.com/news/13/714.html"},
        {"date": "2024-06-12", "title": "高端电子化学品基地社会稳定风险评估公示", "url": "https://www.befar.com/news/13/713.html"},
        {"date": "2024-05-06", "title": "碳四下游高附加值产品一体化项目（一期）环境影响报告书报批前公示", "url": "https://www.befar.com/news/13/674.html"},
        {"date": "2024-04-24", "title": "山东滨华新材料有限公司 碳四下游高附加值产品一体化项目（一期） 环境影响评价公众参与公告", "url": "https://www.befar.com/news/13/664.html"}
    ],
    "page4": [
        {"date": "2024-04-15", "title": "山东滨州嘉源环保有限责任公司清洁生产审核信息公示", "url": "https://www.befar.com/news/13/663.html"},
        {"date": "2024-04-15", "title": "山东滨化东瑞化工有限责任公司清洁生产审核信息公示", "url": "https://www.befar.com/news/13/662.html"},
        {"date": "2024-04-10", "title": "大晟芯材料科技（山东）有限公司 电子化学品系列化项目(一期) 环境影响报告书（报批前）公示", "url": "https://www.befar.com/news/13/658.html"},
        {"date": "2024-04-07", "title": "滨化技术有限公司项目设计研究中心功能提升项目配套环境保护设施竣工及调试日期公示", "url": "https://www.befar.com/news/13/657.html"},
        {"date": "2024-04-07", "title": "山东滨华氢能源有限公司500kg/12h加氢站项目配套环境保护设施竣工及调试日期公示", "url": "https://www.befar.com/news/13/656.html"},
        {"date": "2024-04-07", "title": "山东滨化东瑞化工有限责任公司聚醚装置产品质量提升改造项目配套环境保护设施竣工及调试日期公示", "url": "https://www.befar.com/news/13/655.html"},
        {"date": "2024-03-20", "title": "山东滨华新材料有限公司碳四下游高附加值产品一体化项目（一期）社会稳定风险评估公示", "url": "https://www.befar.com/news/13/647.html"},
        {"date": "2024-02-18", "title": "大晟芯材料科技（山东）有限公司电子化学品系列化项目社会稳定风险评估公示", "url": "https://www.befar.com/news/13/636.html"},
        {"date": "2024-02-18", "title": "大晟芯材料科技（山东）有限公司氟化工罐区项目社会稳定风险评估公示", "url": "https://www.befar.com/news/13/635.html"},
        {"date": "2024-01-24", "title": "大晟芯材料科技（山东）有限公司 电子化学品系列化项目(一期) 环境影响报告书（征求意见稿）公示", "url": "https://www.befar.com/news/13/630.html"},
        {"date": "2023-10-19", "title": "关于选聘2023年度会计师事务所的通知", "url": "https://www.befar.com/news/13/583.html"},
        {"date": "2023-09-12", "title": "滨州市昱泰检测有限公司声明", "url": "https://www.befar.com/news/13/301.html"}
    ],
    "page5": [
        {"date": "2023-08-18", "title": "大晟芯材料科技（山东）有限公司电子化学品系列化项目(一期) 首次环境影响评价信息公开", "url": "https://www.befar.com/news/13/294.html"},
        {"date": "2023-07-01", "title": "滨州蓝润环境资源有限公司60万吨/年工业废盐资源化利用项目社会稳定风险评估公示", "url": "https://www.befar.com/news/13/628.html"}
    ]
}

ITEMS = []
for page_data in LIST_DATA.values():
    ITEMS.extend(page_data)

def parse_date(s):
    s = s.strip()[:10].replace('.', '-')
    try:
        return datetime.strptime(s, '%Y-%m-%d').date()
    except:
        return None

def date_rank(d):
    return int(d.strftime('%Y%m%d'))

def safe_insert(c, conn, values, max_retries=10):
    for attempt in range(max_retries):
        try:
            c.execute("""INSERT OR IGNORE INTO gov_raw 
                (site_name, title, page_url, source_url, publish_date, date_rank, summary, status, category, content, visits, tags)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, '')""", values)
            return c.rowcount > 0, None
        except sqlite3.OperationalError as e:
            if 'locked' in str(e):
                time.sleep(1)
                continue
            return False, str(e)
    return False, "DB locked"

def main():
    global COUNT_NEW, COUNT_SKIP, COUNT_ERR
    
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    
    print("[%s] 共 %d 条列表数据" % (SITE_NAME, len(ITEMS)))
    
    for item in ITEMS:
        item_date = parse_date(item['date'])
        if not item_date or item_date < THRESHOLD:
            COUNT_SKIP += 1
            continue
        
        url = item['url']
        
        c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
        if c.fetchone():
            COUNT_SKIP += 1
            continue
        
        # Check if it's a PDF (no detail page to fetch)
        if url.endswith('.pdf'):
            content_html = '<p><a href="%s" target="_blank">查看PDF附件</a></p>' % url
            title = item['title']
            date_str = item_date.strftime('%Y-%m-%d')
            dr = date_rank(item_date)
            summary = item['title'][:200]
            values = (SITE_NAME, title, url, url, date_str, dr, summary, 'published', CATEGORY, content_html)
            ok, e = safe_insert(c, conn, values)
            if ok:
                COUNT_NEW += 1
                print("  + [PDF] %s | %s" % (title[:50], date_str))
            continue
        
        # Try to fetch detail from server (will fail if WAF blocks)
        # For now just store with available info
        title = item['title']
        date_str = item_date.strftime('%Y-%m-%d')
        dr = date_rank(item_date)
        content_html = '<p>详情页需要通过浏览器访问获取</p>'
        summary = item['title'][:200]
        
        values = (SITE_NAME, title, url, url, date_str, dr, summary, 'published', CATEGORY, content_html)
        ok, e = safe_insert(c, conn, values)
        if ok:
            COUNT_NEW += 1
            print("  + %s | %s" % (title[:50], date_str))
        elif e:
            COUNT_ERR += 1
        
        time.sleep(0.1)
    
    conn.commit()
    conn.close()
    
    print("\n===== %s 完成 =====" % SITE_NAME)
    print("新增: %d" % COUNT_NEW)
    print("跳过: %d" % COUNT_SKIP)
    print("错误: %d" % COUNT_ERR)

if __name__ == '__main__':
    main()