', html, re.DOTALL)
if not content_match:
content_match = re.search(r'
(.*?)本文网址', html, re.DOTALL)
if content_match:
content_html = content_match.group(1).strip()
# 补全图片src
content_html = re.sub(r'src="(?!http)', f'src="{BASE_URL}', content_html)
content_html = re.sub(r'href="(?!http)', f'href="{BASE_URL}', content_html)
result['content'] = content_html
else:
result['content'] = ''
return result
def get_db():
"""获取数据库连接"""
conn = sqlite3.connect(DB_PATH, timeout=60)
c = conn.cursor()
c.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT,
content TEXT,
publish_date TEXT,
url TEXT UNIQUE,
site_name TEXT,
source TEXT,
attachments TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)''')
conn.commit()
return conn
def save_item(conn, item):
"""保存单条数据"""
c = conn.cursor()
try:
c.execute('''
INSERT OR IGNORE INTO gov_raw (title, content, publish_date, url, site_name, source)
VALUES (?, ?, ?, ?, ?, ?)
''', (
item['title'],
item.get('content', ''),
item.get('publish_date', ''),
item['url'],
SITE_NAME,
f'{SITE_NAME} - 环评公参公示',
))
if c.rowcount > 0:
print(f" ✅ 新增: {item['title'][:40]}...")
else:
print(f" ⏭️ 已存在: ID={item.get('id', '?')}")
conn.commit()
except Exception as e:
print(f" ❌ 写入失败: {e}")
def run(max_pages=None):
"""主运行函数"""
print(f"={'='*50}")
print(f" 🌐 {SITE_NAME} ({SITE_DOMAIN}) 爬虫")
print(f"={'='*50}")
conn = get_db()
# 先获取第一页，确认总页数
page = 1
items, total_pages, total = fetch_list_page(page)
if not items:
print("⚠️ 无法获取列表数据，退出")
conn.close()
return
print(f"\n📊 总计 {total} 条, 共 {total_pages} 页")
if max_pages and max_pages < total_pages:
print(f" ⚠️ 限制只爬 {max_pages} 页")
total_pages = max_pages
# 处理第一页
print(f"\n📄 第1页 ({len(items)} 条)")
for item in items:
print(f" ▶️ {item['title'][:50]}...")
detail = fetch_detail(item['url'])
if detail:
item['title'] = detail.get('title', item['title'])
item['publish_date'] = detail.get('publish_date', item['publish_date'])
item['content'] = detail.get('content', '')
save_item(conn, item)
time.sleep(0.5)
# 处理后续页
page = 2
pbar_len = 50
while page <= total_pages:
items, _, _ = fetch_list_page(page)
if not items:
break
print(f"\n📄 第{page}页 ({len(items)} 条)")
for idx, item in enumerate(items):
print(f" ▶️ [{idx+1}/{len(items)}] {item['title'][:50]}...")
detail = fetch_detail(item['url'])
if detail:
item['title'] = detail.get('title', item['title'])
item['publish_date'] = detail.get('publish_date', item['publish_date'])
item['content'] = detail.get('content', '')
save_item(conn, item)
time.sleep(0.5)
page += 1
conn.close()
# 统计
conn2 = sqlite3.connect(DB_PATH, timeout=60)
count = conn2.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
conn2.close()
print(f"\n={'='*50}")
print(f" ✅ 完成! {SITE_NAME} 共导入 {count} 条数据到 {DB_PATH}")
print(f"={'='*50}")
if __name__ == '__main__':
max_pages = None
if len(sys.argv) > 1:
max_pages = int(sys.argv[1])
run(max_pages)