]*class="text"[^>]*>(.*?)\s*(?:)?', html, re.S)
if text_m:
content_html = text_m.group(1)
# 去除表格内的和wzy_titlebox信息表，避免重复
content_html_no_tables = re.sub(r']*wzy_titlebox[^>]*>.*?', '', content_html, flags=re.S)
content_html_no_tables = re.sub(r']*>.*?', '', content_html_no_tables, flags=re.S)
# 智能解析：保留为[text](url)，![]()
为
paragraphs = []
for p in re.findall(r']*>(.*?)
', content_html_no_tables, re.S):
# 处理标签
p_text = re.sub(r']*href="([^"]*)"[^>]*>([^<]*)', r'[\2](\1)', p)
# 处理![]()
标签
p_text = re.sub(r']*src="([^"]*)"[^>]*>', r'', p_text)
# 去掉剩余标签
text = re.sub(r'<[^>]+>', '', p_text).strip()
text = re.sub(r'\s+', ' ', text)
text = re.sub(r' ', ' ', text).strip()
if text:
paragraphs.append(text)
if paragraphs:
content = '\n\n'.join(paragraphs)
else:
content = re.sub(r'<[^>]+>', '', content_html).strip()
content = re.sub(r'\s+', ' ', content).strip()
# 表格 - 只从正文区提取（跳过信息表），保留 HTML
tables = []
scope_html = text_m.group(1) if text_m else html
for tm in re.finditer(r']*>.*?', scope_html, re.S):
whole_table = tm.group(0)
if 'wzy_titlebox' in whole_table:
continue
tables.append(whole_table)
if tables:
content += '\n\n' + '\n\n'.join(tables)
# 附件 - 只从正文区提取
attachments = []
for m in re.finditer(r']*href="([^"]*)"[^>]*>([^<]*)', scope_html, re.S):
href = m.group(1).strip()
text = m.group(2).strip()
if '/document/download' in href or re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', href, re.I):
if not href.startswith('http'):
href = urllib.parse.urljoin(BASE_URL, href)
attachments.append({
'name': text or href.split('/')[-1],
'url': href
})
# 在正文末尾添加附件链接
content += f'\n\n附件：{text}'
return {
'title': title,
'publish_date': publish_date,
'content': content.strip(),
'attachments': json.dumps(attachments, ensure_ascii=False) if attachments else '[]'
}, None
def crawl():
opener = init_session()
print(f"[{SITE_NAME}] Session initialized")
all_items = []
total_api_count = 0
for page in range(1, MAX_PAGES + 1):
try:
items, total_count, got = fetch_list_page(opener, page)
total_api_count = total_count
print(f" Page {page}/{MAX_PAGES}: got {got} items (total in system: {total_count})")
all_items.extend(items)
time.sleep(0.3)
except Exception as e:
print(f" Page {page}: ERROR - {e}")
break
print(f"\nTotal items from list: {len(all_items)}")
# 过滤环评审批（col44281）文章
hj_items = [it for it in all_items if '/col/col44281/' in it['url']]
print(f"环评审批 items: {len(hj_items)}")
saved = 0
errors = 0
for idx, item in enumerate(hj_items):
try:
detail, err = fetch_detail(opener, item['url'])
if err:
print(f" [{idx+1}/{len(hj_items)}] ERROR fetching {item['title'][:30]}: {err}")
errors += 1
time.sleep(0.5)
continue
import sqlite3
conn = sqlite3.connect(DB_PATH, timeout=60)
c = conn.cursor()
content_text = detail['content'][:50000] if detail['content'] else ''
summary = re.sub(r'\s+', ' ', content_text[:200]).strip() or detail['title']
c.execute('''INSERT OR IGNORE INTO gov_raw
(page_url, title, publish_date, site_name, group_name, summary, content, attachments, source_url)
VALUES (?,?,?,?,?,?,?,?,?)''',
(item['url'], detail['title'], detail['publish_date'],
SITE_NAME, GROUP, summary, content_text,
detail.get('attachments', '[]'), item['url']))
n = c.rowcount
conn.commit()
conn.close()
saved += 1
if (idx + 1) % 10 == 0:
print(f" [{idx+1}/{len(hj_items)}] saved {saved}, errors {errors}")
time.sleep(0.3)
except Exception as e:
print(f" [{idx+1}/{len(hj_items)}] ERROR saving {item['title'][:30]}: {e}")
errors += 1
time.sleep(0.5)
print(f"\n{'='*50}")
print(f"[{SITE_NAME}] Done! Saved: {saved}, Errors: {errors}")
print(f"{'='*50}")
if __name__ == '__main__':
crawl()