0:
start = r.text.find('>', idx) + 1
depth = 1
pos = start
while pos < len(r.text) and depth > 0:
if r.text[pos:pos+4] == '
':
depth -= 1
pos += 6
else:
pos += 1
content = r.text[start:pos-6].strip()
if content:
break
if not content:
m = re.search(r']*>(.*?)
', r.text, re.DOTALL)
if m:
content = m.group(1).strip()
return title, content
def run(col, incremental=False):
site = get_site(col)
print(f" Site: {site['name']}")
records = []
seen = set()
max_pages = 1 if incremental else 50
for pg in range(max_pages):
items = fetch_urls_from_page(site, pg)
if not items:
break
print(f" Page {pg+1}: {len(items)} items")
for it in items:
if it['pub_date'] < CUTOFF:
continue
if it['url'] in seen:
continue
seen.add(it['url'])
title, content = fetch_detail(it['url'])
if not title or not content.strip():
continue
records.append({
'title': title,
'url': it['url'],
'pub_date': it['pub_date'],
'site_name': site['name'],
'content': content,
'summary': '',
})
print(f" Total: {len(records)} items")
valid = [r for r in records if r['content'].strip()]
if valid:
push_to_searchdb(valid, "changshu_" + col)
return len(valid)
if __name__ == '__main__':
col = 'gongshi'
for a in sys.argv[1:]:
if a.startswith('--col='):
col = a.split('=', 1)[1]
inc = '--incremental' in sys.argv
cnt = run(col, incremental=inc)
print(f"Done: {cnt} records")