#!/usr/bin/env python3
"""Deeper analysis of col4795 and other columns."""
import requests, re, html as html_mod

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

# col4795 - check for embedded data and search form
r = requests.get("https://www.jxlc.gov.cn/col/col4795/index.html?number=D00004D00004D00007",
                 timeout=30, headers=headers, verify=False)
html = r.text

print("=== col4795: looking for embedded articles ===")
# Look for art/ URLs with col4795
arts = re.findall(r'art/\d+/\d+/art_4795_\d+\.html', html)
print(f"art_4795 links in HTML: {len(arts)}")
if arts:
    for a in arts[:5]:
        print(f"  {a}")

# Look for <li> with dates
lis = re.findall(r'<li[^>]*>.*?\d{4}-\d{2}-\d{2}.*?</li>', html, re.DOTALL)
print(f"List items with dates: {len(lis)}")
if lis:
    for li in lis[:3]:
        print(f"  {li[:200]}")

# Look for the "more" link pattern
more = re.findall(r'<a[^>]*more[^>]*>', html, re.I)
print(f"More links: {len(more)}")

# Look for jpage data (pagination elements)
jpage = re.findall(r'jpage\([^)]+\)', html)
print(f"jpage calls: {len(jpage)}")
for j in jpage[:5]:
    print(f"  {j[:150]}")

# Look for xxgk/tree iframe
trees = re.findall(r'xxgk/tree\.jsp[^"]*', html)
print(f"\nTree frames: {len(trees)}")
for t in trees:
    print(f"  {t}")

# Check if split form (infotype + vc_title + vc_number) is used
search_forms = re.findall(r'<form[^>]*search[^>]*>.*?</form>', html, re.DOTALL | re.I)
print(f"\nSearch forms: {len(search_forms)}")

# Try different infotypeId patterns
print("\n=== Trying alternative API calls ===")
api_url = "https://www.jxlc.gov.cn/module/xxgk/search.jsp"
for infotypeId in ["D00004D00004", "D00004", "", "D00004D00004D00007"]:
    data = f"infotypeId={infotypeId}&jdid=5&area=&divid=div1432&vc_title=&vc_number=&currpage=1&vc_filenumber=&vc_all=&texttype=&fbtime="
    r2 = requests.post(api_url, data=data, 
        headers={"User-Agent": headers["User-Agent"], "X-Requested-With": "XMLHttpRequest",
                 "Referer": "https://www.jxlc.gov.cn/col/col4795/index.html?number=D00004D00004D00007",
                 "Content-Type": "application/x-www-form-urlencoded"},
        timeout=30, verify=False)
    r2.encoding = "utf-8"
    html2 = r2.text
    items = re.findall(r'<li>(.*?)</li>', html2, re.DOTALL)
    nTotal = re.search(r'nTotalCount.*?value="(\d+)"', html2)
    total = nTotal.group(1) if nTotal else "?"
    print(f"  infotypeId={infotypeId:30s} items={len(items)} total={total}")

# Now check col1384 more carefully
print("\n=== col1384: looking for embedded articles ===")
r3 = requests.get("http://www.jxlc.gov.cn/col/col1384/index.html",
                  timeout=30, headers=headers, verify=False)
html3 = r3.text
arts1384 = re.findall(r'content/col1384|art_1384|col1384.*?\.html', html3)
print(f"art_1384 refs: {len(arts1384)}")
for a in arts1384[:5]:
    print(f"  {a}")

# Look for the jpage/ajax loading area
jpage1384 = re.findall(r'param_(\d+)\s*=', html3)
print(f"param IDs: {jpage1384}")
for pid in jpage1384[:5]:
    # find the full param object
    m = re.search(rf'param_{pid}\s*=\s*{{[^}}]+}}', html3)
    if m:
        print(f"  param_{pid}: {m.group()[:200]}")

# Look for list areas
ul_areas = re.findall(r'<ul[^>]*class="[^"]*list[^"]*"[^>]*>.*?</ul>', html3, re.DOTALL | re.I)
print(f"List ULs: {len(ul_areas)}")
# Check col1432 search differently
print("\n=== col1432: alternative search ===")
data1432 = "infotypeId=D00004&jdid=5&area=&divid=div1432&vc_title=&vc_number=&currpage=1"
r4 = requests.post(api_url, data=data1432, 
    headers={"User-Agent": headers["User-Agent"], "X-Requested-With": "XMLHttpRequest",
             "Referer": "https://www.jxlc.gov.cn/col/col1432/",
             "Content-Type": "application/x-www-form-urlencoded"},
    timeout=30, verify=False)
r4.encoding = "utf-8"
html4 = r4.text
items4 = re.findall(r'<li>(.*?)</li>', html4, re.DOTALL)
nTotal4 = re.search(r'nTotalCount.*?value="(\d+)"', html4)
total4 = nTotal4.group(1) if nTotal4 else "?"
print(f"  infotypeId=D00004 items={len(items4)} total={total4}")