#!/usr/bin/env python3
"""Fix content[:500] to not break in middle of HTML tags"""
FILE = '/root/gov_crawler/crawl_yuanjiang_eia.py'

with open(FILE, 'r') as f:
    c = f.read()

# Add a helper function after import section
import_marker = "import requests\n"
helper_func = '''
def _safe_summary(text, max_len=500):
    """Truncate text safely without breaking HTML tags"""
    if len(text) <= max_len:
        return text
    # Truncate, then cut at last complete tag boundary
    truncated = text[:max_len]
    # Count unclosed < tags
    opens = truncated.count('<')
    closes = truncated.count('</')
    # If < > pair count mismatch, might be cut mid-tag - rsplit at last safe <
    if '<' in truncated:
        last_open = truncated.rfind('<')
        last_close = truncated.rfind('>')
        if last_open > last_close:
            # Cut in middle of an opening tag
            truncated = truncated[:last_open]
        elif truncated.count('<') % 2 != 0:
            # Odd number of '<' - might be a self-closing tag or cut mid-entity
            pass
    return truncated

'''

c = c.replace(import_marker, import_marker + helper_func)

# Replace content[:500] with _safe_summary(content)
c = c.replace('content[:500]', '_safe_summary(content)')

with open(FILE, 'w') as f:
    f.write(c)

# Verify syntax
try:
    compile(c, FILE, 'exec')
    print("Syntax: OK")
    print("Written: OK")
except SyntaxError as e:
    print(f"Syntax error: {e}")