from __future__ import annotations import json import logging import os import re from collections import OrderedDict, defaultdict import hashlib from datetime import datetime, date from decimal import Decimal from io import BytesIO import threading import time from typing import Any, Dict, Iterable, List, Optional, Set, Tuple import uuid import pg8000.dbapi as pg from openpyxl import load_workbook from openpyxl.utils.exceptions import InvalidFileException # Configure logger logger = logging.getLogger(__name__) if not logger.handlers: handler = logging.StreamHandler() handler.setFormatter(logging.Formatter("[%(levelname)s] %(name)s: %(message)s")) logger.addHandler(handler) logger.setLevel(logging.INFO) logger.propagate = False # Separate configuration so legacy fs_law_risk integration keeps using PG_* LIC_DEFAULT_DB = "licensing_risks" _UNSET = object() ARTICLE_HEADING_RE = re.compile(r"(?m)^(第[一二三四五六七八九十百零0-9]+条)") ARTICLE_TOKEN_RE = re.compile(r"(? str: """Render Chinese legal excerpts as Markdown-friendly text.""" if not summary: return "" text = summary.replace("\r\n", "\n").strip() if not text: return "" text = ARTICLE_HEADING_RE.sub(lambda m: f"**{m.group(1)}**", text) text = CN_ENUM_INLINE_RE.sub(lambda m: f"{m.group(1)}\n- （{m.group(2)}） ", text) text = CN_ENUM_LINE_RE.sub(lambda m: f"- （{m.group(1)}） ", text) text = ARABIC_ENUM_INLINE_RE.sub(lambda m: f"{m.group(1)}\n {m.group(2)}", text) text = ARABIC_ENUM_LINE_RE.sub(lambda m: f" {m.group(1)}.", text) text = NESTED_ENUM_INLINE_RE.sub(lambda m: f"{m.group(1)}\n - （{m.group(2)}）", text) text = NESTED_ENUM_LINE_RE.sub(lambda m: f" - （{m.group(1)}）", text) text = ARTICLE_TOKEN_RE.sub(lambda m: f"**{m.group(1)}**", text) text = ARTICLE_NEWLINE_RE.sub(lambda m: f"\n{m.group(1)}", text) text = COLON_NEWLINE_RE.sub("：\n", text) text = EXTRA_NEWLINES_RE.sub("\n\n", text) text = TRAILING_SPACE_RE.sub("\n", text) text = re.sub(r"\n\s+\n", "\n\n", text) return text.strip() def _clean_text(value: Any) -> str: """Return a stripped string representation for Excel parsing.""" if value is None: return "" if isinstance(value, str): return value.strip() return str(value).strip() def _to_isoformat(value: Any) -> Optional[str]: if value is None: return None if isinstance(value, datetime): return value.isoformat() if isinstance(value, date): return datetime.combine(value, datetime.min.time()).isoformat() return str(value) def _to_optional_str(value: Any) -> Optional[str]: if value is None: return None if isinstance(value, (uuid.UUID,)): return str(value) return str(value) def _normalize_permit_token(value: Any) -> str: """Normalize permit names for dictionary lookups (case/whitespace insensitive).""" text = _clean_text(value) if not text: return "" return text.lower() def _permit_name_aliases(value: Any) -> Set[str]: """Return canonical + token aliases for a permit name.""" aliases: Set[str] = set() canonical = _clean_text(value) token = _normalize_permit_token(value) if canonical: aliases.add(canonical) if token and token != canonical: aliases.add(token) return aliases def _normalize_theme_binding_value(value: Any) -> str: """Normalize theme binding payload values (including virtual markers).""" text = _clean_text(value) if not text: return "" lowered = text.lower() if lowered == ALL_THEMES_SENTINEL.lower(): return ALL_THEMES_SENTINEL if text == ALL_THEMES_DISPLAY_NAME: return ALL_THEMES_SENTINEL return text def _is_all_theme_marker(value: Any) -> bool: """Return True if the provided value represents the virtual 'all themes' marker.""" return _normalize_theme_binding_value(value) == ALL_THEMES_SENTINEL def _normalize_header_label(value: Any) -> str: """Normalise header labels by removing spaces and lowercasing.""" if value is None: return "" text = _clean_text(value) if not text: return "" compact = re.sub(r"[\s\u3000]+", "", text) return compact.lower() def _resolve_import_header(value: Any) -> List[str]: """Map an Excel header cell to one or more canonical field names.""" normalized = _normalize_header_label(value) if not normalized: return [] matches: List[str] = [] for canonical, candidates in _IMPORT_HEADER_ALIASES.items(): if normalized in candidates: matches.append(canonical) if not matches: for canonical, keywords in _IMPORT_HEADER_KEYWORDS: if any(keyword in normalized for keyword in keywords): matches.append(canonical) return matches def _score_import_header(canonical: str, cell_text: str, col_idx: int) -> float: """Heuristic score to choose the best header cell when duplicates exist.""" score = float(len(cell_text)) text = cell_text if canonical == "risk_content": if "内容" in text: score += 10 if "提示" in text: score += 4 if "风险" in text: score += 2 elif canonical == "permit_name": if "事项" in text: score += 6 if "名称" in text: score += 3 if "许可" in text: score += 2 elif canonical == "theme_names": if "主题" in text: score += 4 elif canonical == "permit_status": if "情况" in text or "状态" in text: score += 3 elif canonical == "summary": if "摘要" in text: score += 3 elif canonical == "document_no": if "文号" in text: score += 5 elif canonical == "remark": if "备注" in text: score += 3 elif canonical == "serial_number": if "序号" in text: score += 10 if "风险提示" == text: # Exact match for the weird template case score += 5 if "编号" in text: score += 3 score += col_idx * 0.1 return score def _split_multi_value(value: Any, *, allow_dunhao: bool = False) -> List[str]: """Split multi-value cells using common punctuation characters. 默认不把中文顿号（、）视作分隔符，以避免误拆“文化、旅游”等合法的 许可名称。对于确实需要用顿号分隔的字段（如主题、经营范围等），调用 方可以显式传入 allow_dunhao=True。 """ text = _clean_text(value) if not text: return [] pattern = TEXT_SPLIT_PATTERN_WITH_DUNHAO if allow_dunhao else TEXT_SPLIT_PATTERN return [item.strip() for item in pattern.split(text) if item.strip()] def _clean_empty(value: Any) -> Optional[str]: """Convert empty strings to None for database writes.""" text = _clean_text(value) return text or None def _canonicalize_region_label(label: str) -> str: """Return a canonical region label based on known keywords.""" text = _clean_text(label) if not text: return "" for canonical, keywords in _CANONICAL_REGION_KEYWORDS.items(): for keyword in keywords: if keyword and keyword in text: return canonical return text def _normalize_sheet_token(value: Any) -> str: """Normalize sheet/region identifiers for override maps.""" text = _clean_text(value) if not text: return "" canonical = _canonicalize_region_label(text) token_source = canonical or text return token_source.lower() def _normalize_import_row( raw_row: Dict[str, Any], sheet_name: str, sheet_defaults: Optional[Dict[str, Any]] = None, ) -> Optional[Dict[str, Any]]: """Convert a raw Excel row into canonical import structure.""" sheet_defaults = sheet_defaults or {} permit_name = _clean_text(raw_row.get("permit_name") or sheet_defaults.get("permit_name")) risk_content = _clean_text(raw_row.get("risk_content")) serial_number = _clean_text(raw_row.get("serial_number")) if not permit_name or not risk_content: return None row_index = raw_row.get("row_index") legal_basis = _clean_empty(raw_row.get("legal_basis")) document_no = _clean_empty(raw_row.get("document_no")) summary = _clean_empty(raw_row.get("summary")) remark = _clean_empty(raw_row.get("remark")) permit_status = _clean_empty(raw_row.get("permit_status") or sheet_defaults.get("permit_status")) filler_name = _clean_empty(raw_row.get("filler_name") or sheet_defaults.get("filler_name")) unit_name = _clean_empty(raw_row.get("unit_name") or sheet_defaults.get("unit_name")) responsible_contact = _clean_empty( raw_row.get("responsible_contact") or sheet_defaults.get("responsible_contact") ) jurisdiction_scope = _clean_empty( raw_row.get("jurisdiction_scope") or sheet_defaults.get("jurisdiction_scope") ) source_update_date = _clean_empty( raw_row.get("source_update_date") or sheet_defaults.get("source_update_date") ) # Validate date format YYYY-MM-DD if source_update_date: # Simple regex check for YYYY-MM-DD if not re.match(r"^\d{4}-\d{2}-\d{2}$", source_update_date): # Try to parse or warn? User said "Strictly YYYY-MM-DD" # We will store it as is, but maybe log a warning? # For now keep as text. pass subitem_names = _split_multi_value( raw_row.get("subitem_text") or sheet_defaults.get("subitem_text"), allow_dunhao=True ) return { "row_index": int(row_index) if isinstance(row_index, int) else row_index, "sheet_name": sheet_name, "permit_name": permit_name, "risk_content": risk_content, "serial_number": serial_number, "legal_basis": legal_basis, "document_no": document_no, "summary": summary, "remark": remark, "permit_status": permit_status, "responsible_contact": responsible_contact, "jurisdiction_scope": jurisdiction_scope, "subitem_names": subitem_names, "filler_name": filler_name, "unit_name": unit_name, "source_update_date": source_update_date, } def _parse_import_workbook(file_bytes: bytes, filename: str) -> Dict[str, Any]: """Parse the uploaded Excel workbook into structured sheet data.""" if not file_bytes: raise ValueError("上传文件为空") try: workbook = load_workbook(BytesIO(file_bytes), data_only=True) except InvalidFileException as exc: raise ValueError(f"Excel 文件格式无法识别：{exc}") from exc except Exception as exc: raise ValueError(f"Excel 解析失败：{exc}") from exc sheets: Dict[str, Dict[str, Any]] = {} total_rows = 0 for worksheet in workbook.worksheets: sheet_title = worksheet.title or "" sheet_name = _clean_text(sheet_title) or f"Sheet{len(sheets) + 1}" header_row_index: Optional[int] = None header_values: List[Any] = [] header_map: Dict[int, str] = {} resolved_by_name: Dict[str, Tuple[int, int, str]] = {} metadata_rows: List[Tuple[int, Dict[int, Tuple[str, str]], Tuple[Any, ...]]] = [] max_header_row = min(worksheet.max_row or 0, 120) or 120 for row_idx, row_values in enumerate( worksheet.iter_rows(min_row=1, max_row=max_header_row, values_only=True), start=1, ): if not row_values: continue if not any(_clean_text(cell) for cell in row_values): continue first_cell_text = _clean_text(row_values[0]) if len(row_values) else "" is_section_row = bool(first_cell_text) and bool(re.fullmatch(r"[一二三四五六七八九十百零]+", first_cell_text)) row_candidate_map: Dict[str, Tuple[int, str, float]] = {} for col_idx, header_cell in enumerate(row_values, start=1): cell_text = _clean_text(header_cell) if not cell_text: continue if len(cell_text) > 60: continue canonicals = _resolve_import_header(cell_text) if not canonicals: continue for canonical in canonicals: if ( is_section_row and col_idx >= 3 and canonical in {"permit_name", "permit_status", "responsible_contact", "subitem_summary"} ): continue if len(cell_text) > 35 and canonical in {"permit_name", "risk_content", "summary"}: continue score = _score_import_header(canonical, cell_text, col_idx) previous = row_candidate_map.get(canonical) if not previous or score > previous[2] or (score == previous[2] and col_idx < previous[0]): row_candidate_map[canonical] = (col_idx, cell_text, score) candidate_map: Dict[int, Tuple[str, str]] = { col_idx: (canonical, cell_text) for canonical, (col_idx, cell_text, _score) in row_candidate_map.items() } row_canonicals: Set[str] = {canonical for canonical, _ in candidate_map.values()} metadata_rows.append((row_idx, candidate_map, tuple(row_values))) if candidate_map: display_entries = [] for idx, (name, text) in sorted(candidate_map.items()): preview = text[:30] + ("…" if len(text) > 30 else "") display_entries.append(f"{idx}->{name}({preview})") logger.info( "[PERMIT-IMPORT] Sheet %s candidate row %d resolved: %s", sheet_name, row_idx, ", ".join(display_entries), ) # Update resolved columns, allowing later rows to refine mapping. for col_idx, (canonical, cell_text) in candidate_map.items(): previous = resolved_by_name.get(canonical) if previous and previous[0] != col_idx: header_map.pop(previous[0], None) resolved_by_name[canonical] = (col_idx, row_idx, cell_text) header_map[col_idx] = canonical if ( "risk_content" in row_canonicals and ( {"legal_basis", "document_no", "summary"}.intersection(row_canonicals) or ("permit_name" in row_canonicals) or ("permit_name" in resolved_by_name) or (any(k in row_canonicals for k in ["permit_status", "filler_name", "unit_name", "source_update_date"])) ) ): header_row_index = row_idx header_values = list(row_values) break if header_row_index is None or not header_map: logger.warning( "[PERMIT-IMPORT] Sheet %s skipped: 未找到包含许可名称与风险内容的表头", sheet_name, ) continue sheet_defaults: Dict[str, Any] = {} for row_idx, candidate_map, row_values in metadata_rows: if row_idx >= header_row_index: continue for col_idx, (canonical, _label_text) in candidate_map.items(): value = None # Prefer the cell to the right of the label as the value. if col_idx < len(row_values): value = row_values[col_idx] if value is None and (col_idx + 1) < len(row_values): value = row_values[col_idx + 1] cleaned = _clean_text(value) if cleaned and canonical not in sheet_defaults: sheet_defaults[canonical] = value # Keep only headers that were identified on or before the selected header row. effective_header_map: Dict[int, str] = {} for canonical, (col_idx, row_idx, _cell_text) in resolved_by_name.items(): if row_idx == header_row_index: effective_header_map[col_idx] = canonical header_map = dict(sorted(effective_header_map.items())) has_risk_column = "risk_content" in header_map.values() has_permit_column = "permit_name" in header_map.values() has_permit_default = bool(_clean_text(sheet_defaults.get("permit_name"))) if not has_risk_column or (not has_permit_column and not has_permit_default): logger.warning( "[PERMIT-IMPORT] Sheet %s skipped: 表头缺少许可名称或风险内容列（行 %d） | header_map=%s | permit_default=%s", sheet_name, header_row_index, ", ".join(f"{idx}:{name}" for idx, name in sorted(header_map.items())), _clean_text(sheet_defaults.get("permit_name") or ""), ) continue logger.info( "[PERMIT-IMPORT] Sheet %s header row %d candidates: %s", sheet_name, header_row_index, ", ".join(_clean_text(cell) or "<空>" for cell in header_values), ) logger.info( "[PERMIT-IMPORT] Sheet %s resolved headers: %s", sheet_name, ", ".join(f"{idx}->{name}" for idx, name in sorted(header_map.items())), ) normalized_rows: List[Dict[str, Any]] = [] for row_idx, row_values in enumerate( worksheet.iter_rows(min_row=header_row_index + 1, values_only=True), start=header_row_index + 1, ): raw_row: Dict[str, Any] = {"row_index": row_idx} has_data = False for col_idx, cell_value in enumerate(row_values, start=1): canonical = header_map.get(col_idx) if not canonical: continue if cell_value is None or (isinstance(cell_value, str) and not cell_value.strip()): continue has_data = True raw_row[canonical] = cell_value if not has_data: logger.debug( "[PERMIT-IMPORT] Sheet %s row %d ignored: 空行", sheet_name, row_idx, ) continue normalized = _normalize_import_row(raw_row, sheet_name, sheet_defaults) if normalized: normalized_rows.append(normalized) else: logger.debug( "[PERMIT-IMPORT] Sheet %s row %d ignored: 缺少许可名称或风险内容", sheet_name, row_idx, ) if not normalized_rows: logger.warning( "[PERMIT-IMPORT] Sheet %s skipped: 没有有效数据行", sheet_name, ) continue sheets[sheet_name] = { "sheet_name": sheet_name, "rows": normalized_rows, } total_rows += len(normalized_rows) if not sheets: logger.error( "[PERMIT-IMPORT] Workbook %s has no importable sheets (headers missing or rows空)", filename, ) raise ValueError("Excel 中未找到可导入的数据") return { "filename": os.path.basename(filename or ""), "sheets": sheets, "total_rows": total_rows, } def _cleanup_expired_import_sessions() -> None: """Remove expired import sessions to avoid unbounded growth.""" now = time.time() expired: List[str] = [] with _PERMIT_IMPORT_LOCK: for session_id, payload in list(_PERMIT_IMPORT_SESSIONS.items()): created_at = payload.get("created_at", now) if now - created_at > PERMIT_IMPORT_TTL_SECONDS: expired.append(session_id) for session_id in expired: _PERMIT_IMPORT_SESSIONS.pop(session_id, None) def _resolve_themes_for_permit(conn: pg.Connection, permit_name: str) -> List[str]: """Resolve theme names from permit_theme_rules.""" if not permit_name: return [] cur = conn.cursor() # Find theme_ids for this permit (exact match on name for now) sql = """ SELECT t.name FROM permit_theme_rules ptr JOIN themes t ON t.id = ptr.theme_id WHERE ptr.permit_name = %s """ cur.execute(sql, [permit_name]) return [row[0] for row in cur.fetchall()] def start_permit_import_session( file_bytes: bytes, filename: str, *, content_type: Optional[str] = None, uploaded_by: Optional[str] = None, uploader_department_id: Optional[str] = None, bound_department_id: Optional[str] = None, binding_mode: str = "auto", ) -> Dict[str, Any]: """Parse the uploaded workbook and create an import session.""" if not file_bytes: raise ValueError("上传的文件为空") if len(file_bytes) > MAX_PERMIT_FILE_SIZE_BYTES: raise ValueError("上传的文件超过 500KB 限制，请拆分或压缩内容后重试") parsed = _parse_import_workbook(file_bytes, filename) workbook_filename = parsed.get("filename") or os.path.basename(filename or "") raw_sheet_payloads: Dict[str, Dict[str, Any]] = parsed.get("sheets", {}) canonical_sheets: Dict[str, Dict[str, Any]] = {} sheet_name_mapping: Dict[str, str] = {} for original_name, sheet_data in raw_sheet_payloads.items(): canonical_name = _canonicalize_region_label(original_name) or original_name sheet_name_mapping[original_name] = canonical_name bucket = canonical_sheets.setdefault( canonical_name, { "canonical_name": canonical_name, "original_names": [], "rows": [], }, ) bucket["original_names"].append(original_name) for row in sheet_data.get("rows", []): row_copy = dict(row) row_copy["sheet_name"] = canonical_name bucket["rows"].append(row_copy) sheet_payloads: Dict[str, Dict[str, Any]] = canonical_sheets sheet_names = list(sheet_payloads.keys()) sheet_tokens = sorted({name.lower() for name in sheet_names if name}) logger.info( "[PERMIT-IMPORT] Workbook %s parsed sheets: %s", workbook_filename or filename, ", ".join(sheet_names), ) for original_name, canonical_name in sheet_name_mapping.items(): if original_name != canonical_name: logger.info( "[PERMIT-IMPORT] Sheet alias: %s -> %s", original_name, canonical_name, ) region_lookup: Dict[str, Dict[str, Any]] = {} permit_lookup: Dict[str, Dict[str, str]] = {} # Pre-resolve department region if bound_department_id is provided dept_region_info: Optional[Dict[str, Any]] = None binding_mode_lower = (binding_mode or "auto").lower() with _lic_pg_conn() as conn: cur = conn.cursor() if bound_department_id: try: dept_info = _fetch_service_department(cur, str(bound_department_id)) if dept_info and dept_info.get("region_id"): dept_region_info = { "id": str(dept_info["region_id"]), "uuid": dept_info["region_id"], "name": str(dept_info.get("region_name") or ""), } logger.info( "[PERMIT-IMPORT] Resolved department region: %s (%s)", dept_region_info["name"], dept_region_info["id"], ) except Exception as exc: logger.warning("[PERMIT-IMPORT] Failed to fetch department region: %s", exc) if sheet_tokens: cur.execute( """ SELECT id, name FROM regions WHERE LOWER(name) = ANY(%s) """, (sheet_tokens,), ) for region_id, region_name in cur.fetchall(): key = str(region_name).lower() region_lookup[key] = { "id": str(region_id), "uuid": region_id, "name": str(region_name), } if region_lookup: logger.info( "[PERMIT-IMPORT] Resolved regions from sheet names: %s", ", ".join(f"{entry['name']}({entry['id']})" for entry in region_lookup.values()), ) # Build list of all active regions for this session for permit lookup active_region_uuids = [entry["uuid"] for entry in region_lookup.values()] if dept_region_info and dept_region_info["uuid"] not in active_region_uuids: active_region_uuids.append(dept_region_info["uuid"]) if active_region_uuids: cur.execute( """ SELECT DISTINCT rtp.region_id, p.id, p.name FROM region_theme_permits rtp JOIN permits p ON p.id = rtp.permit_id WHERE rtp.region_id = ANY(%s) """, (active_region_uuids,), ) for region_id, permit_id, permit_name in cur.fetchall(): rid = str(region_id) canonical_name = _clean_text(permit_name) normalized_permit = _normalize_permit_token(permit_name) permit_id_str = str(permit_id) if canonical_name: permit_lookup.setdefault(rid, {})[canonical_name] = permit_id_str if normalized_permit: permit_lookup.setdefault(rid, {})[normalized_permit] = permit_id_str sheet_summaries: List[Dict[str, Any]] = [] session_sheets: Dict[str, Dict[str, Any]] = {} for sheet_name, sheet_data in sheet_payloads.items(): rows: List[Dict[str, Any]] = sheet_data.get("rows", []) permit_groups: Dict[str, List[Dict[str, Any]]] = defaultdict(list) for row in rows: permit_groups[row["permit_name"]].append(row) region_key = sheet_name.lower() region_info = region_lookup.get(region_key) # Override logic for department-based binding # If binding_mode is 'department', force the department's region if we have one. if binding_mode_lower == "department" and dept_region_info: region_info = dept_region_info region_id_str = region_info["id"] if region_info else None existing_permits = permit_lookup.get(region_id_str or "", {}) duplicate_permits = sorted( [ name for name in permit_groups.keys() if _normalize_permit_token(name) in existing_permits ] ) new_permits = sorted( [ name for name in permit_groups.keys() if _normalize_permit_token(name) not in existing_permits ] ) logger.info( "[PERMIT-IMPORT] Sheet %s summary: permits=%d, duplicates=%d, new=%d, region=%s (%s)", sheet_name, len(permit_groups), len(duplicate_permits), len(new_permits), region_info["name"] if region_info else "None", region_id_str, ) session_sheets[sheet_name] = { "sheet_name": sheet_name, "region_name": region_info["name"] if region_info else sheet_name, "region_id": region_id_str, "rows": rows, "permit_groups": {name: group for name, group in permit_groups.items()}, "existing_permits": dict(existing_permits), "duplicate_permits": duplicate_permits, "new_permits": new_permits, "original_sheet_names": sheet_data.get("original_names", [sheet_name]), } sheet_summaries.append( { "sheet_name": sheet_name, "region_name": region_info["name"] if region_info else sheet_name, "region_id": region_id_str or "", "row_count": len(rows), "permit_count": len(permit_groups), "risk_count": len(rows), "duplicate_permits": duplicate_permits, "new_permits": new_permits, "missing_region": region_info is None, "original_sheet_names": sheet_data.get("original_names", [sheet_name]), } ) _cleanup_expired_import_sessions() session_id = str(uuid.uuid4()) session_payload = { "id": session_id, "filename": workbook_filename, "created_at": time.time(), "sheets": session_sheets, "file_bytes": bytes(file_bytes), "file_size": len(file_bytes), "content_type": content_type or "application/octet-stream", "uploaded_by": uploaded_by, "uploader_department_id": uploader_department_id, "bound_department_id": bound_department_id, "binding_mode": (binding_mode or "auto").lower(), } with _PERMIT_IMPORT_LOCK: _PERMIT_IMPORT_SESSIONS[session_id] = session_payload return { "session_id": session_id, "filename": workbook_filename, "sheet_summaries": sheet_summaries, "total_rows": parsed.get("total_rows", 0), "expires_in": PERMIT_IMPORT_TTL_SECONDS, "file_size": len(file_bytes), "content_type": content_type or "application/octet-stream", } def _ensure_region(conn: pg.Connection, region_name: str) -> str: name = _clean_text(region_name) if not name: raise ValueError("地区名称不能为空") cur = conn.cursor() cur.execute( """ INSERT INTO regions (name) VALUES (%s) ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id """, (name,), ) region_id = cur.fetchone()[0] return str(region_id) def _ensure_theme(conn: pg.Connection, theme_name: str) -> str: name = _clean_text(theme_name) or "不涉及" cur = conn.cursor() cur.execute( """ INSERT INTO themes (name) VALUES (%s) ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id """, (name,), ) theme_id = cur.fetchone()[0] return str(theme_id) def _ensure_permit(conn: pg.Connection, permit_name: str) -> str: name = _clean_text(permit_name) if not name: raise ValueError("许可名称不能为空") cur = conn.cursor() cur.execute( """ INSERT INTO permits (name) VALUES (%s) ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id """, (name,), ) permit_id = cur.fetchone()[0] return str(permit_id) def _ensure_business_scope(conn: pg.Connection, description: str) -> Optional[str]: text = _clean_text(description) if not text: return None cur = conn.cursor() cur.execute( """ INSERT INTO business_scopes (description) VALUES (%s) ON CONFLICT (description) DO UPDATE SET description = EXCLUDED.description RETURNING id """, (text,), ) scope_id = cur.fetchone()[0] return str(scope_id) def _ensure_permit_subitem(conn: pg.Connection, description: str) -> Optional[str]: text = _clean_text(description) if not text: return None cur = conn.cursor() cur.execute( """ INSERT INTO permit_subitems (description) VALUES (%s) ON CONFLICT (description) DO UPDATE SET description = EXCLUDED.description RETURNING id """, (text,), ) subitem_id = cur.fetchone()[0] return str(subitem_id) def _ensure_risk( conn: pg.Connection, *, risk_content: str, legal_basis: Optional[str], document_no: Optional[str], summary: Optional[str], remark: Optional[str] = None, ) -> str: cur = conn.cursor() cur.execute( """ INSERT INTO risks (risk_content, legal_basis, document_no, summary, remark) VALUES (%s, %s, %s, %s, %s) ON CONFLICT (risk_content, legal_basis, document_no, summary, remark) DO UPDATE SET risk_content = EXCLUDED.risk_content RETURNING id """, (risk_content, legal_basis, document_no, summary, remark), ) risk_id = cur.fetchone()[0] return str(risk_id) def _fetch_region_permit_name_map(conn: pg.Connection, region_id: str) -> Dict[str, str]: cur = conn.cursor() cur.execute( """ SELECT DISTINCT p.name, p.id FROM region_theme_permits rtp JOIN permits p ON p.id = rtp.permit_id WHERE rtp.region_id = %s """, (region_id,), ) mapping: Dict[str, str] = {} for name, pid in cur.fetchall(): canonical = _clean_text(name) token = _normalize_permit_token(name) permit_id = str(pid) if canonical: mapping[canonical] = permit_id if token: mapping[token] = permit_id return mapping def _backup_permit_before_import( conn: pg.Connection, *, region_id: str, permit_id: str, region_name: str, permit_name: str, filename: str, sheet_name: str, edited_by: Optional[str], change_summary: Optional[str], ) -> Dict[str, Any]: cur = conn.cursor() cur.execute( """ SELECT risk_id FROM region_permit_risks WHERE region_id = %s AND permit_id = %s ORDER BY risk_id FOR UPDATE """, (region_id, permit_id), ) risk_ids = [str(risk_id) for (risk_id,) in cur.fetchall()] if not risk_ids: return {"snapshot_count": 0, "batch_id": ""} batch_id = str(uuid.uuid4()) base_summary = change_summary or ( f"Excel导入前快照（{filename} › {sheet_name} › {permit_name}）" ) for idx, risk_id in enumerate(risk_ids, start=1): detail_summary = f"{base_summary} - 风险 {idx}/{len(risk_ids)}" _create_snapshot_with_connection( conn, region_id, permit_id, risk_id, edited_by=edited_by, change_summary=detail_summary, batch_id=batch_id, ) logger.info( "[PERMIT-IMPORT] Captured %d snapshots before overwriting permit %s (%s) in region %s (%s)", len(risk_ids), permit_id, permit_name, region_id, region_name, ) return {"snapshot_count": len(risk_ids), "batch_id": batch_id} def _purge_region_permit_relations(conn: pg.Connection, region_id: str, permit_id: str) -> Dict[str, int]: cur = conn.cursor() delete_counts: Dict[str, int] = {} overrides_available = _PERMIT_THEME_OVERRIDE_SCHEMA_READY if overrides_available is None: try: _ensure_permit_theme_override_schema() overrides_available = True except Exception: overrides_available = False statements = [ ("region_permit_risks", "DELETE FROM region_permit_risks WHERE region_id = %s AND permit_id = %s"), ("region_permit_scopes", "DELETE FROM region_permit_scopes WHERE region_id = %s AND permit_id = %s"), ("region_permit_subitems", "DELETE FROM region_permit_subitems WHERE region_id = %s AND permit_id = %s"), ("region_permit_details", "DELETE FROM region_permit_details WHERE region_id = %s AND permit_id = %s"), ("region_theme_permits", "DELETE FROM region_theme_permits WHERE region_id = %s AND permit_id = %s"), ] if overrides_available: statements.append( ( "region_permit_theme_overrides", "DELETE FROM region_permit_theme_overrides WHERE region_id = %s AND permit_id = %s", ) ) for key, sql in statements: cur.execute(sql, (region_id, permit_id)) delete_counts[key] = int(cur.rowcount or 0) if _permit_sources_available(conn): cur.execute( """ DELETE FROM permit_sources WHERE region_id = %s AND permit_id = %s """, (region_id, permit_id), ) delete_counts["permit_sources"] = int(cur.rowcount or 0) return delete_counts def commit_permit_import_session( session_id: str, sheet_names: Iterable[str], *, overrides: Optional[Dict[str, Iterable[str]]] = None, edited_by: Optional[str] = None, change_summary: Optional[str] = None, theme_bindings: Optional[Dict[str, Dict[str, Iterable[str]]]] = None, ) -> Dict[str, Any]: if not session_id: raise ValueError("导入会话无效") selected_sheets: List[str] = [] seen_sheet_names: Set[str] = set() for sheet_name in sheet_names or []: name = _clean_text(sheet_name) if not name or name in seen_sheet_names: continue seen_sheet_names.add(name) selected_sheets.append(name) if not selected_sheets: raise ValueError("请选择至少一个Sheet进行导入") with _PERMIT_IMPORT_LOCK: session_payload = _PERMIT_IMPORT_SESSIONS.get(session_id) if not session_payload: raise ValueError("导入会话不存在或已过期，请重新上传Excel文件") session_file_bytes: Optional[bytes] = session_payload.get("file_bytes") session_file_content_type: str = session_payload.get("content_type") or "application/octet-stream" session_uploaded_by: Optional[str] = session_payload.get("uploaded_by") session_uploader_department: Optional[str] = session_payload.get("uploader_department_id") session_bound_department: Optional[str] = session_payload.get("bound_department_id") session_binding_mode: str = (session_payload.get("binding_mode") or "auto").lower() session_sheets: Dict[str, Dict[str, Any]] = session_payload.get("sheets", {}) workbook_filename = session_payload.get("filename") or "" overrides_map: Dict[str, Set[str]] = {} if overrides: for sheet_key, permit_names in overrides.items(): sheet_token = _normalize_sheet_token(sheet_key) or _clean_text(sheet_key) if not sheet_token: continue bucket = overrides_map.setdefault(sheet_token, set()) for raw_name in permit_names or []: aliases = _permit_name_aliases(raw_name) if aliases: bucket.update(aliases) theme_binding_map: Dict[str, Dict[str, List[str]]] = {} if theme_bindings: for sheet_key, permit_map in theme_bindings.items(): sheet_token = _normalize_sheet_token(sheet_key) or _clean_text(sheet_key) if not sheet_token: continue permit_binding = theme_binding_map.setdefault(sheet_token, {}) for permit_key, theme_values in (permit_map or {}).items(): canonical_permit = _clean_text(permit_key) permit_token = _normalize_permit_token(permit_key) if not canonical_permit and not permit_token: continue normalized_themes: List[str] = [] for raw_theme in theme_values or []: normalized = _normalize_theme_binding_value(raw_theme) if normalized: normalized_themes.append(normalized) if not normalized_themes: continue if canonical_permit: permit_binding[canonical_permit] = normalized_themes if permit_token and permit_token != canonical_permit: permit_binding[permit_token] = normalized_themes if overrides_map: override_debug = ", ".join(f"{sheet}:{len(names)}" for sheet, names in overrides_map.items()) logger.info("[PERMIT-IMPORT] Confirmed overrides => %s", override_debug) else: logger.info("[PERMIT-IMPORT] No override confirmations supplied") default_change_summary = change_summary or (f"Excel导入：{workbook_filename}" if workbook_filename else "Excel导入") effective_bound_department_id = None if session_binding_mode != "none": effective_bound_department_id = session_bound_department or session_uploader_department result: Dict[str, Any] = { "session_id": session_id, "filename": workbook_filename, "processed_sheets": [], "created_permits": [], "overwritten_permits": [], "skipped_permits": [], "snapshot_count": 0, "risk_count": 0, } logger.info( "[PERMIT-IMPORT] Committing session %s with %d sheet(s): %s", session_id, len(selected_sheets), ", ".join(selected_sheets), ) stored_file_meta: Optional[Dict[str, Any]] = None stored_file_id: Optional[str] = None region_theme_cache: Dict[str, List[Dict[str, str]]] = {} region_theme_cache: Dict[str, List[Dict[str, str]]] = {} print("DEBUG: Getting DB connection...") with _lic_pg_conn(autocommit=False) as conn: try: print("DEBUG: Connection acquired. Ensuring schemas...") _ensure_service_department_schema(conn) _ensure_permit_sources_table(conn) _ensure_permit_theme_override_schema(conn) print("DEBUG: Schemas ensured. Starting sheet processing...") if session_file_bytes: _ensure_permit_file_schema(conn) stored_file_meta = _insert_permit_file_record( conn, file_bytes=session_file_bytes, filename=workbook_filename or "许可导入.xlsx", content_type=session_file_content_type, uploaded_by=session_uploaded_by, ) stored_file_id = stored_file_meta.get("file_id") cur = conn.cursor() for sheet_name in selected_sheets: sheet_data = session_sheets.get(sheet_name) if not sheet_data: raise ValueError(f"导入会话中未找到名为 {sheet_name} 的Sheet") region_name = sheet_data.get("region_name") or sheet_name region_id = sheet_data.get("region_id") if region_id: # 确保地区仍然存在 existing_map = _fetch_region_permit_name_map(conn, region_id) sheet_data["existing_permits"] = existing_map else: region_id = _ensure_region(conn, region_name) sheet_data["region_id"] = region_id sheet_data["existing_permits"] = {} existing_permits = dict(sheet_data.get("existing_permits", {})) sheet_clean_name = _clean_text(sheet_name) sheet_token = _normalize_sheet_token(sheet_name) or sheet_clean_name override_set = overrides_map.get(sheet_token) if override_set is None and sheet_clean_name != sheet_token: override_set = overrides_map.get(sheet_clean_name) if override_set is None: override_set = set() binding_sheet_map = theme_binding_map.get(sheet_token) if binding_sheet_map is None and sheet_clean_name != sheet_token: binding_sheet_map = theme_binding_map.get(sheet_clean_name) if binding_sheet_map is None: binding_sheet_map = {} permit_groups: Dict[str, List[Dict[str, Any]]] = sheet_data.get("permit_groups", {}) sheet_snapshot_count = 0 sheet_risk_count = 0 sheet_created: List[str] = [] sheet_overwritten: List[str] = [] sheet_skipped: List[str] = [] for permit_name, permit_rows in permit_groups.items(): print(f"DEBUG: Processing permit '{permit_name}' with {len(permit_rows)} rows...") canonical_permit_name = _clean_text(permit_name) permit_token = _normalize_permit_token(permit_name) if not canonical_permit_name or not permit_token: continue permit_id = existing_permits.get(permit_token) or existing_permits.get(canonical_permit_name) should_override = (permit_token in override_set) or (canonical_permit_name in override_set) permit_modified = False if permit_id and not should_override: sheet_skipped.append(canonical_permit_name) result["skipped_permits"].append( { "sheet": sheet_name, "permit_name": canonical_permit_name, "region_id": region_id, "reason": "exists", } ) continue if permit_id: backup_info = _backup_permit_before_import( conn, region_id=region_id, permit_id=permit_id, region_name=region_name, permit_name=canonical_permit_name, filename=workbook_filename, sheet_name=sheet_name, edited_by=edited_by, change_summary=default_change_summary, ) sheet_snapshot_count += backup_info["snapshot_count"] result["snapshot_count"] += backup_info["snapshot_count"] _purge_region_permit_relations(conn, region_id, permit_id) sheet_overwritten.append(canonical_permit_name) result["overwritten_permits"].append( { "sheet": sheet_name, "permit_name": canonical_permit_name, "region_id": region_id, "snapshot_batch_id": backup_info.get("batch_id", ""), } ) permit_modified = True permit_modified = True else: print(f"DEBUG: Creating new permit '{canonical_permit_name}'...") permit_id = _ensure_permit(conn, canonical_permit_name) print(f"DEBUG: Created permit id={permit_id}") for alias in _permit_name_aliases(canonical_permit_name) or {canonical_permit_name}: existing_permits[alias] = permit_id sheet_created.append(canonical_permit_name) result["created_permits"].append( { "sheet": sheet_name, "permit_name": canonical_permit_name, "region_id": region_id, } ) permit_modified = True # Insert details if permit_rows: print("DEBUG: Inserting permit details...") # ... (lines 1412+) pass binding_override = None for alias in _permit_name_aliases(canonical_permit_name) or {canonical_permit_name}: binding_override = binding_sheet_map.get(alias) if binding_override: break override_theme_names: Set[str] = set() binds_all_themes = False if binding_override: for override_value in binding_override: if _is_all_theme_marker(override_value): binds_all_themes = True else: cleaned_override = _clean_text(override_value) if cleaned_override: override_theme_names.add(cleaned_override) theme_names: Set[str] = set() if not binding_override or binds_all_themes: # Prioritize automated resolution from rules table resolved_rule_themes = _resolve_themes_for_permit(conn, canonical_permit_name) if resolved_rule_themes: theme_names.update(resolved_rule_themes) scope_descriptions: Set[str] = set() subitem_names: Set[str] = set() permit_status_val: Optional[str] = None subitem_summary_val: Optional[str] = None responsible_contact_val: Optional[str] = None jurisdiction_scope_val: Optional[str] = None for row in permit_rows: # Workbook theme fallback removed - template has no theme column scope_descriptions.update(row.get("scope_descriptions") or []) subitem_names.update(row.get("subitem_names") or []) if not permit_status_val and row.get("permit_status"): permit_status_val = row.get("permit_status") if not subitem_summary_val and row.get("subitem_summary"): subitem_summary_val = row.get("subitem_summary") if not responsible_contact_val and row.get("responsible_contact"): responsible_contact_val = row.get("responsible_contact") if not jurisdiction_scope_val and row.get("jurisdiction_scope"): jurisdiction_scope_val = row.get("jurisdiction_scope") if binding_override and not binds_all_themes and override_theme_names: theme_names = set(override_theme_names) if binds_all_themes and region_id: cached_theme_options = region_theme_cache.get(region_id) if cached_theme_options is None: fetched_map = _fetch_region_theme_map(conn, [region_id]) cached_theme_options = fetched_map.get(region_id, []) region_theme_cache[region_id] = cached_theme_options for option in cached_theme_options or []: option_label = _clean_text(option.get("name")) if option_label: theme_names.add(option_label) # resolved_themes check moved up if not theme_names: theme_names.add("不涉及") theme_ids: List[str] = [] for theme_name in sorted(theme_names): if _is_all_theme_marker(theme_name): continue theme_id = _ensure_theme(conn, theme_name) theme_ids.append(theme_id) cur.execute( """ INSERT INTO region_themes (region_id, theme_id) VALUES (%s, %s) ON CONFLICT DO NOTHING """, (region_id, theme_id), ) if cur.rowcount: cache_entry = region_theme_cache.setdefault(region_id, []) if not any(entry.get("id") == theme_id for entry in cache_entry): cache_entry.append({"id": theme_id, "name": theme_name}) _propagate_all_theme_bindings(conn, region_id, theme_id) cur.execute( """ INSERT INTO region_theme_permits (region_id, theme_id, permit_id) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING """, (region_id, theme_id, permit_id), ) if binds_all_themes: _set_permit_bind_all_themes( conn, region_id, permit_id, created_by=edited_by or session_uploaded_by, ) elif binding_override: _clear_permit_bind_all_themes(conn, region_id, permit_id) for scope_desc in sorted(scope_descriptions): scope_id = _ensure_business_scope(conn, scope_desc) if not scope_id: continue cur.execute( """ INSERT INTO region_permit_scopes (region_id, permit_id, scope_id) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING """, (region_id, permit_id, scope_id), ) for subitem_desc in sorted(subitem_names): subitem_id = _ensure_permit_subitem(conn, subitem_desc) if not subitem_id: continue cur.execute( """ INSERT INTO region_permit_subitems (region_id, permit_id, subitem_id) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING """, (region_id, permit_id, subitem_id), ) _ensure_contact_info_column(conn) # Resolve contact info for the permit contact_info_val = None if effective_bound_department_id: dept_data = _fetch_service_department(cur, effective_bound_department_id) if dept_data: contact_info_val = dept_data.get("phone") cur.execute( """ INSERT INTO region_permit_details ( region_id, permit_id, permit_status, subitem_summary, responsible_contact, jurisdiction_scope, filler_name, unit_name, source_update_date, contact_info ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (region_id, permit_id) DO UPDATE SET permit_status = EXCLUDED.permit_status, subitem_summary = EXCLUDED.subitem_summary, responsible_contact = EXCLUDED.responsible_contact, jurisdiction_scope = EXCLUDED.jurisdiction_scope, updated_at = now(), filler_name = EXCLUDED.filler_name, unit_name = EXCLUDED.unit_name, source_update_date = EXCLUDED.source_update_date, contact_info = COALESCE(EXCLUDED.contact_info, region_permit_details.contact_info) """, ( region_id, permit_id, permit_status_val, subitem_summary_val, responsible_contact_val, jurisdiction_scope_val, row.get("filler_name"), row.get("unit_name"), row.get("source_update_date"), contact_info_val, ), ) for row in permit_rows: risk_id = _ensure_risk( conn, risk_content=row.get("risk_content", ""), legal_basis=row.get("legal_basis"), document_no=row.get("document_no"), summary=row.get("summary"), remark=row.get("remark"), ) cur.execute( """ INSERT INTO region_permit_risks (region_id, permit_id, risk_id, serial_number) VALUES (%s, %s, %s, %s) ON CONFLICT (region_id, permit_id, risk_id) DO UPDATE SET serial_number = EXCLUDED.serial_number """, (region_id, permit_id, risk_id, row.get("serial_number")), ) sheet_risk_count += len(permit_rows) result["risk_count"] += len(permit_rows) source_detail_payload = { "sheet": sheet_name, "permit": canonical_permit_name, "risk_rows": len(permit_rows), "imported_at": datetime.utcnow().isoformat(), } cur.execute( """ INSERT INTO permit_sources ( region_id, permit_id, source_type, source_name, source_detail, uploader_department_id, bound_department_id, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s, %s, %s, now(), now()) ON CONFLICT (region_id, permit_id) DO UPDATE SET source_type = EXCLUDED.source_type, source_name = EXCLUDED.source_name, source_detail = EXCLUDED.source_detail, uploader_department_id = EXCLUDED.uploader_department_id, bound_department_id = EXCLUDED.bound_department_id, updated_at = now() """, ( region_id, permit_id, "excel", workbook_filename or sheet_name, json.dumps(source_detail_payload, ensure_ascii=False), session_uploader_department, effective_bound_department_id, ), ) if stored_file_id and permit_modified: _link_file_to_permit( conn, file_id=stored_file_id, region_id=region_id, permit_id=permit_id, created_by=session_uploaded_by or edited_by, ) result["processed_sheets"].append( { "sheet_name": sheet_name, "region_id": region_id, "region_name": region_name, "created_permits": sheet_created, "overwritten_permits": sheet_overwritten, "skipped_permits": sheet_skipped, "snapshot_count": sheet_snapshot_count, "risk_count": sheet_risk_count, } ) logger.info( "[PERMIT-IMPORT] Sheet %s processed (region=%s) -> created=%d, overwritten=%d, skipped=%d, risks=%d, snapshots=%d", sheet_name, region_id, len(sheet_created), len(sheet_overwritten), len(sheet_skipped), sheet_risk_count, sheet_snapshot_count, ) conn.commit() except Exception: conn.rollback() raise result["file_attachment"] = stored_file_meta # Log the import operation try: log_operation( operator=edited_by or session_uploaded_by or "unknown", operation_type="IMPORT", target_type="PERMIT_BATCH", target_name=workbook_filename, change_summary=change_summary or f"Imported {len(sheet_names)} sheets from {workbook_filename}", details={ "filename": workbook_filename, "sheets_processed": len(sheet_names), "created_count": len(result["created_permits"]), "overwritten_count": len(result["overwritten_permits"]), "skipped_count": len(result["skipped_permits"]), "snapshot_count": result["snapshot_count"] } ) except Exception as e: logger.error(f"[PERMIT-IMPORT] Failed to log operation: {e}") with _PERMIT_IMPORT_LOCK: _PERMIT_IMPORT_SESSIONS.pop(session_id, None) logger.info( "[PERMIT-IMPORT] Completed session %s: created=%d overwritten=%d skipped=%d snapshots=%d risks=%d", session_id, len(result["created_permits"]), len(result["overwritten_permits"]), len(result["skipped_permits"]), result["snapshot_count"], result["risk_count"], ) return result def describe_permit_import_session(session_id: str) -> Dict[str, Any]: """Return preview data for a pending permit import session.""" if not session_id: raise ValueError("导入会话无效") with _PERMIT_IMPORT_LOCK: session_payload = _PERMIT_IMPORT_SESSIONS.get(session_id) if not session_payload: raise ValueError("导入会话不存在或已过期，请重新上传Excel文件") session_sheets: Dict[str, Dict[str, Any]] = session_payload.get("sheets") or {} if not session_sheets: raise ValueError("导入会话不存在或已过期，请重新上传Excel文件") region_ids = sorted({sheet.get("region_id") for sheet in session_sheets.values() if sheet.get("region_id")}) region_theme_map: Dict[str, List[Dict[str, str]]] = {} all_theme_options: List[Dict[str, str]] = [] with _lic_pg_conn() as conn: all_theme_options = _fetch_all_theme_options(conn) if region_ids: region_theme_map = _fetch_region_theme_map(conn, region_ids) preview_sheets: List[Dict[str, Any]] = [] total_permits = 0 total_risks = 0 with _lic_pg_conn() as conn: for sheet_name in sorted(session_sheets.keys()): sheet_data = session_sheets[sheet_name] region_id = sheet_data.get("region_id") or "" region_name = sheet_data.get("region_name") or sheet_name duplicate_permits = set(sheet_data.get("duplicate_permits") or []) new_permits = set(sheet_data.get("new_permits") or []) permit_groups: Dict[str, List[Dict[str, Any]]] = sheet_data.get("permit_groups") or {} permit_summaries: List[Dict[str, Any]] = [] sheet_risk_total = 0 for permit_name in sorted(permit_groups.keys()): permit_rows = permit_groups[permit_name] sheet_risk_total += len(permit_rows) total_risks += len(permit_rows) permit_risks = [] common_meta = { "responsible_contact": "", "jurisdiction_scope": "", "unit_name": "", "permit_status": "" } for row in permit_rows: permit_risks.append({ "serial_number": row.get("serial_number"), "risk_content": row.get("risk_content"), "legal_basis": row.get("legal_basis"), "document_no": row.get("document_no"), "summary": row.get("summary"), "remark": row.get("remark"), }) for key in common_meta: if not common_meta[key] and row.get(key): common_meta[key] = row.get(key) # Try to resolve themes via rules (Base Table Logic) resolved_theme_names = _resolve_themes_for_permit(conn, permit_name) permit_summaries.append( { "permit_name": permit_name, "canonical_name": _clean_text(permit_name), "risk_count": len(permit_rows), "is_duplicate": permit_name in duplicate_permits, "is_new": permit_name in new_permits, "default_theme_names": [], "resolved_theme_names": resolved_theme_names, "risks": permit_risks, "common_meta": common_meta, "sample_serial": permit_rows[0].get("serial_number") if permit_rows else None, "sample_risk": permit_rows[0].get("risk_content") if permit_rows else "", } ) total_permits += len(permit_summaries) theme_options: List[Dict[str, Any]] = [] seen_theme_labels: Set[str] = set() theme_options.append( { "id": ALL_THEMES_SENTINEL, "name": ALL_THEMES_DISPLAY_NAME, "source": "virtual", "description": "选择后将与当前及未来的新主题自动绑定", "is_all": True, } ) seen_theme_labels.add(ALL_THEMES_SENTINEL.lower()) seen_theme_labels.add(ALL_THEMES_DISPLAY_NAME.lower()) candidate_theme_lists: List[List[Dict[str, str]]] = [] region_theme_options = region_theme_map.get(region_id) if region_theme_options: candidate_theme_lists.append(region_theme_options) if all_theme_options: candidate_theme_lists.append(all_theme_options) for option_list in candidate_theme_lists: for option in option_list: label = option.get("name") or "" normalized = label.lower() if normalized in seen_theme_labels: continue seen_theme_labels.add(normalized) theme_options.append( { "id": option.get("id") or "", "name": label, "source": "existing", } ) # Workbook themes are no longer supported as the template has no theme column pass preview_sheets.append( { "sheet_name": sheet_name, "region_name": region_name, "region_id": region_id, "missing_region": not bool(region_id), "theme_options": theme_options, "permits": permit_summaries, "duplicate_permits": list(duplicate_permits), "new_permits": list(new_permits), "permit_count": len(permit_summaries), "risk_count": sheet_risk_total, } ) return { "session_id": session_id, "filename": session_payload.get("filename") or "", "sheet_count": len(preview_sheets), "permit_total": total_permits, "risk_total": total_risks, "sheets": preview_sheets, } def _permit_sources_available(conn: pg.Connection) -> bool: """Return True if permit_sources table exists (cached).""" global _PERMIT_SOURCES_TABLE_PRESENT if _PERMIT_SOURCES_TABLE_PRESENT is True: return True if _PERMIT_SOURCES_TABLE_PRESENT is False: return False with _PERMIT_SOURCES_TABLE_LOCK: if _PERMIT_SOURCES_TABLE_PRESENT is not None: return bool(_PERMIT_SOURCES_TABLE_PRESENT) cur = conn.cursor() cur.execute( """ SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'permit_sources' LIMIT 1 """ ) exists = cur.fetchone() is not None _PERMIT_SOURCES_TABLE_PRESENT = exists return exists def _ensure_contact_info_column(conn: pg.Connection) -> None: """Ensure contact_info column exists in region_permit_details.""" cur = conn.cursor() cur.execute( """ ALTER TABLE IF EXISTS region_permit_details ADD COLUMN IF NOT EXISTS contact_info text """ ) # We don't commit here as it might be part of a larger transaction def _create_permit_sources_schema(conn: pg.Connection) -> None: """Create permit_sources table and ancillary indexes if missing.""" cur = conn.cursor() cur.execute( """ CREATE TABLE IF NOT EXISTS permit_sources ( region_id uuid NOT NULL REFERENCES regions(id) ON DELETE CASCADE, permit_id uuid NOT NULL REFERENCES permits(id) ON DELETE CASCADE, source_type text NOT NULL, source_name text NOT NULL, source_detail text, uploader_department_id uuid REFERENCES service_departments(id), bound_department_id uuid REFERENCES service_departments(id), created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (region_id, permit_id) ) """ ) cur.execute( """ CREATE INDEX IF NOT EXISTS permit_sources_source_name_idx ON permit_sources (source_name) """ ) # 兼容已有表：补齐绑定相关字段及索引 cur.execute( """ ALTER TABLE IF EXISTS permit_sources ADD COLUMN IF NOT EXISTS uploader_department_id uuid REFERENCES service_departments(id) """ ) cur.execute( """ ALTER TABLE IF EXISTS permit_sources ADD COLUMN IF NOT EXISTS bound_department_id uuid REFERENCES service_departments(id) """ ) cur.execute( """ CREATE INDEX IF NOT EXISTS idx_permit_sources_bound_dept ON permit_sources (bound_department_id) """ ) cur.execute( """ CREATE INDEX IF NOT EXISTS idx_permit_sources_uploader ON permit_sources (uploader_department_id) """ ) def _ensure_permit_sources_table(conn: Optional[pg.Connection] = None) -> None: """Ensure the permit_sources table exists and cache the result.""" global _PERMIT_SOURCES_TABLE_PRESENT if _PERMIT_SOURCES_TABLE_PRESENT is True: return with _PERMIT_SOURCES_TABLE_LOCK: if _PERMIT_SOURCES_TABLE_PRESENT is True: return if conn is not None: original_autocommit = conn.autocommit try: conn.autocommit = True _create_permit_sources_schema(conn) finally: conn.autocommit = original_autocommit else: with _lic_pg_conn(autocommit=True) as ensure_conn: _create_permit_sources_schema(ensure_conn) _PERMIT_SOURCES_TABLE_PRESENT = True def _create_service_department_schema(conn: pg.Connection) -> None: cur = conn.cursor() cur.execute( """ CREATE TABLE IF NOT EXISTS service_departments ( id uuid PRIMARY KEY, name text NOT NULL, code text NOT NULL UNIQUE, phone text, parent_id uuid REFERENCES service_departments(id) ON DELETE SET NULL, region_id uuid REFERENCES regions(id) ON DELETE SET NULL, description text, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now() ) """ ) cur.execute( """ CREATE UNIQUE INDEX IF NOT EXISTS service_departments_name_idx ON service_departments (name) """ ) cur.execute( """ ALTER TABLE IF EXISTS service_departments ADD COLUMN IF NOT EXISTS parent_id uuid REFERENCES service_departments(id) ON DELETE SET NULL """ ) cur.execute( """ ALTER TABLE IF EXISTS service_departments ADD COLUMN IF NOT EXISTS phone text """ ) cur.execute( """ ALTER TABLE IF EXISTS service_departments ADD COLUMN IF NOT EXISTS grade int DEFAULT 0 """ ) cur.execute( """ CREATE INDEX IF NOT EXISTS service_departments_parent_idx ON service_departments (parent_id) """ ) cur.execute( """ CREATE INDEX IF NOT EXISTS service_departments_grade_idx ON service_departments (grade) """ ) cur.execute( """ CREATE TABLE IF NOT EXISTS service_department_permits ( department_id uuid NOT NULL REFERENCES service_departments(id) ON DELETE CASCADE, region_id uuid NOT NULL REFERENCES regions(id) ON DELETE CASCADE, permit_id uuid NOT NULL REFERENCES permits(id) ON DELETE CASCADE, created_at timestamptz NOT NULL DEFAULT now(), created_by text, PRIMARY KEY (department_id, region_id, permit_id) ) """ ) cur.execute( """ CREATE INDEX IF NOT EXISTS service_dept_permits_region_idx ON service_department_permits (region_id, permit_id) """ ) def _ensure_service_department_schema(conn: Optional[pg.Connection] = None) -> None: global _SERVICE_DEPARTMENT_SCHEMA_READY if _SERVICE_DEPARTMENT_SCHEMA_READY: return with _SERVICE_DEPARTMENT_SCHEMA_LOCK: if _SERVICE_DEPARTMENT_SCHEMA_READY: return if conn is not None: original_autocommit = conn.autocommit try: conn.autocommit = True _create_service_department_schema(conn) finally: conn.autocommit = original_autocommit else: with _lic_pg_conn(autocommit=True) as ensure_conn: _create_service_department_schema(ensure_conn) _SERVICE_DEPARTMENT_SCHEMA_READY = True _SERVICE_DEPARTMENT_SELECT = """ SELECT sd.id, sd.name, sd.code, sd.phone, sd.parent_id, parent.name AS parent_name, sd.region_id, r.name AS region_name, sd.description, sd.grade, sd.unit_level, sd.created_at, sd.updated_at FROM service_departments sd LEFT JOIN service_departments parent ON parent.id = sd.parent_id LEFT JOIN regions r ON r.id = sd.region_id """ def _serialize_service_department_row(record: Dict[str, Any]) -> Dict[str, Any]: return { "id": _to_optional_str(record.get("id")), "name": record.get("name"), "code": record.get("code"), "phone": record.get("phone"), "parent_id": _to_optional_str(record.get("parent_id")), "parent_name": record.get("parent_name"), "region_id": _to_optional_str(record.get("region_id")), "region_name": record.get("region_name"), "description": record.get("description"), "grade": record.get("grade", 0), "unit_level": record.get("unit_level", "unit"), "created_at": _to_isoformat(record.get("created_at")), "updated_at": _to_isoformat(record.get("updated_at")), } def _fetch_service_department(cur: pg.Cursor, department_id: str) -> Optional[Dict[str, Any]]: cur.execute( _SERVICE_DEPARTMENT_SELECT + " WHERE sd.id = %s LIMIT 1", (department_id,), ) row = cur.fetchone() if not row: return None columns = tuple(col[0] for col in cur.description) record = {columns[idx]: row[idx] for idx in range(len(columns))} return _serialize_service_department_row(record) def list_service_departments(region_id: Optional[str] = None) -> List[Dict[str, Any]]: """List service departments, optionally filtered by region. Args: region_id: If provided, only return departments associated with this region Returns: List of department dictionaries """ with _lic_pg_conn() as conn: _ensure_service_department_schema(conn) cur = conn.cursor() if region_id: # Get departments associated with the specific region through service_departments.region_id # This includes both direct region match and hierarchical matching (child regions) sql = _SERVICE_DEPARTMENT_SELECT + """ WHERE sd.region_id = %s ORDER BY sd.created_at ASC """ cur.execute(sql, (region_id,)) else: # Get all departments cur.execute(_SERVICE_DEPARTMENT_SELECT + " ORDER BY sd.created_at ASC") rows = cur.fetchall() columns = tuple(col[0] for col in cur.description) departments: List[Dict[str, Any]] = [] for row in rows: record = {columns[idx]: row[idx] for idx in range(len(columns))} departments.append(_serialize_service_department_row(record)) return departments def build_service_department_tree() -> List[Dict[str, Any]]: """ Build a tree structure of service departments with parent-child relationships. Returns a list of root-level departments with nested children. """ departments = list_service_departments() # Create a dictionary for fast lookup dept_dict = {dept["id"]: dept.copy() for dept in departments} # Initialize children arrays for dept_id in dept_dict: dept_dict[dept_id]["children"] = [] # Build tree structure tree: List[Dict[str, Any]] = [] for dept_id, dept in dept_dict.items(): if dept.get("parent_id"): # Has a parent, add to parent's children parent_id = dept["parent_id"] if parent_id in dept_dict: dept_dict[parent_id]["children"].append(dept) else: # Root level department tree.append(dept) # Sort tree recursively def sort_tree(nodes: List[Dict[str, Any]]) -> None: nodes.sort(key=lambda x: x.get("name", "")) for node in nodes: if node.get("children"): sort_tree(node["children"]) sort_tree(tree) return tree def get_subordinate_departments(department_id: str, level: Optional[str] = None) -> List[str]: """获取指定单位的所有下级单位ID列表。 Args: department_id: 父单位ID level: 指定下级单位的unit_level（None表示所有级别） Returns: 下级单位ID列表 """ with _lic_pg_conn() as conn: cur = conn.cursor() if level: cur.execute(""" SELECT id FROM service_departments WHERE parent_id = %s AND unit_level = %s """, (department_id, level)) else: cur.execute(""" SELECT id FROM service_departments WHERE parent_id = %s """, (department_id,)) return [str(row[0]) for row in cur.fetchall()] def _fetch_department_descendants(cur: pg.Cursor, root_id: str) -> List[str]: """返回包含自身在内的下级单位ID列表（递归）。""" if not root_id: return [] cur.execute( """ WITH RECURSIVE sub AS ( SELECT id FROM service_departments WHERE id = %s UNION ALL SELECT sd.id FROM service_departments sd JOIN sub ON sd.parent_id = sub.id ) SELECT id FROM sub """, (root_id,), ) return [str(row[0]) for row in cur.fetchall()] def create_service_department( name: str, *, code: Optional[str] = None, phone: Optional[str] = None, parent_id: Optional[str] = None, region_id: Optional[str] = None, description: Optional[str] = None, grade: Optional[int] = None, # grade参数保留但不再使用，自动根据层级计算 unit_level: Optional[str] = None, # 新增：单位级别 operator: str = "admin", ) -> Dict[str, Any]: normalized_name = _clean_text(name) if not normalized_name: raise ValueError("服务部门名称不能为空") normalized_code = (_clean_text(code).upper() if code else "") or uuid.uuid4().hex[:8].upper() parent_token = _clean_text(parent_id) or None region_token = _clean_text(region_id) or None description_text = description.strip() if isinstance(description, str) else description unit_level_token = unit_level.strip().lower() if unit_level else None # 自动根据父节点计算权限等级，不再使用传入的grade参数 grade_value = _calculate_grade_by_parent(parent_token) with _lic_pg_conn() as conn: _ensure_service_department_schema(conn) cur = conn.cursor() dept_id = uuid.uuid4() # 确定unit_level值：未指定时，根节点视为市级，其余视为末级 if unit_level_token: final_unit_level = unit_level_token else: final_unit_level = 'municipal' if parent_token is None else 'unit' cur.execute( """ INSERT INTO service_departments (id, name, code, phone, parent_id, region_id, description, grade, unit_level) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) """, (dept_id, normalized_name, normalized_code, phone, parent_token, region_token, description_text, grade_value, final_unit_level), ) conn.commit() result = _fetch_service_department(cur, str(dept_id)) if not result: raise RuntimeError("创建服务部门失败") # Log the operation log_operation( operator=operator, operation_type="CREATE", target_type="DEPARTMENT", target_id=str(result["id"]), target_name=result["name"], change_summary=f"Created department: {result['name']}", details=result ) return result def _calculate_grade_by_parent(parent_id: Optional[str]) -> int: """ 根据父节点计算权限等级 Args: parent_id: 父节点ID，如果为None表示是根级节点 Returns: int: 权限等级数值 """ # 根级节点（无父节点）= 超级权限 if not parent_id: return 90 # 获取父节点的层级信息 parent_level = _get_department_level(parent_id) # 子节点的层级 = 父节点层级 + 1 child_level = parent_level + 1 # 根据层级映射权限等级 grade = 60 # 默认四级及以下 if child_level == 1: grade = 80 # 二级部门 - 高级权限 elif child_level == 2: grade = 70 # 三级部门 - 中级权限 return grade def _get_department_level(department_id: str) -> int: """ 获取部门在组织架构中的层级（根节点为0） Args: department_id: 部门ID Returns: int: 部门层级 """ with _lic_pg_conn() as conn: cur = conn.cursor() level = 0 current_id = department_id while current_id: cur.execute( "SELECT parent_id FROM service_departments WHERE id = %s", (current_id,) ) result = cur.fetchone() if result and result[0]: level += 1 current_id = result[0] else: break return level def update_service_department( department_id: str, *, name: Optional[str] = None, phone: Optional[str] = None, parent_id: Optional[str] = None, region_id: Optional[str] = None, description: Optional[str] = None, grade: Optional[int] = None, unit_level: Optional[str] = None, # 新增：单位级别 operator: str = "admin", ) -> Optional[Dict[str, Any]]: dept_token = _clean_text(department_id) if not dept_token: raise ValueError("department_id 不能为空") updates: List[str] = [] values: List[Any] = [] if name is not None: normalized_name = _clean_text(name) if not normalized_name: raise ValueError("服务部门名称不能为空") updates.append("name = %s") values.append(normalized_name) if phone is not None: updates.append("phone = %s") values.append(phone.strip() if isinstance(phone, str) else phone) if parent_id is not None: parent_token = _clean_text(parent_id) or None if parent_token and parent_token == dept_token: raise ValueError("服务部门不能设置为自己的上级") updates.append("parent_id = %s") values.append(parent_token) if region_id is not None: updates.append("region_id = %s") values.append(_clean_text(region_id) or None) if description is not None: updates.append("description = %s") values.append(description.strip() if isinstance(description, str) else description) # 如果修改了parent_id，则自动计算grade（根据新层级计算） # grade字段不再接受手动设置，完全基于层级自动计算 if parent_id is not None: new_grade = _calculate_grade_by_parent(parent_id) updates.append("grade = %s") values.append(new_grade) elif grade is not None: # 如果手动设置grade（向后兼容），仍然允许但会记录警告 updates.append("grade = %s") values.append(int(grade)) # 添加unit_level字段处理 if unit_level is not None: unit_level_value = unit_level.strip().lower() if unit_level else None if unit_level_value and unit_level_value not in ('admin', 'municipal', 'district', 'unit'): raise ValueError(f"无效的unit_level值: {unit_level_value}") updates.append("unit_level = %s") values.append(unit_level_value) if not updates: return _get_service_department_by_id(dept_token) updates.append("updated_at = now()") with _lic_pg_conn() as conn: _ensure_service_department_schema(conn) cur = conn.cursor() query = f"UPDATE service_departments SET {', '.join(updates)} WHERE id = %s" values.append(dept_token) cur.execute(query, tuple(values)) conn.commit() result = _get_service_department_by_id(dept_token) # Log the operation log_operation( operator=operator, operation_type="UPDATE", target_type="DEPARTMENT", target_id=dept_token, target_name=result.get("name") if result else None, change_summary=f"Updated department: {result.get('name')}" if result else f"Updated department {dept_token}", details=result ) return result def _get_service_department_by_id(department_id: str) -> Optional[Dict[str, Any]]: with _lic_pg_conn() as conn: cur = conn.cursor() return _fetch_service_department(cur, department_id) def delete_service_department(department_id: str, operator: str = "admin") -> Dict[str, Any]: dept_token = _clean_text(department_id) if not dept_token: raise ValueError("department_id 不能为空") with _lic_pg_conn() as conn: _ensure_service_department_schema(conn) cur = conn.cursor() cur.execute( "SELECT COUNT(*) FROM auth_users WHERE service_department_id = %s", (dept_token,), ) bound_count = int(cur.fetchone()[0] or 0) if bound_count > 0: raise ValueError("仍有账号绑定该服务部门，请先解除绑定后再删除") cur.execute( "UPDATE service_departments SET parent_id = NULL WHERE parent_id = %s", (dept_token,), ) cur.execute("SELECT name FROM service_departments WHERE id = %s", (dept_token,)) dept_name_row = cur.fetchone() dept_name = dept_name_row[0] if dept_name_row else dept_token cur.execute( "DELETE FROM service_departments WHERE id = %s RETURNING id", (dept_token,), ) row = cur.fetchone() if not row: conn.rollback() return {"deleted": False} conn.commit() # Log the operation log_operation( operator=operator, operation_type="DELETE", target_type="DEPARTMENT", target_id=dept_token, target_name=dept_name, change_summary=f"Deleted department: {dept_name}" ) return {"deleted": True} _THEME_SUMMARY_SELECT = """ SELECT t.id, t.name, COUNT(DISTINCT rtp.permit_id) AS permit_count, COUNT(DISTINCT rtp.region_id) AS region_count, STRING_AGG(DISTINCT r.name, ',') AS region_names FROM themes t LEFT JOIN region_theme_permits rtp ON rtp.theme_id = t.id LEFT JOIN regions r ON rtp.region_id = r.id """ def _serialize_theme_row(record: Dict[str, Any]) -> Dict[str, Any]: region_names_str = record.get("region_names") or "" levels = set() r_names = [n.strip() for n in region_names_str.split(',') if n.strip()] for r_name in r_names: if r_name == "市级" or r_name == "佛山市": levels.add("市级") else: # Assuming any other region is District level (e.g. 禅城区, 南海区) levels.add("区级") impl_level_list = [] if "市级" in levels: impl_level_list.append("市级") if "区级" in levels: impl_level_list.append("区级") impl_level = "、".join(impl_level_list) if impl_level_list else "-" return { "id": _to_optional_str(record.get("id")), "name": record.get("name"), "permit_count": int(record.get("permit_count") or 0), "region_count": int(record.get("region_count") or 0), "implementation_level": impl_level, } def _fetch_theme_summary(cur: pg.Cursor, theme_id: str) -> Optional[Dict[str, Any]]: cur.execute( _THEME_SUMMARY_SELECT + " WHERE t.id = %s GROUP BY t.id, t.name", (theme_id,), ) row = cur.fetchone() if not row: return None columns = tuple(col[0] for col in cur.description) record = {columns[idx]: row[idx] for idx in range(len(columns))} return _serialize_theme_row(record) def list_all_themes( start_date: Optional[str] = None, end_date: Optional[str] = None, department_ids: Optional[List[str]] = None, ) -> List[Dict[str, Any]]: # Base query components select_fields = """ t.id, t.name, COUNT(DISTINCT CASE WHEN {filter_condition} THEN rtp.permit_id END) AS permit_count, COUNT(DISTINCT CASE WHEN {filter_condition} THEN rtp.region_id END) AS region_count, STRING_AGG(DISTINCT r.name, ',') AS region_names """ # Base joins joins = [ "LEFT JOIN region_theme_permits rtp ON rtp.theme_id = t.id", "LEFT JOIN regions r ON rtp.region_id = r.id" ] conditions = [] params = [] # Conditionally add joins and conditions if start_date or end_date: joins.append("LEFT JOIN region_permit_details rpd ON rpd.permit_id = rtp.permit_id AND rpd.region_id = rtp.region_id") if department_ids: joins.append("LEFT JOIN permit_sources ps ON ps.permit_id = rtp.permit_id AND ps.region_id = rtp.region_id") if start_date: conditions.append("rpd.updated_at >= %s") params.append(start_date) if end_date: conditions.append("rpd.updated_at <= %s") params.append(end_date) if department_ids: expanded_ids = _expand_department_family(department_ids) if expanded_ids: placeholders = ','.join(['%s'] * len(expanded_ids)) conditions.append(f"(ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))") params.extend(expanded_ids * 2) filter_condition = " AND ".join(conditions) if conditions else "TRUE" join_clause = "\n".join(joins) sql = f""" SELECT {select_fields.format(filter_condition=filter_condition)} FROM themes t {join_clause} GROUP BY t.id, t.name ORDER BY t.name ASC """ with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, params) rows = cur.fetchall() columns = tuple(col[0] for col in cur.description) items: List[Dict[str, Any]] = [] for row in rows: record = {columns[idx]: row[idx] for idx in range(len(columns))} items.append(_serialize_theme_row(record)) return items def list_unbound_permits( visibility: Optional[str] = None, search_text: Optional[str] = None, department_ids: Optional[List[str]] = None, region_id: Optional[str] = None, ) -> List[Dict[str, Any]]: """Return all permits that are in a region but not bound to any theme in that region. Args: visibility: 'visible', 'hidden', or None (all) search_text: Filter by permit name (keyword) department_ids: Filter by uploader/bound department region_id: Filter by region """ filters = ["rtp.theme_id IS NULL"] params = [] if visibility == 'visible': filters.append("(rpd.is_v2_visible IS TRUE OR rpd.is_v2_visible IS NULL)") elif visibility == 'hidden': filters.append("rpd.is_v2_visible IS FALSE") if search_text: filters.append("p.name ILIKE %s") params.append(f"%{search_text}%") if department_ids: expanded_ids = _expand_department_family(department_ids) if expanded_ids: placeholders = ','.join(['%s'] * len(expanded_ids)) filters.append(f"(ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))") params.extend(expanded_ids * 2) if region_id: filters.append("rpd.region_id = %s") params.append(region_id) ps_join = "" if department_ids: ps_join = """ LEFT JOIN permit_sources ps ON ps.permit_id = rpd.permit_id AND ps.region_id = rpd.region_id """ where_clause = " AND ".join(filters) sql = f""" SELECT r.id AS region_id, r.name AS region_name, p.id AS permit_id, p.name AS permit_name, rpd.unit_name, rpd.updated_at, COALESCE(rpd.is_v2_visible, true) AS is_v2_visible FROM region_permit_details rpd JOIN regions r ON r.id = rpd.region_id JOIN permits p ON p.id = rpd.permit_id LEFT JOIN region_theme_permits rtp ON rtp.region_id = rpd.region_id AND rtp.permit_id = rpd.permit_id {ps_join} WHERE {where_clause} ORDER BY r.name, p.name """ items: List[Dict[str, Any]] = [] try: with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, params) rows = cur.fetchall() columns = tuple(col[0] for col in cur.description) for row in rows: record = {columns[idx]: row[idx] for idx in range(len(columns))} # Serialize UUIDs and timestamps record["region_id"] = str(record["region_id"]) record["permit_id"] = str(record["permit_id"]) if record["updated_at"]: record["updated_at"] = record["updated_at"].isoformat() items.append(record) except Exception as e: logger.error(f"Error listing unbound permits: {e}") return items def create_theme(name: str, operator: str = "admin") -> Dict[str, Any]: normalized = _clean_text(name) if not normalized: raise ValueError("主题名称不能为空") with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute( """ INSERT INTO themes (name) VALUES (%s) ON CONFLICT (name) DO NOTHING RETURNING id """, (normalized,), ) row = cur.fetchone() if not row: cur.execute("SELECT id FROM themes WHERE name = %s LIMIT 1", (normalized,)) row = cur.fetchone() conn.commit() if not row: raise RuntimeError("创建主题失败") theme_id = str(row[0]) summary = _fetch_theme_summary(cur, theme_id) if not summary: raise RuntimeError("无法加载主题信息") # Log the operation log_operation( operator=operator, operation_type="CREATE", target_type="THEME", target_id=str(summary["id"]), target_name=summary["name"], change_summary=f"Created theme: {summary['name']}" ) return summary def rename_theme(theme_id: str, new_name: str, operator: str = "admin") -> Dict[str, Any]: theme_token = _clean_text(theme_id) if not theme_token: raise ValueError("theme_id 不能为空") normalized = _clean_text(new_name) if not normalized: raise ValueError("主题名称不能为空") with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute( "UPDATE themes SET name = %s WHERE id = %s RETURNING id", (normalized, theme_token), ) row = cur.fetchone() if not row: conn.rollback() raise ValueError("主题不存在或已被删除") conn.commit() summary = _fetch_theme_summary(cur, theme_token) if not summary: raise RuntimeError("无法刷新主题信息") # Log the operation log_operation( operator=operator, operation_type="UPDATE", target_type="THEME", target_id=theme_token, target_name=normalized, change_summary=f"Renamed theme to: {normalized}" ) return summary def delete_theme(theme_id: str, operator: str = "admin") -> Dict[str, Any]: theme_token = _clean_text(theme_id) if not theme_token: raise ValueError("theme_id 不能为空") with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute("SELECT name FROM themes WHERE id = %s", (theme_token,)) theme_name_row = cur.fetchone() theme_name = theme_name_row[0] if theme_name_row else theme_token cur.execute( "DELETE FROM region_theme_permits WHERE theme_id = %s", (theme_token,), ) rtp_deleted = int(cur.rowcount or 0) cur.execute( "DELETE FROM region_themes WHERE theme_id = %s", (theme_token,), ) rt_deleted = int(cur.rowcount or 0) cur.execute( "DELETE FROM themes WHERE id = %s RETURNING id", (theme_token,), ) row = cur.fetchone() if not row: conn.rollback() return {"deleted": False, "message": "主题不存在"} conn.commit() # Log the operation log_operation( operator=operator, operation_type="DELETE", target_type="THEME", target_id=theme_token, target_name=theme_name, change_summary=f"Deleted theme: {theme_name}" ) return { "deleted": True, "region_theme_permits_deleted": rtp_deleted, "region_themes_deleted": rt_deleted, } def _create_permit_file_schema(conn: pg.Connection) -> None: """Create permit file storage tables on demand.""" cur = conn.cursor() cur.execute( """ CREATE TABLE IF NOT EXISTS permit_files ( id uuid PRIMARY KEY, filename text NOT NULL, content_type text, file_size integer NOT NULL, file_data bytea NOT NULL, checksum text, uploaded_by text, created_at timestamptz NOT NULL DEFAULT now() ) """ ) cur.execute( """ CREATE TABLE IF NOT EXISTS permit_file_links ( id uuid PRIMARY KEY, region_id uuid NOT NULL REFERENCES regions(id) ON DELETE CASCADE, permit_id uuid NOT NULL REFERENCES permits(id) ON DELETE CASCADE, file_id uuid NOT NULL REFERENCES permit_files(id) ON DELETE CASCADE, created_by text, created_at timestamptz NOT NULL DEFAULT now() ) """ ) cur.execute( """ CREATE UNIQUE INDEX IF NOT EXISTS permit_file_links_region_permit_idx ON permit_file_links (region_id, permit_id) """ ) cur.execute( """ CREATE INDEX IF NOT EXISTS permit_file_links_permit_idx ON permit_file_links (permit_id) """ ) def _ensure_permit_file_schema(conn: Optional[pg.Connection] = None) -> None: """Ensure permit file tables exist (lazy creation, thread safe).""" global _PERMIT_FILE_SCHEMA_READY if _PERMIT_FILE_SCHEMA_READY: return with _PERMIT_FILE_SCHEMA_LOCK: if _PERMIT_FILE_SCHEMA_READY: return if conn is not None: original_autocommit = conn.autocommit try: conn.autocommit = True _create_permit_file_schema(conn) finally: conn.autocommit = original_autocommit else: with _lic_pg_conn(autocommit=True) as ensure_conn: _create_permit_file_schema(ensure_conn) _PERMIT_FILE_SCHEMA_READY = True def _create_permit_theme_override_schema(conn: pg.Connection) -> None: """Create region-level theme override table for all-theme bindings.""" cur = conn.cursor() cur.execute( """ CREATE TABLE IF NOT EXISTS region_permit_theme_overrides ( region_id uuid NOT NULL REFERENCES regions(id) ON DELETE CASCADE, permit_id uuid NOT NULL REFERENCES permits(id) ON DELETE CASCADE, binds_all_themes boolean NOT NULL DEFAULT false, created_at timestamptz NOT NULL DEFAULT now(), created_by text, PRIMARY KEY (region_id, permit_id) ) """ ) def _ensure_permit_theme_override_schema(conn: Optional[pg.Connection] = None) -> None: """Ensure the override table exists; it is required for 'all theme' bindings.""" global _PERMIT_THEME_OVERRIDE_SCHEMA_READY if _PERMIT_THEME_OVERRIDE_SCHEMA_READY: return with _PERMIT_THEME_OVERRIDE_SCHEMA_LOCK: if _PERMIT_THEME_OVERRIDE_SCHEMA_READY: return if conn is not None: original_autocommit = conn.autocommit try: conn.autocommit = True _create_permit_theme_override_schema(conn) finally: conn.autocommit = original_autocommit else: with _lic_pg_conn(autocommit=True) as ensure_conn: _create_permit_theme_override_schema(ensure_conn) _PERMIT_THEME_OVERRIDE_SCHEMA_READY = True def _create_permit_approval_departments_schema(conn: pg.Connection) -> None: """Create table for mapping permits to approval departments.""" cur = conn.cursor() cur.execute( """ CREATE TABLE IF NOT EXISTS permit_approval_departments ( id uuid PRIMARY KEY, permit_name text NOT NULL, department_name text NOT NULL, created_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now() ) """ ) cur.execute( """ CREATE UNIQUE INDEX IF NOT EXISTS permit_approval_departments_name_idx ON permit_approval_departments (permit_name) """ ) def _ensure_permit_approval_departments_schema(conn: Optional[pg.Connection] = None) -> None: """Ensure approval departments table exists.""" global _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY if _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY: return with _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_LOCK: if _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY: return if conn is not None: original_autocommit = conn.autocommit try: conn.autocommit = True _create_permit_approval_departments_schema(conn) finally: conn.autocommit = original_autocommit else: with _lic_pg_conn(autocommit=True) as ensure_conn: _create_permit_approval_departments_schema(ensure_conn) _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY = True def _create_operation_log_schema(conn: pg.Connection) -> None: """Create a table for auditing user operations.""" cur = conn.cursor() cur.execute( """ CREATE TABLE IF NOT EXISTS operation_logs ( id uuid PRIMARY KEY, operator text NOT NULL, operation_type text NOT NULL, target_type text, target_id text, target_name text, change_summary text, details jsonb, created_at timestamptz NOT NULL DEFAULT now() ) """ ) cur.execute("CREATE INDEX IF NOT EXISTS operation_logs_operator_idx ON operation_logs (operator)") cur.execute("CREATE INDEX IF NOT EXISTS operation_logs_created_at_idx ON operation_logs (created_at)") def _ensure_operation_log_schema(conn: Optional[pg.Connection] = None) -> None: """Ensure the operation_logs table exists.""" global _OPERATION_LOG_SCHEMA_READY if _OPERATION_LOG_SCHEMA_READY: return with _OPERATION_LOG_SCHEMA_LOCK: if _OPERATION_LOG_SCHEMA_READY: return if conn is not None: _create_operation_log_schema(conn) else: with _lic_pg_conn(autocommit=True) as ensure_conn: _create_operation_log_schema(ensure_conn) _OPERATION_LOG_SCHEMA_READY = True def log_operation( operator: str, operation_type: str, *, target_type: Optional[str] = None, target_id: Optional[str] = None, target_name: Optional[str] = None, change_summary: Optional[str] = None, details: Optional[Dict[str, Any]] = None ) -> str: """Persist an audit log entry for a user operation.""" _ensure_operation_log_schema() log_id = uuid.uuid4() with _lic_pg_conn(autocommit=True) as conn: cur = conn.cursor() cur.execute( """ INSERT INTO operation_logs ( id, operator, operation_type, target_type, target_id, target_name, change_summary, details ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) """, ( str(log_id), operator, operation_type, target_type, target_id, target_name, change_summary, json.dumps(details, ensure_ascii=False) if details else None ) ) return str(log_id) def list_operation_logs( *, operator: Optional[str] = None, operation_type: Optional[str] = None, limit: int = 50, offset: int = 0 ) -> List[Dict[str, Any]]: """Return recent operation logs with optional filters.""" _ensure_operation_log_schema() filters = [] params = [] if operator: filters.append("operator = %s") params.append(operator) if operation_type: filters.append("operation_type = %s") params.append(operation_type) where_clause = f"WHERE {' AND '.join(filters)}" if filters else "" sql = f""" SELECT id, operator, operation_type, target_type, target_id, target_name, change_summary, details, created_at FROM operation_logs {where_clause} ORDER BY created_at DESC LIMIT %s OFFSET %s """ params.extend([limit, offset]) logs = [] with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, tuple(params)) for row in cur.fetchall(): l_id, op, o_type, t_type, t_id, t_name, summary, details_raw, created_at = row # Details handle details = None if details_raw: if isinstance(details_raw, dict): details = details_raw else: try: details = json.loads(details_raw) except: details = str(details_raw) logs.append({ "id": str(l_id), "operator": op, "operation_type": o_type, "target_type": t_type, "target_id": t_id, "target_name": t_name, "change_summary": summary, "details": details, "created_at": _convert_snapshot_value(created_at) }) return logs def _insert_permit_file_record( conn: pg.Connection, *, file_bytes: bytes, filename: str, content_type: Optional[str], uploaded_by: Optional[str], ) -> Dict[str, Any]: """Persist an uploaded file and return its metadata.""" normalized_name = filename or "许可导入.xlsx" content_type = content_type or "application/octet-stream" file_id = uuid.uuid4() checksum = hashlib.sha256(file_bytes).hexdigest() cur = conn.cursor() cur.execute( """ INSERT INTO permit_files ( id, filename, content_type, file_size, file_data, checksum, uploaded_by ) VALUES (%s, %s, %s, %s, %s, %s, %s) """, ( file_id, normalized_name, content_type, len(file_bytes), file_bytes, checksum, uploaded_by, ), ) return { "file_id": str(file_id), "filename": normalized_name, "content_type": content_type, "file_size": len(file_bytes), "checksum": checksum, "uploaded_by": uploaded_by, } def _link_file_to_permit( conn: pg.Connection, *, file_id: str, region_id: str, permit_id: str, created_by: Optional[str], ) -> None: """Associate a stored file with a region-permit pair.""" if not (file_id and region_id and permit_id): return rid = str(region_id) pid = str(permit_id) cur = conn.cursor() cur.execute( """ INSERT INTO permit_file_links (id, region_id, permit_id, file_id, created_by) VALUES (%s, %s, %s, %s, %s) ON CONFLICT (region_id, permit_id) DO UPDATE SET file_id = EXCLUDED.file_id, created_by = EXCLUDED.created_by, created_at = now() """, (uuid.uuid4(), rid, pid, file_id, created_by), ) def _set_permit_bind_all_themes( conn: pg.Connection, region_id: str, permit_id: str, *, created_by: Optional[str] = None, ) -> None: """Mark a permit so it always links to every theme under the region.""" if not (region_id and permit_id): return _ensure_permit_theme_override_schema(conn) cur = conn.cursor() cur.execute( """ INSERT INTO region_permit_theme_overrides (region_id, permit_id, binds_all_themes, created_by) VALUES (%s, %s, TRUE, %s) ON CONFLICT (region_id, permit_id) DO UPDATE SET binds_all_themes = TRUE, created_by = COALESCE(EXCLUDED.created_by, region_permit_theme_overrides.created_by), created_at = CASE WHEN region_permit_theme_overrides.binds_all_themes IS DISTINCT FROM EXCLUDED.binds_all_themes THEN now() ELSE region_permit_theme_overrides.created_at END """, (region_id, permit_id, created_by), ) global _PERMIT_THEME_OVERRIDE_SCHEMA_READY _PERMIT_THEME_OVERRIDE_SCHEMA_READY = True def _clear_permit_bind_all_themes(conn: pg.Connection, region_id: str, permit_id: str) -> None: """Remove the 'all themes' override for a permit.""" if not (region_id and permit_id): return if not _PERMIT_THEME_OVERRIDE_SCHEMA_READY: try: _ensure_permit_theme_override_schema() except Exception: return cur = conn.cursor() cur.execute( """ DELETE FROM region_permit_theme_overrides WHERE region_id = %s AND permit_id = %s """, (region_id, permit_id), ) def _fetch_permit_all_theme_flags( conn: pg.Connection, region_id: str, permit_ids: Iterable[str] ) -> Dict[str, bool]: """Return permit_id -> True when the permit is flagged for all themes.""" global _PERMIT_THEME_OVERRIDE_SCHEMA_READY permit_list = [str(pid) for pid in permit_ids if pid] if not permit_list: return {} if _PERMIT_THEME_OVERRIDE_SCHEMA_READY is False: return {} cur = conn.cursor() try: cur.execute( """ SELECT permit_id FROM region_permit_theme_overrides WHERE region_id = %s AND permit_id = ANY(%s) AND binds_all_themes """, (region_id, permit_list), ) except pg.DatabaseError as exc: # type: ignore[attr-defined] sqlstate = getattr(exc, "sqlstate", "") if sqlstate == "42P01": try: conn.rollback() except Exception: pass _PERMIT_THEME_OVERRIDE_SCHEMA_READY = False return {} raise rows = cur.fetchall() _PERMIT_THEME_OVERRIDE_SCHEMA_READY = True return {str(permit_id): True for (permit_id,) in rows} def update_permit_v2_visibility( region_id: str, permit_id: str, is_visible: bool, operator: str = "admin" ) -> bool: """Toggle the visibility of a permit in V2 API retrieval for a specific region.""" with _lic_pg_conn() as conn: _ensure_v2_visibility_column(conn) cur = conn.cursor() cur.execute( """ UPDATE region_permit_details SET is_v2_visible = %s, updated_at = now() WHERE region_id = %s AND permit_id = %s """, (is_visible, region_id, permit_id), ) success = cur.rowcount > 0 if success: conn.commit() log_operation( operator=operator, operation_type="UPDATE", target_type="PERMIT_VISIBILITY", target_id=permit_id, target_name=f"Visibility set to {is_visible}", change_summary=f"Updated v2_visibility for permit {permit_id} in region {region_id} to {is_visible}", details={"region_id": region_id, "permit_id": permit_id, "is_v2_visible": is_visible}, ) return success def _permit_binds_all_themes(conn: pg.Connection, region_id: str, permit_id: str) -> bool: """Check override flag for a single region-permit pair.""" global _PERMIT_THEME_OVERRIDE_SCHEMA_READY if not (region_id and permit_id): return False if _PERMIT_THEME_OVERRIDE_SCHEMA_READY is False: return False cur = conn.cursor() try: cur.execute( """ SELECT binds_all_themes FROM region_permit_theme_overrides WHERE region_id = %s AND permit_id = %s LIMIT 1 """, (region_id, permit_id), ) except pg.DatabaseError as exc: # type: ignore[attr-defined] sqlstate = getattr(exc, "sqlstate", "") if sqlstate == "42P01": try: conn.rollback() except Exception: pass _PERMIT_THEME_OVERRIDE_SCHEMA_READY = False return False raise row = cur.fetchone() if row is None: return False _PERMIT_THEME_OVERRIDE_SCHEMA_READY = True return bool(row[0]) def _propagate_all_theme_bindings(conn: pg.Connection, region_id: str, theme_id: str) -> None: """Ensure all-theme permits also link to a newly attached theme.""" if not (region_id and theme_id): return if _PERMIT_THEME_OVERRIDE_SCHEMA_READY is False: return cur = conn.cursor() cur.execute( """ INSERT INTO region_theme_permits (region_id, theme_id, permit_id) SELECT %s, %s, rpto.permit_id FROM region_permit_theme_overrides rpto WHERE rpto.region_id = %s AND rpto.binds_all_themes ON CONFLICT DO NOTHING """, (region_id, theme_id, region_id), ) def _load_permit_file_metadata( conn: pg.Connection, region_id: str, permit_ids: Iterable[str], ) -> Dict[str, Dict[str, Any]]: """Load file metadata for a batch of permits.""" ids = [str(pid) for pid in permit_ids if pid] if not ids: return {} _ensure_permit_file_schema(conn) rows = _select_permit_files(conn, region_id, ids) out: Dict[str, Dict[str, Any]] = {} for row in rows: ( permit_id, file_id, filename, content_type, file_size, created_at, uploaded_by, ) = row out[str(permit_id)] = { "file_id": str(file_id), "filename": filename or "", "content_type": content_type or "", "file_size": int(file_size or 0), "created_at": created_at.isoformat() if created_at else None, "uploaded_by": uploaded_by or "", } return out def _select_permit_files(conn: pg.Connection, region_id: str, permit_ids: Iterable[str]): """Execute the permit file metadata query, recreating tables if missing.""" sql = """ SELECT pfl.permit_id, pf.id, pf.filename, pf.content_type, pf.file_size, pf.created_at, pf.uploaded_by FROM permit_file_links pfl JOIN permit_files pf ON pf.id = pfl.file_id WHERE pfl.region_id = %s AND pfl.permit_id = ANY(%s) """ attempts = 0 while attempts < 2: try: cur = conn.cursor() cur.execute(sql, (region_id, permit_ids)) return cur.fetchall() except pg.DatabaseError as exc: # type: ignore[attr-defined] sqlstate = getattr(exc, "sqlstate", "") if sqlstate != "42P01": raise attempts += 1 try: conn.rollback() except Exception: pass global _PERMIT_FILE_SCHEMA_READY _PERMIT_FILE_SCHEMA_READY = None _ensure_permit_file_schema(conn) if attempts >= 2: logger.warning("[PERMIT-FILES] permit_file_links table missing after recreate attempt, skipping metadata fetch") return [] return [] def _lic_pg_conn(autocommit: bool = False) -> pg.Connection: host = os.getenv("LIC_PG_HOST", "172.24.240.1") port = int(os.getenv("LIC_PG_PORT", os.getenv("PG_PORT", "5432"))) user = os.getenv("LIC_PG_USER", os.getenv("PG_USER", "postgres")) password = os.getenv("LIC_PG_PASSWORD", "") database = os.getenv("LIC_PG_DATABASE", LIC_DEFAULT_DB) conn = pg.connect(host=host, port=port, user=user, password=password, database=database) conn.autocommit = autocommit return conn def list_region_theme_options() -> List[Dict[str, str]]: """Return all region-theme pairs usable for LLM selection.""" sql = """ SELECT rt.region_id, r.name AS region_name, rt.theme_id, t.name AS theme_name FROM region_themes rt JOIN regions r ON r.id = rt.region_id JOIN themes t ON t.id = rt.theme_id WHERE EXISTS ( SELECT 1 FROM region_theme_permits rtp JOIN region_permit_details rpd ON rpd.region_id = rtp.region_id AND rpd.permit_id = rtp.permit_id WHERE rtp.region_id = rt.region_id AND rtp.theme_id = rt.theme_id AND COALESCE(rpd.is_v2_visible, true) = true ) ORDER BY r.name, t.name """ out: List[Dict[str, str]] = [] with _lic_pg_conn() as conn: _ensure_v2_visibility_column(conn) cur = conn.cursor() cur.execute(sql) for region_id, region_name, theme_id, theme_name in cur.fetchall(): rid = str(region_id) tid = str(theme_id) out.append( { "option_id": f"{rid}:{tid}", "region_id": rid, "region_name": str(region_name), "theme_id": tid, "theme_name": str(theme_name), "display_name": f"{region_name} · {theme_name}", } ) return out def load_business_scopes(region_id: str) -> List[Dict[str, str]]: """List business scopes bound to a region.""" sql = """ SELECT bs.id, bs.description FROM region_scopes rs JOIN business_scopes bs ON bs.id = rs.scope_id WHERE rs.region_id = %s ORDER BY bs.description """ scopes: List[Dict[str, str]] = [] with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, (region_id,)) for scope_id, description in cur.fetchall(): scopes.append({"id": str(scope_id), "description": str(description)}) return scopes def get_visible_permits( current_user: Optional[Dict[str, Any]] = None, filters: Optional[Dict[str, Any]] = None ) -> List[Dict[str, str]]: """根据部门绑定关系返回可见的许可列表（仅依赖部门树，不再按区域授予权限）。 Args: current_user: 当前认证用户 filters: 筛选条件（municipal_dept_id, district_dept_id, region, search_text） Returns: 根据用户权限可见的许可列表 """ filters = filters or {} username = current_user.get("username", "anonymous") if current_user else "anonymous" user_department = current_user.get("department", {}) if current_user else {} user_dept_id = user_department.get("id") if not current_user or not user_dept_id: logger.warning("Permission denied: User %s has no department binding", username) return [] with _lic_pg_conn() as conn: _ensure_service_department_schema(conn) _ensure_permit_sources_table(conn) cur = conn.cursor() # 取用户部门及其下级列表 cur.execute( """ SELECT id, region_id FROM service_departments WHERE id = %s """, (user_dept_id,), ) dept_row = cur.fetchone() if not dept_row: logger.warning("Permission denied: User %s department not found", username) return [] dept_id_value, dept_region_id = dept_row accessible_dept_ids = _fetch_department_descendants(cur, str(dept_id_value)) if not accessible_dept_ids: logger.warning("Permission denied: User %s has no accessible departments", username) return [] # 解析筛选部门（用部门树收缩范围） def _resolve_target_departments(dept_token: Optional[str]) -> List[str]: if not dept_token: return [] try: target_uuid = str(dept_token).strip() except Exception: return [] return _fetch_department_descendants(cur, target_uuid) filter_dept_sets: List[List[str]] = [] if filters.get("municipal_dept_id"): filter_dept_sets.append(_resolve_target_departments(filters.get("municipal_dept_id"))) if filters.get("district_dept_id"): filter_dept_sets.append(_resolve_target_departments(filters.get("district_dept_id"))) sql = """ SELECT DISTINCT p.id, p.name FROM permit_sources ps JOIN permits p ON p.id = ps.permit_id LEFT JOIN regions r ON r.id = ps.region_id """ where_clauses: List[str] = [] params: List[Any] = [] # 用户可见范围：自身及下级部门上传/绑定的许可 where_clauses.append( "(ps.uploader_department_id = ANY(%s) OR ps.bound_department_id = ANY(%s))" ) params.extend([accessible_dept_ids, accessible_dept_ids]) # 额外的部门筛选（前端传入） for target_set in filter_dept_sets: if not target_set: continue where_clauses.append( "(ps.uploader_department_id = ANY(%s) OR ps.bound_department_id = ANY(%s))" ) params.extend([target_set, target_set]) if filters.get("region"): where_clauses.append( "(ps.region_id::text = %s OR LOWER(r.name) = LOWER(%s))" ) region = filters["region"] params.extend([region, region]) if filters.get("search_text"): where_clauses.append("LOWER(p.name) LIKE LOWER(%s)") params.append(f"%{filters['search_text']}%") if where_clauses: sql += " WHERE " + " AND ".join(where_clauses) sql += " ORDER BY p.name" permits: List[Dict[str, str]] = [] cur.execute(sql, params) for permit_id, permit_name in cur.fetchall(): permits.append({"id": str(permit_id), "name": str(permit_name)}) logger.info("User %s can view %d permits after filtering", username, len(permits)) return permits def list_permits_for_region(region: str, current_user: Optional[Dict[str, Any]] = None) -> List[Dict[str, str]]: """Return all permits available within a region (accepts id or name). Args: region: Region ID or name current_user: Current authenticated user (optional). If provided, will filter permits based on user's department tree (region仅作为附加过滤，不授予权限) Returns: List of permits visible to the user based on their department tree """ # 使用新的权限模型 filters = {"region": region} if region else {} return get_visible_permits(current_user, filters) def list_region_permit_catalog(region_id: str) -> List[Dict[str, Any]]: """Return permit entries for a region (deduplicated per permit) with risk totals.""" sql = """ SELECT rtp.permit_id, p.name AS permit_name, rtp.theme_id, COALESCE(t.name, '') AS theme_name, COUNT(rpr.risk_id) OVER (PARTITION BY rtp.permit_id) AS risk_total, COALESCE(rpd.is_v2_visible, true) AS is_v2_visible FROM region_theme_permits rtp JOIN permits p ON p.id = rtp.permit_id LEFT JOIN themes t ON t.id = rtp.theme_id LEFT JOIN region_permit_details rpd ON rpd.region_id = rtp.region_id AND rpd.permit_id = rtp.permit_id LEFT JOIN region_permit_risks rpr ON rpr.region_id = rtp.region_id AND rpr.permit_id = rtp.permit_id WHERE rtp.region_id = %s ORDER BY LOWER(p.name), LOWER(COALESCE(t.name, '')) """ catalog_map: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() with _lic_pg_conn() as conn: _ensure_v2_visibility_column(conn) cur = conn.cursor() cur.execute(sql, (region_id,)) for permit_id, permit_name, theme_id, theme_name, risk_total, v2_visible in cur.fetchall(): pid = str(permit_id) entry = catalog_map.setdefault( pid, { "id": pid, "name": str(permit_name), "risk_count": int(risk_total or 0), "is_v2_visible": bool(v2_visible), "theme": {"id": "", "name": ""}, "themes": [], }, ) if entry["theme"].get("id") or entry["theme"].get("name"): pass else: entry["theme"] = { "id": str(theme_id) if theme_id else "", "name": str(theme_name) if theme_name else "", } if theme_id or theme_name: theme_payload = { "id": str(theme_id) if theme_id else "", "name": str(theme_name) if theme_name else "", } if not any( candidate.get("id") == theme_payload["id"] and candidate.get("name") == theme_payload["name"] for candidate in entry["themes"] ): entry["themes"].append(theme_payload) return list(catalog_map.values()) def _fetch_all_theme_options(conn: pg.Connection) -> List[Dict[str, str]]: """Return the list of all theme records for cross-region binding.""" cur = conn.cursor() cur.execute( """ SELECT id, name FROM themes ORDER BY name """ ) return [{"id": str(theme_id), "name": str(theme_name)} for theme_id, theme_name in cur.fetchall()] def _fetch_region_theme_map( conn: pg.Connection, region_ids: Iterable[str] ) -> Dict[str, List[Dict[str, str]]]: """Return mapping of region_id -> theme metadata list.""" uuid_list: List[uuid.UUID] = [] for region_id in region_ids: if not region_id: continue try: uuid_list.append(uuid.UUID(str(region_id))) except (TypeError, ValueError): continue if not uuid_list: return {} cur = conn.cursor() cur.execute( """ SELECT rt.region_id, t.id, t.name FROM region_themes rt JOIN themes t ON t.id = rt.theme_id WHERE rt.region_id = ANY(%s) ORDER BY t.name """, (uuid_list,), ) region_theme_map: Dict[str, List[Dict[str, str]]] = {} for region_uuid, theme_id, theme_name in cur.fetchall(): rid = str(region_uuid) region_theme_map.setdefault(rid, []).append( {"id": str(theme_id), "name": str(theme_name)} ) return region_theme_map def resolve_region_permit_theme(region_id: str, permit_id: str) -> Optional[Dict[str, str]]: """Return the first theme associated with a region-permit pair (if any).""" sql = """ SELECT rtp.theme_id, t.name FROM region_theme_permits rtp LEFT JOIN themes t ON t.id = rtp.theme_id WHERE rtp.region_id = %s AND rtp.permit_id = %s ORDER BY t.name NULLS LAST LIMIT 1 """ with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, (region_id, permit_id)) row = cur.fetchone() if not row: return None theme_id, theme_name = row return { "id": str(theme_id) if theme_id else "", "name": str(theme_name) if theme_name else "", } def _load_permit_scopes_for_region( conn: pg.Connection, region_id: str, permit_ids: List[str] ) -> Dict[str, List[Dict[str, str]]]: """Return mapping of permit_id -> business scopes for that permit within region.""" scope_map: Dict[str, List[Dict[str, str]]] = {pid: [] for pid in permit_ids} if not permit_ids: return scope_map sql = """ SELECT rps.permit_id, bs.id, bs.description FROM region_permit_scopes rps JOIN business_scopes bs ON bs.id = rps.scope_id WHERE rps.region_id = %s ORDER BY rps.permit_id, bs.description """ cur = conn.cursor() try: cur.execute(sql, (region_id,)) except pg.ProgrammingError as exc: # 42P01 => undefined_table; allow fallback when migration not yet applied. sqlstate = getattr(exc, "sqlstate", "") if sqlstate == "42P01": return scope_map raise for permit_id, scope_id, description in cur.fetchall(): pid = str(permit_id) if pid not in scope_map: continue scope_map[pid].append({"id": str(scope_id), "description": str(description)}) return scope_map def _load_permit_sources_for_region( conn: pg.Connection, region_id: str, permit_ids: Iterable[str] ) -> Dict[str, Dict[str, Any]]: """Return mapping of permit_id -> source metadata for the permit.""" permit_ids_list = [str(pid) for pid in permit_ids] if not permit_ids_list: return {} if not _permit_sources_available(conn): return {} cur = conn.cursor() cur.execute( """ SELECT permit_id, source_type, source_name, source_detail, updated_at FROM permit_sources WHERE region_id = %s AND permit_id = ANY(%s) """, (region_id, permit_ids_list), ) sources: Dict[str, Dict[str, Any]] = {} for permit_id, source_type, source_name, source_detail, updated_at in cur.fetchall(): pid = str(permit_id) sources[pid] = { "source_type": source_type or "", "source_name": source_name or "", "source_detail": source_detail or "", "updated_at": _convert_snapshot_value(updated_at), } return sources def load_permits_and_risks( region_id: str, theme_id: Optional[str] = None, permit_id: Optional[str] = None, only_visible: bool = False ) -> List[Dict[str, object]]: """Return permits with attached risk entries for a region (optionally filtered by theme).""" # Ensure optional permit file tables exist before running user queries. try: _ensure_permit_file_schema() except Exception as exc: logger.warning("[PERMIT-FILES] Failed to ensure permit file schema before loading permits: %s", exc) try: _ensure_permit_approval_departments_schema() except Exception as exc: logger.warning("[PERMIT-APPROVAL] Failed to ensure approval departments schema: %s", exc) sql = """ SELECT rtp.theme_id, t.name AS theme_name, p.id AS permit_id, p.name AS permit_name, rk.id AS risk_id, rk.risk_content, rk.legal_basis, rk.document_no, rk.summary, rk.remark, rpr.serial_number, rpd.permit_status, rpd.subitem_summary, rpd.responsible_contact, rpd.jurisdiction_scope, rpd.filler_name, COALESCE(pad.department_name, rpd.unit_name) AS unit_name, rpd.source_update_date, rpd.contact_info, COALESCE(rpd.is_v2_visible, true) AS is_v2_visible FROM region_permit_details rpd JOIN permits p ON p.id = rpd.permit_id LEFT JOIN permit_approval_departments pad ON (p.name = pad.permit_name OR p.name LIKE CONCAT(pad.permit_name, '%')) LEFT JOIN region_theme_permits rtp ON rtp.region_id = rpd.region_id AND rtp.permit_id = rpd.permit_id LEFT JOIN themes t ON t.id = rtp.theme_id LEFT JOIN region_permit_risks rpr ON rpr.region_id = rpd.region_id AND rpr.permit_id = rpd.permit_id LEFT JOIN risks rk ON rk.id = rpr.risk_id WHERE rpd.region_id = %s """ params: List[Any] = [region_id] theme_filter = theme_id if (theme_id and not _is_all_theme_marker(theme_id)) else None if theme_filter: sql += " AND (rtp.theme_id = %s OR t.name = '所有主题事项')" params.append(theme_filter) if permit_id is not None: sql += " AND rpd.permit_id = %s" params.append(permit_id) if only_visible: sql += " AND COALESCE(rpd.is_v2_visible, true) = true" sql += """ ORDER BY p.name, LENGTH(rpr.serial_number), rpr.serial_number, rk.risk_content """ permits: Dict[str, Dict[str, object]] = {} risk_seen_map: Dict[str, Set[str]] = {} # pid -> set of risk_ids with _lic_pg_conn() as conn: _ensure_v2_visibility_column(conn) _ensure_contact_info_column(conn) cur = conn.cursor() cur.execute(sql, tuple(params)) for row in cur.fetchall(): ( row_theme_id, row_theme_name, permit_id, permit_name, risk_id, risk_content, legal_basis, document_no, summary, remark, serial_number, permit_status, subitem_summary, responsible_contact, jurisdiction_scope, filler_name, unit_name, source_update_date, contact_info, v2_visible, ) = row pid = str(permit_id) theme_id_value = str(row_theme_id) if row_theme_id else "" theme_name_value = str(row_theme_name) if row_theme_name else "" entry = permits.setdefault( pid, { "id": pid, "name": str(permit_name), "business_scopes": [], "risks": [], "permit_status": None, "subitem_summary": None, "responsible_contact": None, "jurisdiction_scope": None, "filler_name": None, "unit_name": None, "source_update_date": None, "contact_info": None, "theme": { "id": theme_id_value, "name": theme_name_value, }, "themes": [], "binds_all_themes": False, }, ) # ... (theme logic skipped for brevity if identical, but I must match exact lines to be safe) if theme_id_value and not entry["theme"].get("id"): entry["theme"]["id"] = theme_id_value if theme_name_value and not entry["theme"].get("name"): entry["theme"]["name"] = theme_name_value if theme_id_value or theme_name_value: # Same theme duplication logic theme_list = entry.get("themes") or [] duplicate = False for theme_entry in theme_list: if theme_id_value: if theme_entry.get("id") == theme_id_value: duplicate = True break else: if not theme_entry.get("id") and theme_entry.get("name") == theme_name_value: duplicate = True break if not duplicate: theme_list.append( { "id": theme_id_value, "name": theme_name_value, } ) entry["themes"] = theme_list if entry["permit_status"] is None and permit_status: entry["permit_status"] = permit_status.strip() or None if entry["subitem_summary"] is None and subitem_summary: entry["subitem_summary"] = subitem_summary.strip() or None if entry["responsible_contact"] is None and responsible_contact: entry["responsible_contact"] = responsible_contact.strip() or None if entry["jurisdiction_scope"] is None and jurisdiction_scope: entry["jurisdiction_scope"] = jurisdiction_scope.strip() or None if entry["filler_name"] is None and filler_name: entry["filler_name"] = filler_name.strip() or None if entry["unit_name"] is None and unit_name: entry["unit_name"] = unit_name.strip() or None if entry["source_update_date"] is None and source_update_date: entry["source_update_date"] = source_update_date.strip() or None if entry["contact_info"] is None and contact_info: entry["contact_info"] = contact_info.strip() or None if risk_id is not None: risk_id_str = str(risk_id) # Avoid duplicates when a permit has multiple themes seen_risk_ids = risk_seen_map.setdefault(pid, set()) if risk_id_str not in seen_risk_ids: seen_risk_ids.add(risk_id_str) summary_markdown = _format_summary_markdown(summary or "") remark_markdown = _format_summary_markdown(remark or "") entry["risks"].append( { "id": risk_id_str, "risk_content": risk_content or "", "legal_basis": legal_basis or "", "document_no": document_no or "", "summary": summary_markdown, "remark": remark_markdown, "serial_number": serial_number, } ) permit_ids = list(permits.keys()) scope_map = _load_permit_scopes_for_region(conn, region_id, permit_ids) source_map = _load_permit_sources_for_region(conn, region_id, permit_ids) try: file_meta_map = _load_permit_file_metadata(conn, region_id, permit_ids) except pg.DatabaseError as exc: # type: ignore[attr-defined] sqlstate = getattr(exc, "sqlstate", "") if sqlstate == "42P01": logger.warning("[PERMIT-FILES] permit_file_links missing while loading permits, recreating schema lazily") global _PERMIT_FILE_SCHEMA_READY _PERMIT_FILE_SCHEMA_READY = None _ensure_permit_file_schema() file_meta_map = {} else: raise all_theme_flags = _fetch_permit_all_theme_flags(conn, region_id, permit_ids) for pid in permit_ids: permits[pid]["business_scopes"] = scope_map.get(pid, []) if pid in source_map: permits[pid]["permit_source"] = source_map[pid] else: permits[pid]["permit_source"] = { "source_type": "", "source_name": "", "source_detail": "", "updated_at": None, } if pid in file_meta_map: permits[pid]["permit_file"] = file_meta_map[pid] else: permits[pid]["permit_file"] = { "file_id": "", "filename": "", "content_type": "", "file_size": 0, "created_at": None, "uploaded_by": "", } if "themes" not in permits[pid] or permits[pid]["themes"] is None: permits[pid]["themes"] = [] if all_theme_flags.get(pid): permits[pid]["binds_all_themes"] = True theme_list = permits[pid].get("themes") or [] has_marker = any(_is_all_theme_marker(theme.get("id")) or _is_all_theme_marker(theme.get("name")) for theme in theme_list) if not has_marker: theme_list.append( { "id": ALL_THEMES_SENTINEL, "name": ALL_THEMES_DISPLAY_NAME, "is_virtual": True, } ) permits[pid]["themes"] = theme_list theme_meta = permits[pid].get("theme") or {} if not (theme_meta.get("id") or theme_meta.get("name")): permits[pid]["theme"] = { "id": ALL_THEMES_SENTINEL, "name": ALL_THEMES_DISPLAY_NAME, } else: permits[pid]["binds_all_themes"] = False return list(permits.values()) def fetch_permit_file(region_id: str, permit_id: str) -> Optional[Dict[str, Any]]: """Return file payload for a region-permit pair if available.""" if not region_id or not permit_id: return None with _lic_pg_conn() as conn: _ensure_permit_file_schema(conn) try: row = _select_permit_file_blob(conn, region_id, permit_id) except pg.DatabaseError as exc: # type: ignore[attr-defined] sqlstate = getattr(exc, "sqlstate", "") if sqlstate == "42P01": logger.warning( "[PERMIT-FILES] permit_file_links missing when downloading file (region=%s permit=%s); recreating schema", region_id, permit_id, ) global _PERMIT_FILE_SCHEMA_READY _PERMIT_FILE_SCHEMA_READY = None _ensure_permit_file_schema() return None raise if not row: return None ( file_id, filename, content_type, file_size, file_data, created_at, uploaded_by, ) = row return { "file_id": str(file_id), "filename": filename or "", "content_type": content_type or "application/octet-stream", "file_size": int(file_size or 0), "file_data": bytes(file_data) if file_data is not None else b"", "created_at": created_at.isoformat() if created_at else None, "uploaded_by": uploaded_by or "", } def list_stored_permit_files( limit: int = 20, offset: int = 0, keyword: Optional[str] = None, ) -> Dict[str, Any]: """Return paginated permit file metadata with optional keyword filtering.""" try: limit = int(limit) except (TypeError, ValueError): limit = 20 limit = max(1, min(limit, 100)) try: offset = int(offset) except (TypeError, ValueError): offset = 0 offset = max(0, offset) normalized_keyword = _clean_text(keyword) or None with _lic_pg_conn() as conn: _ensure_permit_file_schema(conn) cur = conn.cursor() where_clauses: List[str] = [] params: List[Any] = [] if normalized_keyword: like_pattern = f"%{normalized_keyword}%" where_clauses.append( """ ( pf.filename ILIKE %s OR EXISTS ( SELECT 1 FROM permit_file_links sub JOIN permits sp ON sp.id = sub.permit_id WHERE sub.file_id = pf.id AND sp.name ILIKE %s ) ) """ ) params.extend([like_pattern, like_pattern]) where_sql = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else "" count_sql = f"SELECT COUNT(*) FROM permit_files pf {where_sql}" cur.execute(count_sql, tuple(params)) total_rows = int(cur.fetchone()[0] or 0) data_sql = f""" SELECT pf.id, pf.filename, pf.content_type, pf.file_size, pf.created_at, pf.uploaded_by, COALESCE( json_agg( jsonb_build_object( 'region_id', pfl.region_id::text, 'region_name', COALESCE(r.name, ''), 'permit_id', pfl.permit_id::text, 'permit_name', COALESCE(p.name, '') ) ) FILTER (WHERE pfl.permit_id IS NOT NULL), '[]'::json ) AS link_payload FROM permit_files pf LEFT JOIN permit_file_links pfl ON pfl.file_id = pf.id LEFT JOIN regions r ON r.id = pfl.region_id LEFT JOIN permits p ON p.id = pfl.permit_id {where_sql} GROUP BY pf.id, pf.filename, pf.content_type, pf.file_size, pf.created_at, pf.uploaded_by ORDER BY pf.created_at DESC, pf.id DESC LIMIT %s OFFSET %s """ data_params = list(params) + [limit, offset] cur.execute(data_sql, data_params) rows = cur.fetchall() files: List[Dict[str, Any]] = [] for row in rows: ( file_id, filename, content_type, file_size, created_at, uploaded_by, link_payload, ) = row link_items: List[Dict[str, str]] = [] if link_payload: if isinstance(link_payload, str): try: parsed = json.loads(link_payload) except (TypeError, ValueError): parsed = [] else: parsed = link_payload if isinstance(link_payload, list) else [] if isinstance(parsed, list): for entry in parsed: if not isinstance(entry, dict): continue link_items.append( { "region_id": str(entry.get("region_id") or ""), "region_name": str(entry.get("region_name") or ""), "permit_id": str(entry.get("permit_id") or ""), "permit_name": str(entry.get("permit_name") or ""), } ) files.append( { "file_id": str(file_id), "filename": filename or "", "content_type": content_type or "", "file_size": int(file_size or 0), "created_at": created_at.isoformat() if created_at else None, "uploaded_by": uploaded_by or "", "links": link_items, } ) return { "files": files, "pagination": { "limit": limit, "offset": offset, "total": total_rows, }, } def delete_stored_permit_file(file_id: str) -> bool: """Delete a stored permit file (and cascading links).""" normalized = _clean_text(file_id) if not normalized: raise ValueError("file_id 不能为空") with _lic_pg_conn(autocommit=True) as conn: _ensure_permit_file_schema(conn) cur = conn.cursor() cur.execute("DELETE FROM permit_files WHERE id = %s", (normalized,)) return cur.rowcount > 0 def start_import_session_from_file( file_id: str, *, requested_by: Optional[str] = None, uploader_department_id: Optional[str] = None, bound_department_id: Optional[str] = None, binding_mode: str = "auto", ) -> Dict[str, Any]: """Create a fresh import session using an archived permit file.""" normalized = _clean_text(file_id) if not normalized: raise ValueError("file_id 不能为空") with _lic_pg_conn() as conn: _ensure_permit_file_schema(conn) cur = conn.cursor() cur.execute( """ SELECT filename, content_type, file_data, uploaded_by FROM permit_files WHERE id = %s """, (normalized,), ) row = cur.fetchone() if not row: raise ValueError("文件不存在或已删除") filename, content_type, file_data, uploaded_by = row if isinstance(file_data, memoryview): file_bytes = file_data.tobytes() elif isinstance(file_data, bytearray): file_bytes = bytes(file_data) else: file_bytes = bytes(file_data or b"") effective_uploader = requested_by or uploaded_by return start_permit_import_session( file_bytes=file_bytes, filename=filename or "许可导入.xlsx", content_type=content_type or "application/octet-stream", uploaded_by=effective_uploader, uploader_department_id=uploader_department_id, bound_department_id=bound_department_id, binding_mode=binding_mode, ) def _select_permit_file_blob(conn: pg.Connection, region_id: str, permit_id: str): """Fetch a single permit file with binary content, recreating tables if needed.""" sql = """ SELECT pf.id, pf.filename, pf.content_type, pf.file_size, pf.file_data, pf.created_at, pf.uploaded_by FROM permit_file_links pfl JOIN permit_files pf ON pf.id = pfl.file_id WHERE pfl.region_id = %s AND pfl.permit_id = %s LIMIT 1 """ attempts = 0 while attempts < 2: try: cur = conn.cursor() cur.execute(sql, (region_id, permit_id)) return cur.fetchone() except pg.DatabaseError as exc: # type: ignore[attr-defined] sqlstate = getattr(exc, "sqlstate", "") if sqlstate != "42P01": raise attempts += 1 try: conn.rollback() except Exception: pass global _PERMIT_FILE_SCHEMA_READY _PERMIT_FILE_SCHEMA_READY = None _ensure_permit_file_schema(conn) if attempts >= 2: logger.warning( "[PERMIT-FILES] permit_file_links table missing after recreate attempt when downloading file (region=%s permit=%s)", region_id, permit_id, ) return None return None def find_permit_contexts_by_name(permit_name: str) -> List[Dict[str, str]]: """Return region/theme contexts for permits with an exact name match or prefix match.""" if not permit_name: return [] sql = """ SELECT rpd.region_id, r.name AS region_name, rtp.theme_id, t.name AS theme_name, p.id AS permit_id, p.name AS permit_name, COALESCE(rpd.is_v2_visible, true) AS is_v2_visible FROM region_permit_details rpd JOIN permits p ON p.id = rpd.permit_id JOIN regions r ON r.id = rpd.region_id LEFT JOIN region_theme_permits rtp ON rtp.region_id = rpd.region_id AND rtp.permit_id = rpd.permit_id LEFT JOIN themes t ON t.id = rtp.theme_id WHERE p.name = %s ORDER BY r.name, t.name NULLS LAST """ ordered: OrderedDict[Tuple[str, str], Dict[str, str]] = OrderedDict() with _lic_pg_conn() as conn: _ensure_v2_visibility_column(conn) cur = conn.cursor() cur.execute(sql, (permit_name,)) rows = cur.fetchall() # If no exact match, try prefix match (e.g. "药品经营许可" matching "药品经营许可证") if not rows and len(permit_name) >= 2: sql_fuzzy = """ SELECT rpd.region_id, r.name AS region_name, rtp.theme_id, t.name AS theme_name, p.id AS permit_id, p.name AS permit_name, COALESCE(rpd.is_v2_visible, true) AS is_v2_visible FROM region_permit_details rpd JOIN permits p ON p.id = rpd.permit_id JOIN regions r ON r.id = rpd.region_id LEFT JOIN region_theme_permits rtp ON rtp.region_id = rpd.region_id AND rtp.permit_id = rpd.permit_id LEFT JOIN themes t ON t.id = rtp.theme_id WHERE p.name LIKE %s ORDER BY r.name, t.name NULLS LAST """ cur.execute(sql_fuzzy, (permit_name + "%",)) rows = cur.fetchall() for row in rows: region_id, region_name, theme_id, theme_name, permit_id, canonical_name, v2_visible = row rid = str(region_id) pid = str(permit_id) tid = str(theme_id) if theme_id else "" tname = str(theme_name) if theme_name else "" key = (rid, pid) if key in ordered: continue ordered[key] = { "region_id": rid, "region_name": str(region_name), "theme_id": tid, "theme_name": tname, "permit_id": pid, "permit_name": str(canonical_name), "is_v2_visible": bool(v2_visible), } return [item for item in ordered.values() if item.get("is_v2_visible")] def load_theme_payload(region_id: str, theme_id: str, only_visible: bool = False) -> Dict[str, object]: """Assemble full data bundle for a region-theme selection.""" info_sql = """ SELECT r.id, r.name, t.id, t.name FROM regions r JOIN region_themes rt ON rt.region_id = r.id JOIN themes t ON t.id = rt.theme_id WHERE r.id = %s AND t.id = %s LIMIT 1 """ with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(info_sql, (region_id, theme_id)) row = cur.fetchone() if not row: raise ValueError("Region/theme combination not found") region_uuid, region_name, theme_uuid, theme_name = row permits = load_permits_and_risks(region_id, theme_id, only_visible=only_visible) return { "region": {"id": str(region_uuid), "name": str(region_name)}, "theme": {"id": str(theme_uuid), "name": str(theme_name)}, "permits": permits, } def _get_checkpoints_dir() -> str: """Get the directory for storing checkpoint files.""" base_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data") checkpoints_dir = os.path.join(base_dir, "checkpoints") os.makedirs(checkpoints_dir, exist_ok=True) return checkpoints_dir def _get_all_tables() -> List[str]: """Get list of all tables in the licensing_risks database.""" sql = """ SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE' ORDER BY table_name """ with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql) return [row[0] for row in cur.fetchall()] def _get_table_dependencies(conn: pg.Connection) -> Dict[str, List[str]]: """ 获取表依赖关系图。 返回: {被引用表名: [引用它的表列表]} 例如: {'regions': ['region_themes', 'region_scopes'], 'themes': ['region_themes']} """ sql = """ SELECT ccu.table_name AS referenced_table, tc.table_name AS dependent_table FROM information_schema.table_constraints tc JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema JOIN information_schema.constraint_column_usage ccu ON tc.constraint_name = ccu.constraint_name AND tc.table_schema = ccu.table_schema WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = 'public' ORDER BY ccu.table_name, tc.table_name """ cur = conn.cursor() cur.execute(sql) dependencies = {} for row in cur.fetchall(): referenced_table, dependent_table = row if referenced_table not in dependencies: dependencies[referenced_table] = [] if dependent_table not in dependencies[referenced_table]: dependencies[referenced_table].append(dependent_table) return dependencies def _topological_sort_tables(all_tables: List[str], dependencies: Dict[str, List[str]]) -> List[str]: """ 使用拓扑排序确定表恢复顺序。 先返回父表（无外键依赖），再返回子表（引用其他表）。 这确保恢复时不会违反外键约束。 """ from collections import deque # 计算每个表的入度（被多少表引用/依赖） in_degree = {table: 0 for table in all_tables} for parent_table, children in dependencies.items(): for child in children: if child in all_tables: in_degree[child] += 1 # 入度为0的表是父表，可以先恢复 queue = deque([table for table in all_tables if in_degree[table] == 0]) result = [] while queue: table = queue.popleft() result.append(table) # 减少依赖该表的子表的入度 for parent, children in dependencies.items(): if parent == table: for child in children: if child in all_tables and in_degree[child] > 0: in_degree[child] -= 1 if in_degree[child] == 0: queue.append(child) # 如果有循环依赖或孤立节点，将剩余表添加到末尾 for table in all_tables: if table not in result: result.append(table) return result def _reset_all_sequences(conn: pg.Connection, tables: List[str]) -> None: """ Reset sequences for all tables to the max(id) + 1. This prevents 'duplicate key value violates unique constraint' errors after restore. """ cur = conn.cursor() for table in tables: # Check if table has an 'id' column or serial column that is a sequence # Heuristic: verify if 'id' column exists try: # Check if 'id' column exists cur.execute( """ SELECT column_name, data_type FROM information_schema.columns WHERE table_name = %s AND column_name = 'id' """, (table,) ) col = cur.fetchone() if not col: continue # No ID column, skip # Try to reset sequence. # If the column is not a serial/identity, setval might fail or do nothing if no sequence attached. # A robust way is to find the sequence name dynamically. # This query finds the sequence associated with a column (if any) cur.execute( """ SELECT pg_get_serial_sequence(%s, 'id') """, (table,) ) seq_row = cur.fetchone() if seq_row and seq_row[0]: seq_name = seq_row[0] # Get max id cur.execute(f"SELECT MAX(id) FROM {table}") max_id = cur.fetchone()[0] if max_id is not None: # Reset sequence # Using setval with is_called=true ensures next value is max+1 cur.execute(f"SELECT setval(%s, %s, true)", (seq_name, max_id)) logger.info(f"[CHECKPOINT] Reset sequence {seq_name} for table {table} to {max_id}") else: # Table empty, reset to 1 (is_called=false means next will be 1) cur.execute(f"SELECT setval(%s, 1, false)", (seq_name,)) logger.info(f"[CHECKPOINT] Reset sequence {seq_name} for table {table} to 1 (empty)") except Exception as e: # Log but don't fail the entire restore for one sequence issue logger.warning(f"[CHECKPOINT] Failed to reset sequence for table {table}: {e}") def _backup_table(conn: pg.Connection, table_name: str) -> Tuple[List[Dict[str, Any]], int]: """Backup a single table and return its data and row count.""" logger.info(f"[CHECKPOINT] Backing up table: {table_name}") sql = f"SELECT * FROM {table_name}" cur = conn.cursor() cur.execute(sql) rows = cur.fetchall() colnames = [desc[0] for desc in cur.description] data = [] for row in rows: row_dict = {} for i, col in enumerate(colnames): value = row[i] if value is not None: # Convert UUID and other non-serializable types to strings if hasattr(value, 'isoformat'): # UUID, datetime, etc. row_dict[col] = str(value) else: row_dict[col] = value data.append(row_dict) row_count = len(data) logger.info(f"[CHECKPOINT] Backup complete: {table_name} - {row_count} rows, {len(colnames)} columns") return data, row_count def _restore_table(conn: pg.Connection, table_name: str, data: List[Dict[str, Any]], batch_size: int = 1000) -> int: """Restore a table from backup data. Returns number of rows restored.""" if not data: logger.info(f"[CHECKPOINT] Skipping empty table: {table_name}") return 0 conn.autocommit = False try: cur = conn.cursor() logger.info(f"[CHECKPOINT] Truncating table: {table_name}") # Use TRUNCATE with CASCADE to handle foreign key dependencies # This will automatically remove dependent records truncate_sql = f"TRUNCATE TABLE {table_name} CASCADE" cur.execute(truncate_sql) columns = list(data[0].keys()) placeholders = ", ".join(["%s"] * len(columns)) logger.info(f"[CHECKPOINT] Restoring {len(data)} rows into {table_name} (columns: {', '.join(columns)})") # 🚀 优化: 使用批量插入而不是逐行插入 # pg8000 使用 executemany 进行批量插入 batch_size = 1000 # 每批1000行 if len(data) <= batch_size: # 小数据量，直接批量插入 # 小数据量，直接批量插入 values_list = [] for row in data: processed_row = [] for col in columns: val = row.get(col) # Handle Base64 decoding for bytea fields (specifically file_data in permit_files) if table_name == 'permit_files' and col == 'file_data' and isinstance(val, str): try: import base64 val = base64.b64decode(val) except Exception: pass processed_row.append(val) values_list.append(processed_row) cur.executemany(f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({placeholders})", values_list) logger.info(f"[CHECKPOINT] Bulk insert complete: {table_name} - {len(data)} rows inserted") else: # 大数据量，分批插入 total_rows = len(data) for i in range(0, total_rows, batch_size): batch_end = min(i + batch_size, total_rows) batch_data = data[i:batch_end] values_list = [] for row in batch_data: processed_row = [] for col in columns: val = row.get(col) # Handle Base64 decoding for bytea fields (specifically file_data in permit_files) if table_name == 'permit_files' and col == 'file_data' and isinstance(val, str): try: import base64 val = base64.b64decode(val) except Exception: pass # Keep as is if decode fails processed_row.append(val) values_list.append(processed_row) cur.executemany(f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({placeholders})", values_list) logger.info(f"[CHECKPOINT] Progress: {table_name} - {batch_end}/{total_rows} rows inserted") conn.commit() logger.info(f"[CHECKPOINT] Restore complete: {table_name} - {len(data)} rows successfully inserted") return len(data) except Exception as e: conn.rollback() logger.error(f"[CHECKPOINT] Restore FAILED: {table_name} - {str(e)}") raise e finally: conn.autocommit = False def create_checkpoint(description: str = "", operator: str = "admin") -> Dict[str, Any]: """ 安全创建checkpoint，所有表操作在一个事务中。 Args: description: checkpoint的描述信息 Returns: 包含checkpoint信息的字典 """ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") checkpoint_id = f"checkpoint_{timestamp}" logger.info("=" * 80) logger.info(f"[CHECKPOINT] Starting checkpoint creation: {checkpoint_id}") if description: logger.info(f"[CHECKPOINT] Description: {description}") logger.info("=" * 80) tables = _get_all_tables() logger.info(f"[CHECKPOINT] Found {len(tables)} tables to backup: {', '.join(tables)}") checkpoint_data = { "checkpoint_id": checkpoint_id, "timestamp": timestamp, "description": description, "tables": {} } total_rows = 0 table_counts = {} with _lic_pg_conn(autocommit=False) as conn: try: for i, table in enumerate(tables, 1): logger.info(f"[CHECKPOINT] [{i}/{len(tables)}] Processing table: {table}") data, row_count = _backup_table(conn, table) checkpoint_data["tables"][table] = data table_counts[table] = row_count total_rows += row_count logger.info(f"[CHECKPOINT] [{i}/{len(tables)}] Table {table} backed up: {row_count} rows") # 全部成功后才提交 logger.info("[CHECKPOINT] All tables backed up successfully, committing transaction...") conn.commit() logger.info("[CHECKPOINT] Transaction committed") except Exception as e: # 任何失败都回滚 logger.error(f"[CHECKPOINT] ERROR during backup, rolling back: {str(e)}") conn.rollback() raise e finally: conn.autocommit = False checkpoint_data["table_counts"] = table_counts checkpoint_data["total_rows"] = total_rows checkpoints_dir = _get_checkpoints_dir() checkpoint_file = os.path.join(checkpoints_dir, f"{checkpoint_id}.json") logger.info(f"[CHECKPOINT] Saving checkpoint file: {checkpoint_file}") def json_serializer(obj): """Convert non-JSON serializable objects to strings.""" try: import uuid if isinstance(obj, uuid.UUID): return str(obj) except ImportError: pass if isinstance(obj, (bytes, bytearray, memoryview)): import base64 # For data integrity in JSON, base64 is standard return base64.b64encode(obj).decode('ascii') if hasattr(obj, 'isoformat'): return str(obj) raise TypeError(f"Object of type {type(obj)} is not JSON serializable") with open(checkpoint_file, "w", encoding="utf-8") as f: json.dump(checkpoint_data, f, ensure_ascii=False, indent=2, default=json_serializer) logger.info("=" * 80) # Log the operation log_operation( operator=operator, operation_type="CHECKPOINT", target_type="DATABASE", target_id=checkpoint_id, target_name=checkpoint_id, change_summary=f"Created database checkpoint: {checkpoint_id}. {description}", details={ "checkpoint_id": checkpoint_id, "total_rows": total_rows, "table_counts": table_counts } ) return { "checkpoint_id": checkpoint_id, "timestamp": timestamp, "description": description, "total_rows": total_rows, "table_counts": table_counts } def list_checkpoints() -> List[Dict[str, Any]]: """List all available checkpoints.""" checkpoints_dir = _get_checkpoints_dir() checkpoints = [] if not os.path.exists(checkpoints_dir): return checkpoints for filename in os.listdir(checkpoints_dir): if filename.endswith(".json"): filepath = os.path.join(checkpoints_dir, filename) try: with open(filepath, "r", encoding="utf-8") as f: data = json.load(f) checkpoints.append({ "checkpoint_id": data["checkpoint_id"], "timestamp": data["timestamp"], "description": data.get("description", ""), "total_rows": data.get("total_rows", 0), "table_counts": data.get("table_counts", {}), "filename": filename }) except Exception as e: print(f"Error reading checkpoint {filename}: {e}") return sorted(checkpoints, key=lambda x: x["timestamp"], reverse=True) def restore_checkpoint( checkpoint_id: str, create_auto_backup: bool = True, batch_size: int = 1000, operator: str = "admin", ) -> Dict[str, Any]: """ 安全恢复数据库从checkpoint。 ⚠️ 危险操作: 会覆盖现有数据! Args: checkpoint_id: 要恢复的checkpoint ID create_auto_backup: 是否在恢复前自动备份当前状态 batch_size: 批量插入的批次大小 (默认1000行/批) Returns: 包含恢复结果的字典 """ checkpoints_dir = _get_checkpoints_dir() checkpoint_file = os.path.join(checkpoints_dir, f"{checkpoint_id}.json") logger.warning("=" * 80) logger.warning(f"[CHECKPOINT] WARNING: Starting restore operation: {checkpoint_id}") logger.warning("[CHECKPOINT] This will OVERWRITE all existing data in the database!") logger.warning("=" * 80) if not os.path.exists(checkpoint_file): error_msg = f"Checkpoint {checkpoint_id} not found" logger.error(f"[CHECKPOINT] {error_msg}") raise ValueError(error_msg) with open(checkpoint_file, "r", encoding="utf-8") as f: checkpoint_data = json.load(f) # 自动备份当前状态（可选但推荐） auto_backup_info = None if create_auto_backup: logger.info("[CHECKPOINT] Creating auto-backup before restore... (THIS MAY TAKE TIME)") try: import time start_time = time.time() auto_backup_info = create_checkpoint(f"auto_backup_before_restore_{checkpoint_id}") elapsed = time.time() - start_time logger.info(f"[CHECKPOINT] Auto-backup created in {elapsed:.2f}s: {auto_backup_info['checkpoint_id']}") logger.info(f"[CHECKPOINT] Auto-backup contains {auto_backup_info['total_rows']} rows") except Exception as e: logger.error(f"[CHECKPOINT] WARNING: Failed to create auto-backup: {e}") logger.warning("[CHECKPOINT] Continuing without auto-backup...") else: logger.info("[CHECKPOINT] Auto-backup DISABLED by user") tables = checkpoint_data.get("tables", {}) total_rows_in_checkpoint = sum(len(data) for data in tables.values()) logger.info("=" * 80) logger.info(f"[CHECKPOINT] Checkpoint details:") logger.info(f"[CHECKPOINT] ID: {checkpoint_id}") logger.info(f"[CHECKPOINT] Tables: {len(tables)}") logger.info(f"[CHECKPOINT] Total rows: {total_rows_in_checkpoint}") logger.info(f"[CHECKPOINT] Auto-backup: {'Yes (' + auto_backup_info['checkpoint_id'] + ')' if auto_backup_info else 'No'}") logger.info("=" * 80) restore_summary = { "checkpoint_id": checkpoint_id, "auto_backup": auto_backup_info.get("checkpoint_id") if auto_backup_info else None, "tables_restored": 0, "total_rows_restored": 0, "table_details": {}, "errors": [] } with _lic_pg_conn(autocommit=False) as conn: try: # 1. 构建表依赖关系图 logger.info("[CHECKPOINT] Building table dependency graph...") dependencies = _get_table_dependencies(conn) all_tables = list(tables.keys()) logger.info(f"[CHECKPOINT] Found {len(dependencies)} table dependencies") # 2. 拓扑排序获取恢复顺序 logger.info("[CHECKPOINT] Calculating restore order...") restore_order = _topological_sort_tables(all_tables, dependencies) logger.info(f"[CHECKPOINT] Restore order: {' -> '.join(restore_order)}") # 3. 锁定所有表（防止并发写入） logger.info("[CHECKPOINT] Acquiring exclusive locks on all tables...") cur = conn.cursor() for table in restore_order: cur.execute(f"LOCK TABLE {table} IN EXCLUSIVE MODE") logger.info("[CHECKPOINT] All tables locked exclusively") # 4. 按依赖顺序恢复表 logger.info("=" * 80) logger.info("[CHECKPOINT] Starting restore process...") logger.info("=" * 80) import time restore_start_time = time.time() for i, table_name in enumerate(restore_order, 1): data = tables.get(table_name, []) table_start_time = time.time() logger.info(f"[CHECKPOINT] [{i}/{len(restore_order)}] Preparing to restore table: {table_name}") try: rows_restored = _restore_table(conn, table_name, data, batch_size=batch_size) table_elapsed = time.time() - table_start_time restore_summary["tables_restored"] += 1 restore_summary["total_rows_restored"] += rows_restored restore_summary["table_details"][table_name] = rows_restored logger.info(f"[CHECKPOINT] [{i}/{len(restore_order)}] Table {table_name} restored: {rows_restored} rows in {table_elapsed:.2f}s") except Exception as e: error_msg = f"Failed to restore table {table_name}: {str(e)}" logger.error(f"[CHECKPOINT] ERROR: {error_msg}") restore_summary["errors"].append(error_msg) raise e restore_elapsed = time.time() - restore_start_time total_tables = len(restore_order) logger.info(f"[CHECKPOINT] All {total_tables} tables restored in {restore_elapsed:.2f}s") restore_elapsed = time.time() - restore_start_time total_tables = len(restore_order) logger.info(f"[CHECKPOINT] All {total_tables} tables restored in {restore_elapsed:.2f}s") # 5. 重置所有表的序列 (Sequences) logger.info("[CHECKPOINT] Resetting sequences for all tables...") _reset_all_sequences(conn, all_tables) logger.info("[CHECKPOINT] Sequences reset successfully") # 6. 提交事务 logger.info("=" * 80) logger.info("[CHECKPOINT] All tables restored successfully, committing transaction...") conn.commit() logger.info("[CHECKPOINT] Transaction committed successfully") logger.info("=" * 80) logger.warning("=" * 80) # Log the operation log_operation( operator=operator, operation_type="RESTORE", target_type="DATABASE", target_id=checkpoint_id, target_name=checkpoint_id, change_summary=f"Restored database from checkpoint: {checkpoint_id}", details=restore_summary ) return { "status": "success", "message": f"Successfully restored {restore_summary['tables_restored']} tables, " f"{restore_summary['total_rows_restored']} total rows", "summary": restore_summary } except Exception as e: # 回滚事务 logger.error(f"[CHECKPOINT] ERROR during restore, rolling back: {str(e)}") conn.rollback() logger.warning("[CHECKPOINT] Transaction rolled back, changes reverted") error_info = { "status": "error", "message": f"Restore failed: {str(e)}", "summary": restore_summary, "auto_backup_available": bool(auto_backup_info) } # 如果有自动备份，提供恢复建议 if auto_backup_info: error_info["recovery_suggestion"] = ( f"Use auto-backup to restore current state: " f"restore_checkpoint('{auto_backup_info['checkpoint_id']}')" ) logger.warning(f"[CHECKPOINT] Recovery suggestion: Use auto-backup '{auto_backup_info['checkpoint_id']}'") logger.error("=" * 80) logger.error(f"[CHECKPOINT] RESTORE FAILED: {str(e)}") if auto_backup_info: logger.error(f"[CHECKPOINT] You can restore from auto-backup: {auto_backup_info['checkpoint_id']}") logger.error("=" * 80) return error_info def delete_checkpoint(checkpoint_id: str) -> bool: """Delete a checkpoint file.""" checkpoints_dir = _get_checkpoints_dir() checkpoint_file = os.path.join(checkpoints_dir, f"{checkpoint_id}.json") if os.path.exists(checkpoint_file): os.remove(checkpoint_file) return True return False # --------------------------------------------------------------------------- # Permit risk snapshot helpers (region_permit_risk_vw) # --------------------------------------------------------------------------- def _convert_snapshot_value(value: Any) -> Any: """Convert database values into JSON-serialisable primitives.""" if isinstance(value, (datetime, date)): return value.isoformat() if isinstance(value, Decimal): return float(value) if isinstance(value, uuid.UUID): return str(value) if isinstance(value, memoryview): return bytes(value).decode("utf-8", errors="replace") if isinstance(value, (bytes, bytearray)): return value.decode("utf-8", errors="replace") if isinstance(value, (list, tuple)): return [_convert_snapshot_value(v) for v in value] return value def _normalize_snapshot_payload(record: Dict[str, Any]) -> Dict[str, Any]: """Return a JSON-safe copy of the permit risk view record.""" return {key: _convert_snapshot_value(val) for key, val in record.items()} def _fetch_permit_risk_row( conn: pg.Connection, region_id: str, permit_id: str, risk_id: str ) -> Dict[str, Any]: """Fetch a single row from the consolidation view.""" if _permit_sources_available(conn): sql = """ SELECT vw.region_id, vw.region_name, vw.permit_id, vw.permit_name, vw.risk_id, vw.risk_content, vw.legal_basis, vw.document_no, vw.summary, vw.theme_ids, vw.theme_names, vw.scope_ids, vw.scope_descriptions, vw.subitem_ids, vw.permit_status, vw.subitem_summary, vw.responsible_contact, vw.jurisdiction_scope, vw.permit_detail_updated_at, vw.permit_risk_key, ps.source_type AS permit_source_type, ps.source_name AS permit_source_name, ps.source_detail AS permit_source_detail, ps.updated_at AS permit_source_updated_at, rpd.contact_info FROM region_permit_risk_vw vw LEFT JOIN permit_sources ps ON ps.region_id = vw.region_id AND ps.permit_id = vw.permit_id LEFT JOIN region_permit_details rpd ON rpd.region_id = vw.region_id AND rpd.permit_id = vw.permit_id WHERE vw.region_id = %s AND vw.permit_id = %s AND vw.risk_id = %s LIMIT 1 """ else: sql = """ SELECT vw.region_id, vw.region_name, vw.permit_id, vw.permit_name, vw.risk_id, vw.risk_content, vw.legal_basis, vw.document_no, vw.summary, vw.theme_ids, vw.theme_names, vw.scope_ids, vw.scope_descriptions, vw.subitem_ids, vw.permit_status, vw.subitem_summary, vw.responsible_contact, vw.jurisdiction_scope, vw.permit_detail_updated_at, vw.permit_risk_key, rpd.contact_info FROM region_permit_risk_vw vw LEFT JOIN region_permit_details rpd ON rpd.region_id = vw.region_id AND rpd.permit_id = vw.permit_id WHERE vw.region_id = %s AND vw.permit_id = %s AND vw.risk_id = %s LIMIT 1 """ cur = conn.cursor() cur.execute(sql, (region_id, permit_id, risk_id)) row = cur.fetchone() if not row: raise ValueError("Permit risk combination not found in consolidation view") columns = [desc[0] for desc in cur.description] return {columns[i]: row[i] for i in range(len(columns))} def _insert_permit_risk_snapshot( conn: pg.Connection, payload: Dict[str, Any], *, edited_by: Optional[str], change_summary: Optional[str], batch_id: str, ) -> Dict[str, Any]: """Insert a snapshot row and return metadata.""" permit_risk_key = str(payload["permit_risk_key"]) cur = conn.cursor() cur.execute( """ SELECT version FROM permit_risk_snapshots WHERE permit_risk_key = %s ORDER BY version DESC LIMIT 1 FOR UPDATE """, (permit_risk_key,), ) row = cur.fetchone() next_version = (int(row[0]) + 1) if row else 1 payload_with_batch = dict(payload) payload_with_batch["snapshot_batch_id"] = batch_id payload_json = json.dumps(payload_with_batch, ensure_ascii=False) cur.execute( """ INSERT INTO permit_risk_snapshots ( region_id, permit_id, risk_id, permit_risk_key, version, payload, edited_by, change_summary ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING snapshot_id, created_at """, ( payload["region_id"], payload["permit_id"], payload["risk_id"], permit_risk_key, next_version, payload_json, edited_by, change_summary, ), ) snapshot_id, created_at = cur.fetchone() return { "snapshot_id": str(snapshot_id), "region_id": payload["region_id"], "permit_id": payload["permit_id"], "risk_id": payload["risk_id"], "permit_risk_key": permit_risk_key, "version": next_version, "created_at": _convert_snapshot_value(created_at), "edited_by": edited_by, "change_summary": change_summary or "", "snapshot_batch_id": batch_id, "payload": payload_with_batch, } def _create_snapshot_with_connection( conn: pg.Connection, region_id: str, permit_id: str, risk_id: str, *, edited_by: Optional[str], change_summary: Optional[str], batch_id: Optional[str] = None, ) -> Dict[str, Any]: """Create a snapshot using an existing DB connection (no commit).""" view_record = _fetch_permit_risk_row(conn, region_id, permit_id, risk_id) payload = _normalize_snapshot_payload(view_record) try: payload["binds_all_themes"] = _permit_binds_all_themes(conn, region_id, permit_id) except Exception: payload["binds_all_themes"] = False snapshot_batch_id = batch_id or str(uuid.uuid4()) metadata = _insert_permit_risk_snapshot( conn, payload, edited_by=edited_by, change_summary=change_summary, batch_id=snapshot_batch_id, ) theme_names = payload.get("theme_names") or [] scopes = payload.get("scope_ids") or [] subitems = payload.get("subitem_ids") or [] permit_status = payload.get("permit_status") or "" preview_text = str(payload.get("risk_content") or "").strip() preview_flat = re.sub(r"\s+", " ", preview_text) if len(preview_flat) > 120: preview_flat = f"{preview_flat[:117]}..." logger.info( "[CHECKPOINT] Snapshot created: %s version %s", metadata["permit_risk_key"], metadata["version"], ) logger.info( "[CHECKPOINT] Snapshot context: region=%s(%s) permit=%s(%s) risk=%s | themes=%s | scopes=%d | subitems=%d | status=%s", payload.get("region_id"), payload.get("region_name"), payload.get("permit_id"), payload.get("permit_name"), payload.get("risk_id"), "、".join(str(name) for name in theme_names) if theme_names else "—", len(scopes), len(subitems), permit_status or "—", ) source_name = payload.get("permit_source_name") if source_name: logger.info( "[CHECKPOINT] Snapshot permit source: %s", source_name, ) if preview_flat: logger.info( "[CHECKPOINT] Snapshot risk preview: %s", preview_flat, ) return metadata def create_permit_risk_snapshot( region_id: str, permit_id: str, risk_id: str, *, edited_by: Optional[str] = None, change_summary: Optional[str] = None, batch_id: Optional[str] = None, ) -> Dict[str, Any]: """ Capture the current state of a region/permit/risk record as a versioned snapshot. Returns metadata about the created snapshot including version number. """ with _lic_pg_conn(autocommit=False) as conn: try: snapshot_meta = _create_snapshot_with_connection( conn, region_id, permit_id, risk_id, edited_by=edited_by, change_summary=change_summary, batch_id=batch_id, ) conn.commit() return snapshot_meta except Exception: conn.rollback() raise def list_permit_risk_snapshots( region_id: Optional[str] = None, permit_id: Optional[str] = None, risk_id: Optional[str] = None, *, permit_risk_key: Optional[str] = None, limit: int = 20, offset: int = 0, ) -> List[Dict[str, Any]]: """ List snapshots for a region/permit/risk combination ordered by version descending. At least one identifier (permit_risk_key or region/permit/risk) must be provided. """ filters: List[str] = [] params: List[Any] = [] if permit_risk_key: filters.append("permit_risk_key = %s") params.append(permit_risk_key) else: if region_id: filters.append("region_id = %s") params.append(region_id) if permit_id: filters.append("permit_id = %s") params.append(permit_id) if risk_id: filters.append("risk_id = %s") params.append(risk_id) if not filters: raise ValueError("At least one identifier must be provided to list snapshots") filters_clause = " AND ".join(filters) sql = f""" SELECT snapshot_id, version, permit_risk_key, edited_by, change_summary, created_at FROM permit_risk_snapshots WHERE {filters_clause} ORDER BY version DESC LIMIT %s OFFSET %s """ params.extend([limit, offset]) with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, tuple(params)) rows = cur.fetchall() snapshots: List[Dict[str, Any]] = [] for snapshot_id, version, key, editor, summary, created_at in rows: snapshots.append( { "snapshot_id": str(snapshot_id), "permit_risk_key": key, "version": int(version), "created_at": _convert_snapshot_value(created_at), "edited_by": editor, "change_summary": summary or "", } ) return snapshots def get_permit_risk_snapshot(snapshot_id: str) -> Optional[Dict[str, Any]]: """Fetch a snapshot payload by its identifier.""" sql = """ SELECT snapshot_id, region_id, permit_id, risk_id, permit_risk_key, version, payload, edited_by, change_summary, created_at FROM permit_risk_snapshots WHERE snapshot_id = %s """ with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, (snapshot_id,)) row = cur.fetchone() if not row: return None ( snap_id, region_id, permit_id, risk_id, permit_risk_key, version, payload, edited_by, change_summary, created_at, ) = row if isinstance(payload, (bytes, bytearray, memoryview)): payload_obj = json.loads(payload) else: payload_obj = payload if isinstance(payload, dict) else json.loads(payload) return { "snapshot_id": str(snap_id), "region_id": str(region_id), "permit_id": str(permit_id), "risk_id": str(risk_id), "permit_risk_key": permit_risk_key, "version": int(version), "created_at": _convert_snapshot_value(created_at), "edited_by": edited_by, "change_summary": change_summary or "", "payload": payload_obj, } def list_permit_risk_snapshot_summaries( *, region_id: Optional[str] = None, permit_id: Optional[str] = None, edited_by: Optional[str] = None, limit: int = 20, offset: int = 0, ) -> List[Dict[str, Any]]: """ Return snapshot summaries for checkpoint history views. Groups results by snapshot_batch_id to return one entry per batch operation. """ filters: List[str] = [] params: List[Any] = [] if region_id: filters.append("region_id = %s") params.append(region_id) if permit_id: filters.append("permit_id = %s") params.append(permit_id) if edited_by: filters.append("edited_by = %s") params.append(edited_by) where_clause = f"WHERE {' AND '.join(filters)}" if filters else "" # Group by batch_id so that we show one history entry per operation (which may contain multiple risk items). # If snapshot_batch_id is missing (legacy data), fall back to snapshot_id to treat each as unique. sql = f""" SELECT MAX(snapshot_id::text) as snapshot_id, MAX(region_id::text) as region_id, MAX(permit_id::text) as permit_id, MAX(risk_id::text) as risk_id, MAX(permit_risk_key) as permit_risk_key, MAX(version) as version, MAX(edited_by) as edited_by, MAX(change_summary) as change_summary, MAX(created_at) as created_at, MAX(payload ->> 'region_name') AS region_name, MAX(payload ->> 'permit_name') AS permit_name, MAX(payload ->> 'risk_content') AS risk_content, MAX(payload ->> 'legal_basis') AS legal_basis, MAX(payload ->> 'document_no') AS document_no, MAX(payload ->> 'permit_status') AS permit_status, COALESCE(MAX(payload ->> 'snapshot_batch_id'), MAX(snapshot_id::text)) AS snapshot_batch_id, MAX(payload ->> 'permit_source_name') AS permit_source_name, MAX(payload ->> 'permit_source_type') AS permit_source_type, COUNT(*) as item_count FROM permit_risk_snapshots {where_clause} GROUP BY COALESCE(payload ->> 'snapshot_batch_id', snapshot_id::text) ORDER BY MAX(created_at) DESC LIMIT %s OFFSET %s """ params.extend([limit, offset]) with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, tuple(params)) rows = cur.fetchall() summaries: List[Dict[str, Any]] = [] for ( snapshot_id, region_uuid, permit_uuid, risk_uuid, permit_risk_key, version, editor, summary_text, created_at, region_name, permit_name, risk_content, legal_basis, document_no, permit_status, snapshot_batch_id, permit_source_name, permit_source_type, item_count, ) in rows: summaries.append( { "snapshot_id": str(snapshot_id), "region_id": str(region_uuid), "permit_id": str(permit_uuid), "risk_id": str(risk_uuid), "permit_risk_key": permit_risk_key, "version": int(version) if version else 0, "created_at": _convert_snapshot_value(created_at), "edited_by": editor, "change_summary": summary_text or "", "region_name": region_name or "", "permit_name": permit_name or "", "risk_content": risk_content or "", "legal_basis": legal_basis or "", "document_no": document_no or "", "permit_status": permit_status or "", "snapshot_batch_id": snapshot_batch_id or "", "permit_source_name": permit_source_name or "", "permit_source_type": permit_source_type or "", "item_count": int(item_count), } ) return summaries def count_permit_risk_snapshots( *, region_id: Optional[str] = None, permit_id: Optional[str] = None, edited_by: Optional[str] = None, ) -> int: """Return total snapshot batches matching the optional filters.""" filters: List[str] = [] params: List[Any] = [] if region_id: filters.append("region_id = %s") params.append(region_id) if permit_id: filters.append("permit_id = %s") params.append(permit_id) if edited_by: filters.append("edited_by = %s") params.append(edited_by) where_clause = f"WHERE {' AND '.join(filters)}" if filters else "" # Count distinct batches (using same COALESCE logic as list query) sql = f"SELECT COUNT(DISTINCT COALESCE(payload ->> 'snapshot_batch_id', snapshot_id::text)) FROM permit_risk_snapshots {where_clause}" with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, tuple(params)) row = cur.fetchone() return int(row[0]) if row else 0 def update_permit_risk_record( region_id: str, permit_id: str, risk_id: str, *, risk_content: Any = _UNSET, legal_basis: Any = _UNSET, document_no: Any = _UNSET, summary: Any = _UNSET, permit_status: Any = _UNSET, subitem_summary: Any = _UNSET, responsible_contact: Any = _UNSET, jurisdiction_scope: Any = _UNSET, contact_info: Any = _UNSET, edited_by: Optional[str] = None, change_summary: Optional[str] = None, ) -> Dict[str, Any]: """ Update the permit risk record while capturing a checkpoint snapshot beforehand. Returns the snapshot metadata (pre-change) and the refreshed view row (post-change). """ update_flags = [ risk_content, legal_basis, document_no, summary, permit_status, subitem_summary, responsible_contact, jurisdiction_scope, contact_info, ] if all(flag is _UNSET for flag in update_flags): raise ValueError("No fields provided to update.") with _lic_pg_conn(autocommit=False) as conn: try: cur = conn.cursor() cur.execute( """ SELECT 1 FROM region_permit_risks WHERE region_id = %s AND permit_id = %s AND risk_id = %s FOR UPDATE """, (region_id, permit_id, risk_id), ) if cur.fetchone() is None: raise ValueError("Permit risk combination not found.") snapshot_meta = _create_snapshot_with_connection( conn, region_id, permit_id, risk_id, edited_by=edited_by, change_summary=change_summary, ) risk_updates: List[str] = [] risk_params: List[Any] = [] risk_fields = ( ("risk_content", risk_content), ("legal_basis", legal_basis), ("document_no", document_no), ("summary", summary), ) for column, value in risk_fields: if value is not _UNSET: risk_updates.append(f"{column} = %s") risk_params.append(value) if risk_updates: risk_params.append(risk_id) cur.execute( f"UPDATE risks SET {', '.join(risk_updates)} WHERE id = %s", tuple(risk_params), ) detail_columns: List[str] = [] detail_values: List[Any] = [] detail_fields = ( ("permit_status", permit_status), ("subitem_summary", subitem_summary), ("responsible_contact", responsible_contact), ("jurisdiction_scope", jurisdiction_scope), ("contact_info", contact_info), ) for column, value in detail_fields: if value is not _UNSET: detail_columns.append(column) detail_values.append(value) details_updated = False if detail_columns: insert_cols = ", ".join(["region_id", "permit_id"] + detail_columns) insert_placeholders = ", ".join(["%s"] * (2 + len(detail_values))) update_assignments = ", ".join( [f"{col} = EXCLUDED.{col}" for col in detail_columns] ) _ensure_contact_info_column(conn) sql = f""" INSERT INTO region_permit_details ({insert_cols}) VALUES ({insert_placeholders}) ON CONFLICT (region_id, permit_id) DO UPDATE SET {update_assignments}, updated_at = now() """ cur.execute( sql, (region_id, permit_id, *detail_values), ) details_updated = True updated_record = _normalize_snapshot_payload( _fetch_permit_risk_row(conn, region_id, permit_id, risk_id) ) conn.commit() logger.info( "[CHECKPOINT] Permit risk updated: %s version %s -> new snapshot ready", snapshot_meta["permit_risk_key"], snapshot_meta["version"], ) return { "snapshot": snapshot_meta, "current": updated_record, "risk_updated": bool(risk_updates), "details_updated": details_updated, } except Exception: conn.rollback() raise def delete_region_permit( region_id: str, theme_id: str, permit_id: str, *, edited_by: Optional[str] = None, change_summary: Optional[str] = None, ) -> Dict[str, Any]: """ 删除指定区划下的许可，同时为所有关联风险生成快照，并清理依赖关系。 返回删除摘要、快照列表以及主题剩余许可数量。 """ with _lic_pg_conn(autocommit=False) as conn: cur = conn.cursor() try: cur.execute( """ SELECT r.name, t.name, p.name FROM region_theme_permits rtp JOIN regions r ON r.id = rtp.region_id JOIN themes t ON t.id = rtp.theme_id JOIN permits p ON p.id = rtp.permit_id WHERE rtp.region_id = %s AND rtp.theme_id = %s AND rtp.permit_id = %s FOR UPDATE """, (region_id, theme_id, permit_id), ) row = cur.fetchone() if not row: raise ValueError("地区-主题-许可组合不存在，无法删除") region_name, theme_name, permit_name = (str(val) for val in row) cur.execute( """ SELECT risk_id FROM region_permit_risks WHERE region_id = %s AND permit_id = %s ORDER BY risk_id FOR UPDATE """, (region_id, permit_id), ) risk_ids = [str(risk_id) for (risk_id,) in cur.fetchall()] snapshots: List[Dict[str, Any]] = [] total_snapshots = 0 summary_base = (change_summary or "").strip() if not summary_base: summary_base = f"删除许可 {permit_name}（地区：{region_name}）" snapshot_batch_id = str(uuid.uuid4()) for idx, risk_id in enumerate(risk_ids, start=1): detail_summary = summary_base if len(risk_ids) > 1: detail_summary = f"{summary_base} - 风险 {idx}/{len(risk_ids)}（ID：{risk_id}）" snapshot_meta = _create_snapshot_with_connection( conn, region_id, permit_id, risk_id, edited_by=edited_by, change_summary=detail_summary, batch_id=snapshot_batch_id, ) snapshots.append( { "snapshot_id": snapshot_meta["snapshot_id"], "permit_risk_key": snapshot_meta["permit_risk_key"], "version": snapshot_meta["version"], "risk_id": snapshot_meta["risk_id"], "created_at": snapshot_meta["created_at"], "snapshot_batch_id": snapshot_meta.get("snapshot_batch_id"), "change_summary": snapshot_meta.get("change_summary", ""), } ) total_snapshots += 1 if total_snapshots: logger.info( "[PERMIT-DELETE] Captured %d snapshots before deleting permit %s (%s) in region %s (%s)", total_snapshots, permit_id, permit_name, region_id, region_name, ) else: logger.info( "[PERMIT-DELETE] No risk snapshots required for permit %s (%s) in region %s (%s)", permit_id, permit_name, region_id, region_name, ) delete_counts: Dict[str, int] = {} cur.execute( """ DELETE FROM region_permit_risks WHERE region_id = %s AND permit_id = %s """, (region_id, permit_id), ) delete_counts["region_permit_risks"] = int(cur.rowcount or 0) cur.execute( """ DELETE FROM region_permit_subitems WHERE region_id = %s AND permit_id = %s """, (region_id, permit_id), ) delete_counts["region_permit_subitems"] = int(cur.rowcount or 0) cur.execute( """ DELETE FROM region_permit_scopes WHERE region_id = %s AND permit_id = %s """, (region_id, permit_id), ) delete_counts["region_permit_scopes"] = int(cur.rowcount or 0) cur.execute( """ DELETE FROM region_permit_details WHERE region_id = %s AND permit_id = %s """, (region_id, permit_id), ) delete_counts["region_permit_details"] = int(cur.rowcount or 0) if _permit_sources_available(conn): cur.execute( """ DELETE FROM permit_sources WHERE region_id = %s AND permit_id = %s """, (region_id, permit_id), ) delete_counts["permit_sources"] = int(cur.rowcount or 0) cur.execute( """ DELETE FROM region_theme_permits WHERE region_id = %s AND theme_id = %s AND permit_id = %s """, (region_id, theme_id, permit_id), ) delete_counts["region_theme_permits"] = int(cur.rowcount or 0) _clear_permit_bind_all_themes(conn, region_id, permit_id) cur.execute( """ SELECT COUNT(*) FROM region_theme_permits WHERE region_id = %s AND theme_id = %s """, (region_id, theme_id), ) remaining_theme_permits = int(cur.fetchone()[0] or 0) theme_detached = False if remaining_theme_permits == 0: cur.execute( """ DELETE FROM region_themes WHERE region_id = %s AND theme_id = %s """, (region_id, theme_id), ) theme_detached = cur.rowcount > 0 if theme_detached: logger.info( "[PERMIT-DELETE] Detached theme %s (%s) from region %s (%s) because no permits remain", theme_id, theme_name, region_id, region_name, ) else: logger.info( "[PERMIT-DELETE] Theme %s (%s) retains %d permit(s) in region %s (%s); theme linkage preserved", theme_id, theme_name, remaining_theme_permits, region_id, region_name, ) conn.commit() logger.info( "[PERMIT-DELETE] Completed deletion for permit %s (%s) in region %s (%s): snapshots=%d, deleted_rows=%s", permit_id, permit_name, region_id, region_name, total_snapshots, delete_counts, ) # Log the operation log_operation( operator=edited_by or "admin", operation_type="DELETE", target_type="PERMIT", target_id=permit_id, target_name=permit_name, change_summary=f"Deleted permit {permit_name} in region {region_name}", details={ "region_id": region_id, "region_name": region_name, "deleted_risk_count": delete_counts.get("region_permit_risks", 0), "snapshot_batch_id": snapshot_batch_id } ) return { "region_id": str(region_id), "region_name": region_name, "theme_id": str(theme_id), "theme_name": theme_name, "permit_id": str(permit_id), "permit_name": permit_name, "risk_ids": risk_ids, "snapshot_count": total_snapshots, "snapshots": snapshots, "snapshot_batch_id": snapshot_batch_id if total_snapshots else "", "deleted_rows": delete_counts, "theme_detached": theme_detached, "remaining_theme_permits": remaining_theme_permits, } except Exception: conn.rollback() raise def restore_permit_risk_snapshot_batch( snapshot_batch_id: str, *, edited_by: Optional[str] = None, change_summary: Optional[str] = None, ) -> Dict[str, Any]: """ Restore region/permit/risk relations based on a snapshot batch (or single snapshot). """ if not snapshot_batch_id: raise ValueError("快照批次 ID 不能为空") with _lic_pg_conn(autocommit=False) as conn: cur = conn.cursor() cur.execute( """ SELECT snapshot_id, payload, created_at, edited_by, change_summary FROM permit_risk_snapshots WHERE payload ->> 'snapshot_batch_id' = %s ORDER BY created_at ASC, snapshot_id ASC """, (snapshot_batch_id,), ) rows = cur.fetchall() resolved_batch_id = snapshot_batch_id if not rows: cur.execute( """ SELECT snapshot_id, payload, created_at, edited_by, change_summary FROM permit_risk_snapshots WHERE snapshot_id::text = %s """, (snapshot_batch_id,), ) rows = cur.fetchall() if not rows: raise ValueError("未找到对应的快照记录") snapshots: List[Dict[str, Any]] = [] for snap_id, payload_raw, created_at, snap_editor, snap_summary in rows: if isinstance(payload_raw, dict): payload_obj = payload_raw elif isinstance(payload_raw, (bytes, bytearray, memoryview)): payload_obj = json.loads(bytes(payload_raw).decode("utf-8")) elif isinstance(payload_raw, str): payload_obj = json.loads(payload_raw) else: payload_obj = json.loads(payload_raw) batch_token = payload_obj.get("snapshot_batch_id") if batch_token: resolved_batch_id = batch_token snapshots.append( { "snapshot_id": str(snap_id), "payload": payload_obj, "created_at": _convert_snapshot_value(created_at), "edited_by": snap_editor, "change_summary": snap_summary or "", } ) payload0 = snapshots[0]["payload"] region_id = str(payload0["region_id"]) permit_id = str(payload0["permit_id"]) region_name = str(payload0.get("region_name") or "") permit_name = str(payload0.get("permit_name") or "") theme_ids: Set[str] = set(str(t) for t in (payload0.get("theme_ids") or []) if t) scope_ids: Set[str] = set() subitem_ids: Set[str] = set() binds_all_override = bool(payload0.get("binds_all_themes")) for snap in snapshots: payload = snap["payload"] for tid in payload.get("theme_ids") or []: if tid: theme_ids.add(str(tid)) for scope in payload.get("scope_ids") or []: if scope: scope_ids.add(str(scope)) for subitem in payload.get("subitem_ids") or []: if subitem: subitem_ids.add(str(subitem)) detail_fields = { "permit_status": payload0.get("permit_status"), "subitem_summary": payload0.get("subitem_summary"), "responsible_contact": payload0.get("responsible_contact"), "responsible_contact": payload0.get("responsible_contact"), "jurisdiction_scope": payload0.get("jurisdiction_scope"), "contact_info": payload0.get("contact_info"), } source_name = _clean_text(payload0.get("permit_source_name")) source_type = _clean_text(payload0.get("permit_source_type")) or "snapshot" source_detail = payload0.get("permit_source_detail") insert_counts = { "region_themes": 0, "region_theme_permits": 0, "region_permit_details": 0, "region_permit_scopes": 0, "region_permit_subitems": 0, "region_permit_risks": 0, "risks_upserted": 0, "permit_sources_synced": 0, } restored_risk_ids: Set[str] = set() try: for theme_id in sorted(theme_ids): cur.execute( """ INSERT INTO region_themes (region_id, theme_id) VALUES (%s, %s) ON CONFLICT DO NOTHING """, (region_id, theme_id), ) insert_counts["region_themes"] += cur.rowcount or 0 cur.execute( """ INSERT INTO region_theme_permits (region_id, theme_id, permit_id) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING """, (region_id, theme_id, permit_id), ) insert_counts["region_theme_permits"] += cur.rowcount or 0 if binds_all_override: _set_permit_bind_all_themes(conn, region_id, permit_id, created_by=snapshots[0].get("edited_by")) else: _clear_permit_bind_all_themes(conn, region_id, permit_id) _ensure_contact_info_column(conn) cur = conn.cursor() # Re-acquire cursor if needed or just use existing cur.execute( """ INSERT INTO region_permit_details ( region_id, permit_id, permit_status, subitem_summary, responsible_contact, jurisdiction_scope, contact_info ) VALUES (%s, %s, %s, %s, %s, %s, %s) ON CONFLICT (region_id, permit_id) DO UPDATE SET permit_status = EXCLUDED.permit_status, subitem_summary = EXCLUDED.subitem_summary, responsible_contact = EXCLUDED.responsible_contact, jurisdiction_scope = EXCLUDED.jurisdiction_scope, contact_info = EXCLUDED.contact_info, updated_at = now() """, ( region_id, permit_id, detail_fields["permit_status"], detail_fields["subitem_summary"], detail_fields["responsible_contact"], detail_fields["jurisdiction_scope"], detail_fields["contact_info"], ), ) insert_counts["region_permit_details"] += cur.rowcount or 0 for scope_id in sorted(scope_ids): cur.execute( """ INSERT INTO region_permit_scopes (region_id, permit_id, scope_id) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING """, (region_id, permit_id, scope_id), ) insert_counts["region_permit_scopes"] += cur.rowcount or 0 for subitem_id in sorted(subitem_ids): cur.execute( """ INSERT INTO region_permit_subitems (region_id, permit_id, subitem_id) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING """, (region_id, permit_id, subitem_id), ) insert_counts["region_permit_subitems"] += cur.rowcount or 0 for snap in snapshots: payload = snap["payload"] risk_id = str(payload["risk_id"]) cur.execute( """ INSERT INTO risks (id, risk_content, legal_basis, document_no, summary) VALUES (%s, %s, %s, %s, %s) ON CONFLICT (id) DO UPDATE SET risk_content = EXCLUDED.risk_content, legal_basis = EXCLUDED.legal_basis, document_no = EXCLUDED.document_no, summary = EXCLUDED.summary """, ( risk_id, payload.get("risk_content"), payload.get("legal_basis"), payload.get("document_no"), payload.get("summary"), ), ) insert_counts["risks_upserted"] += cur.rowcount or 0 cur.execute( """ INSERT INTO region_permit_risks (region_id, permit_id, risk_id) VALUES (%s, %s, %s) ON CONFLICT DO NOTHING """, (region_id, permit_id, risk_id), ) if cur.rowcount: insert_counts["region_permit_risks"] += cur.rowcount restored_risk_ids.add(risk_id) if source_name: _ensure_permit_sources_table(conn) if isinstance(source_detail, (dict, list)): source_detail_text = json.dumps(source_detail, ensure_ascii=False) else: source_detail_text = _clean_text(source_detail) if source_detail is not None else None cur.execute( """ INSERT INTO permit_sources ( region_id, permit_id, source_type, source_name, source_detail, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s, now(), now()) ON CONFLICT (region_id, permit_id) DO UPDATE SET source_type = EXCLUDED.source_type, source_name = EXCLUDED.source_name, source_detail = EXCLUDED.source_detail, updated_at = now() """, ( region_id, permit_id, source_type or "snapshot", source_name, source_detail_text, ), ) insert_counts["permit_sources_synced"] += 1 elif _permit_sources_available(conn): cur.execute( """ DELETE FROM permit_sources WHERE region_id = %s AND permit_id = %s """, (region_id, permit_id), ) if cur.rowcount: insert_counts["permit_sources_synced"] += 1 conn.commit() except Exception: conn.rollback() raise logger.info( "[PERMIT-RESTORE] Restored permit %s (%s) in region %s (%s) from snapshot batch %s: %d risk mappings", permit_id, permit_name, region_id, region_name, resolved_batch_id, len(restored_risk_ids), ) return { "snapshot_batch_id": resolved_batch_id, "snapshot_ids": [snap["snapshot_id"] for snap in snapshots], "restored_risk_count": len(restored_risk_ids), "restored_risks": sorted(restored_risk_ids), "region_id": region_id, "region_name": region_name, "permit_id": permit_id, "permit_name": permit_name, "applied_theme_ids": sorted(theme_ids), "applied_scope_ids": sorted(scope_ids), "applied_subitem_ids": sorted(subitem_ids), "detail_fields": detail_fields, "insert_counts": insert_counts, "edited_by": edited_by, "change_summary": change_summary or "", } def _expand_department_family(department_ids: List[str]) -> List[str]: """ Expand a list of department IDs to include their entire family (parents and children). This enables 'same department' visibility across city (parent) and district (child) levels. """ if not department_ids: return [] # Use a set to avoid duplicates expanded_ids = set() roots = set() with _lic_pg_conn() as conn: cur = conn.cursor() # 1. Find roots for the input departments # We assume a 2-level hierarchy for now (City -> District) based on current seeds. # If deeply nested, Recursive CTE would be better, but this suffices for current requirement. placeholders = ','.join(['%s'] * len(department_ids)) sql_find_roots = f"SELECT id, parent_id FROM service_departments WHERE id IN ({placeholders})" cur.execute(sql_find_roots, department_ids) for dept_id, parent_id in cur.fetchall(): # If it has a parent, the parent is the root (or closer to it) if parent_id: roots.add(str(parent_id)) else: # If no parent, it IS the root roots.add(str(dept_id)) if not roots: return department_ids # 2. Find all departments that share these roots (the roots themselves and their children) root_list = list(roots) root_placeholders = ','.join(['%s'] * len(root_list)) # Select where ID is a root OR Parent ID is a root sql_expand = f""" SELECT id FROM service_departments WHERE id IN ({root_placeholders}) OR parent_id IN ({root_placeholders}) """ # We pass root_list twice because we use the placeholders twice cur.execute(sql_expand, root_list + root_list) for row in cur.fetchall(): expanded_ids.add(str(row[0])) return list(expanded_ids) def filter_permits_advanced( regions: Optional[List[str]] = None, themes: Optional[List[str]] = None, departments: Optional[List[str]] = None, search_text: Optional[str] = None, visibility: Optional[str] = None, # 'visible', 'hidden' or None/all limit: int = 100, offset: int = 0, ) -> Dict[str, Any]: """Filter permits using multiple dimensions (region, theme, department, search text, visibility). Args: regions: List of region IDs to filter by (supports multi-select) themes: List of theme IDs to filter by (supports multi-select) departments: List of department IDs to filter by (supports multi-select) search_text: Search in permit name visibility: Filter by v2 visibility ('visible', 'hidden') limit: Maximum number of results to return offset: Offset for pagination Returns: Dictionary containing filtered permits and metadata """ print(f"[DEBUG] filter_permits_advanced called with limit={limit}, offset={offset}, visibility={visibility}") # Use subquery to avoid DISTINCT with window functions issue # Subquery to get unique permits matching filters with pagination # We use a CTE to ensure limit/offset apply to unique permits, not to rows (which can duplicate per theme) base_where = " WHERE 1=1 " base_params = [] if regions: placeholders = ', '.join(['%s'] * len(regions)) base_where += f" AND rpd.region_id IN ({placeholders})" base_params.extend(regions) if themes: placeholders = ', '.join(['%s'] * len(themes)) base_where += f" AND rtp.theme_id IN ({placeholders})" base_params.extend(themes) if departments: # Expand departments to include family (parent + children) # This allows cross-level visibility for the "same" department expanded_departments = _expand_department_family(departments) placeholders = ', '.join(['%s'] * len(expanded_departments)) base_where += f" AND (ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))" base_params.extend(expanded_departments * 2) if search_text: base_where += f" AND LOWER(p.name) LIKE LOWER(%s)" base_params.append(f"%{search_text}%") if visibility == 'visible': base_where += " AND (rpd.is_v2_visible IS TRUE OR rpd.is_v2_visible IS NULL)" elif visibility == 'hidden': base_where += " AND rpd.is_v2_visible IS FALSE" sql = f""" WITH filtered_p AS ( SELECT rpd.permit_id, rpd.region_id FROM region_permit_details rpd JOIN permits p ON p.id = rpd.permit_id LEFT JOIN region_theme_permits rtp ON rtp.permit_id = rpd.permit_id AND rtp.region_id = rpd.region_id LEFT JOIN permit_sources ps ON ps.permit_id = rpd.permit_id AND ps.region_id = rpd.region_id {base_where} GROUP BY rpd.permit_id, rpd.region_id, p.name ORDER BY LOWER(p.name) LIMIT %s OFFSET %s ) SELECT p.id AS permit_id, p.name AS permit_name, rpd.region_id, r.name AS region_name, rtp.theme_id, t.name AS theme_name, COALESCE(risk_counts.risk_count, 0) AS risk_count, COALESCE(theme_counts.theme_count, 0) AS theme_count, COALESCE(rpd.is_v2_visible, true) AS is_v2_visible FROM filtered_p fp JOIN region_permit_details rpd ON rpd.permit_id = fp.permit_id AND rpd.region_id = fp.region_id JOIN permits p ON p.id = rpd.permit_id JOIN regions r ON r.id = rpd.region_id LEFT JOIN region_theme_permits rtp ON rtp.permit_id = rpd.permit_id AND rtp.region_id = rpd.region_id LEFT JOIN themes t ON t.id = rtp.theme_id LEFT JOIN ( SELECT permit_id, region_id, COUNT(risk_id) AS risk_count FROM region_permit_risks GROUP BY permit_id, region_id ) risk_counts ON risk_counts.permit_id = rpd.permit_id AND risk_counts.region_id = rpd.region_id LEFT JOIN ( SELECT permit_id, region_id, COUNT(DISTINCT theme_id) AS theme_count FROM region_theme_permits GROUP BY permit_id, region_id ) theme_counts ON theme_counts.permit_id = rpd.permit_id AND theme_counts.region_id = rpd.region_id ORDER BY LOWER(p.name), LOWER(r.name), LOWER(COALESCE(t.name, '')) """ params = base_params + [limit, offset] permits_map = {} with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, params) for ( permit_id, permit_name, rid, region_name, tid, theme_name, risk_count, theme_count, v2_visible, ) in cur.fetchall(): pid = str(permit_id) key = f"{pid}_{rid}" if key not in permits_map: permits_map[key] = { "id": pid, "name": str(permit_name), "region": { "id": str(rid), "name": str(region_name), }, "themes": [], "risk_count": int(risk_count or 0), "theme_count": int(theme_count or 0), "is_v2_visible": bool(v2_visible), } if tid or theme_name: theme_payload = { "id": str(tid) if tid else "", "name": str(theme_name) if theme_name else "", } existing_themes = permits_map[key]["themes"] if not any( candidate.get("id") == theme_payload["id"] and candidate.get("name") == theme_payload["name"] for candidate in existing_themes ): existing_themes.append(theme_payload) # Use OrderedDict or sorted permits_list to maintain name order after dict values collection permits_list = list(permits_map.values()) # Sort again by name to ensure order because dict.values() might not be stable depending on Python version/access permits_list.sort(key=lambda x: x["name"].lower()) # Get total count for pagination count_sql = f""" SELECT COUNT(DISTINCT rpd.permit_id || '_' || rpd.region_id) FROM region_permit_details rpd JOIN permits p ON p.id = rpd.permit_id LEFT JOIN region_theme_permits rtp ON rtp.permit_id = rpd.permit_id AND rtp.region_id = rpd.region_id LEFT JOIN permit_sources ps ON ps.permit_id = rpd.permit_id AND ps.region_id = rpd.region_id {base_where} """ with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(count_sql, base_params) total = cur.fetchone()[0] return { "permits": permits_list, "pagination": { "total": total, "limit": limit, "offset": offset, "count": len(permits_list), }, } def _ensure_v2_visibility_column(conn: Optional[pg.Connection] = None) -> None: """Ensure that the is_v2_visible column exists in region_permit_details.""" global _V2_VISIBILITY_SCHEMA_READY if _V2_VISIBILITY_SCHEMA_READY: return with _V2_VISIBILITY_SCHEMA_LOCK: if _V2_VISIBILITY_SCHEMA_READY: return sql = "ALTER TABLE region_permit_details ADD COLUMN IF NOT EXISTS is_v2_visible BOOLEAN DEFAULT TRUE" if conn is not None: original_autocommit = conn.autocommit try: conn.autocommit = True cur = conn.cursor() cur.execute(sql) finally: conn.autocommit = original_autocommit else: with _lic_pg_conn(autocommit=True) as ensure_conn: cur = ensure_conn.cursor() cur.execute(sql) _V2_VISIBILITY_SCHEMA_READY = True def _ensure_contact_info_column(conn: pg.Connection) -> None: "Ensure that the contact_info column exists in region_permit_details." # This check is now redundant since schema fix script was run, but kept for safety pass def update_permit_v2_visibility( region_id: str, permit_id: str, is_visible: bool, operator: str = "admin" ) -> bool: """Update the is_v2_visible flag for a specific region-permit pair.""" _ensure_v2_visibility_column() sql = """ UPDATE region_permit_details SET is_v2_visible = %s, updated_at = now() WHERE region_id = %s AND permit_id = %s """ with _lic_pg_conn() as conn: cur = conn.cursor() cur.execute(sql, (is_visible, region_id, permit_id)) count = cur.rowcount conn.commit() if count > 0: log_operation( operator=operator, operation_type="UPDATE", target_type="PERMIT_VISIBILITY", target_id=f"{region_id}:{permit_id}", target_name=f"Permit {permit_id} in region {region_id}", change_summary=f"Set is_v2_visible to {is_visible}", details={"is_visible": is_visible, "region_id": region_id, "permit_id": permit_id} ) return True return False