6644 lines
239 KiB
Python
6644 lines
239 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
from collections import OrderedDict, defaultdict
|
||
import hashlib
|
||
from datetime import datetime, date
|
||
from decimal import Decimal
|
||
from io import BytesIO
|
||
import threading
|
||
import time
|
||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
||
import uuid
|
||
|
||
import pg8000.dbapi as pg
|
||
from openpyxl import load_workbook
|
||
from openpyxl.utils.exceptions import InvalidFileException
|
||
|
||
# Configure logger
|
||
logger = logging.getLogger(__name__)
|
||
if not logger.handlers:
|
||
handler = logging.StreamHandler()
|
||
handler.setFormatter(logging.Formatter("[%(levelname)s] %(name)s: %(message)s"))
|
||
logger.addHandler(handler)
|
||
logger.setLevel(logging.INFO)
|
||
logger.propagate = False
|
||
|
||
# Separate configuration so legacy fs_law_risk integration keeps using PG_*
|
||
LIC_DEFAULT_DB = "licensing_risks"
|
||
|
||
_UNSET = object()
|
||
|
||
ARTICLE_HEADING_RE = re.compile(r"(?m)^(第[一二三四五六七八九十百零0-9]+条)")
|
||
ARTICLE_TOKEN_RE = re.compile(r"(?<!\*)(第[一二三四五六七八九十百零0-9]+条)(?!\*)")
|
||
ARTICLE_NEWLINE_RE = re.compile(r"(?<!^)(?<!\n)(\*\*第[一二三四五六七八九十百零0-9]+条\*\*)")
|
||
CN_ENUM_INLINE_RE = re.compile(r"([；;:：。．])[ \t]*（([一二三四五六七八九十百零]+)）")
|
||
CN_ENUM_LINE_RE = re.compile(r"(?m)^\s*（([一二三四五六七八九十百零]+)）")
|
||
ARABIC_ENUM_INLINE_RE = re.compile(r"([；;:：。．，,])[ \t]*(\d+\.)")
|
||
ARABIC_ENUM_LINE_RE = re.compile(r"(?m)^\s*(\d+)\.")
|
||
NESTED_ENUM_INLINE_RE = re.compile(r"([；;:：。．])[ \t]*（(\d+)）")
|
||
NESTED_ENUM_LINE_RE = re.compile(r"(?m)^\s*（(\d+)）")
|
||
COLON_NEWLINE_RE = re.compile(r"：\s*\n")
|
||
TRAILING_SPACE_RE = re.compile(r"[ \t]+\n")
|
||
EXTRA_NEWLINES_RE = re.compile(r"\n{3,}")
|
||
|
||
TEXT_SPLIT_PATTERN = re.compile(r"[,\uff0c;\uff1b\n\r]+")
|
||
TEXT_SPLIT_PATTERN_WITH_DUNHAO = re.compile(r"[,\uff0c;\uff1b、\n\r]+")
|
||
|
||
PERMIT_IMPORT_TTL_SECONDS = 1800
|
||
MAX_PERMIT_FILE_SIZE_BYTES = 500 * 1024 # 500 KB limit for uploaded Excel files
|
||
_PERMIT_IMPORT_SESSIONS: Dict[str, Dict[str, Any]] = {}
|
||
_PERMIT_IMPORT_LOCK = threading.Lock()
|
||
|
||
ALL_THEMES_SENTINEL = "__ALL_THEMES__"
|
||
ALL_THEMES_DISPLAY_NAME = "所有主题"
|
||
|
||
_PERMIT_FILE_SCHEMA_READY: Optional[bool] = None
|
||
_PERMIT_FILE_SCHEMA_LOCK = threading.Lock()
|
||
|
||
_OPERATION_LOG_SCHEMA_READY: Optional[bool] = None
|
||
_OPERATION_LOG_SCHEMA_LOCK = threading.Lock()
|
||
|
||
_PERMIT_THEME_OVERRIDE_SCHEMA_READY: Optional[bool] = None
|
||
_PERMIT_THEME_OVERRIDE_SCHEMA_LOCK = threading.Lock()
|
||
|
||
_PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY: Optional[bool] = None
|
||
_PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_LOCK = threading.Lock()
|
||
|
||
_V2_VISIBILITY_SCHEMA_READY: Optional[bool] = None
|
||
_V2_VISIBILITY_SCHEMA_LOCK = threading.Lock()
|
||
|
||
_IMPORT_HEADER_ALIASES: Dict[str, Set[str]] = {
|
||
"permit_name": {
|
||
"许可事项",
|
||
"许可名称",
|
||
"事项名称",
|
||
"事项",
|
||
"事项全称",
|
||
"事项标题",
|
||
"许可（备案）事项名称",
|
||
},
|
||
"risk_content": {
|
||
"风险提示",
|
||
"风险内容",
|
||
"风险点",
|
||
"风险描述",
|
||
"风险信息",
|
||
"风险要点",
|
||
"风险提示内容",
|
||
},
|
||
"legal_basis": {
|
||
"法律依据",
|
||
"主要法律依据",
|
||
"依据内容",
|
||
"法规依据",
|
||
},
|
||
"document_no": {
|
||
"依据文号",
|
||
"文号",
|
||
"法规文号",
|
||
"编号",
|
||
},
|
||
"summary": {
|
||
"风险说明",
|
||
"摘要",
|
||
"风险摘要",
|
||
"补充说明",
|
||
},
|
||
"remark": {
|
||
"备注",
|
||
"其他",
|
||
"备注项",
|
||
},
|
||
"permit_status": {
|
||
"许可状态",
|
||
"事项状态",
|
||
"审批状态",
|
||
"状态",
|
||
"许可情况",
|
||
"事项情况",
|
||
"许可情况（前置事项/后置事项/备案事项）",
|
||
},
|
||
"responsible_contact": {
|
||
"责任部门",
|
||
"责任单位",
|
||
"责任主体",
|
||
"主管部门",
|
||
"负责部门",
|
||
"负责部门及联系方式",
|
||
"联系方式（内部使用）",
|
||
"联系方式",
|
||
},
|
||
"jurisdiction_scope": {
|
||
"适用范围",
|
||
"管辖范围",
|
||
"权限划分",
|
||
"市区权限划分",
|
||
"适用区域",
|
||
"适用地区",
|
||
"事项实施层级",
|
||
"实施层级",
|
||
},
|
||
"subitem_text": {
|
||
"子项",
|
||
"办理子项",
|
||
"事项子项",
|
||
"细化子项",
|
||
},
|
||
"filler_name": {
|
||
"填表人",
|
||
"填报人",
|
||
},
|
||
"unit_name": {
|
||
"单位名称",
|
||
"填报单位",
|
||
"所属单位",
|
||
},
|
||
"source_update_date": {
|
||
"表格更新日期",
|
||
"更新日期",
|
||
"填表日期",
|
||
},
|
||
"serial_number": {
|
||
"序号",
|
||
"编号",
|
||
"排序",
|
||
"风险提示",
|
||
},
|
||
}
|
||
|
||
_IMPORT_HEADER_KEYWORDS: List[Tuple[str, Tuple[str, ...]]] = [
|
||
("permit_status", ("情况", "状态")),
|
||
("permit_name", ("许可", "事项")),
|
||
("risk_content", ("风险",)),
|
||
("legal_basis", ("依据",)),
|
||
("document_no", ("文号", "编号")),
|
||
("summary", ("摘要", "说明")),
|
||
("remark", ("备注",)),
|
||
("responsible_contact", ("责任", "主管")),
|
||
("jurisdiction_scope", ("权限划分", "区域", "层级")),
|
||
]
|
||
|
||
_PERMIT_SOURCES_TABLE_PRESENT: Optional[bool] = None
|
||
_PERMIT_SOURCES_TABLE_LOCK = threading.Lock()
|
||
|
||
_PERMIT_FILE_SCHEMA_READY: Optional[bool] = None
|
||
_PERMIT_FILE_SCHEMA_LOCK = threading.Lock()
|
||
|
||
_SERVICE_DEPARTMENT_SCHEMA_READY: Optional[bool] = None
|
||
_SERVICE_DEPARTMENT_SCHEMA_LOCK = threading.Lock()
|
||
|
||
_CANONICAL_REGION_KEYWORDS: Dict[str, Tuple[str, ...]] = {
|
||
"市级": ("市级", "全市", "佛山市本级", "佛山市市级"),
|
||
"禅城区": ("禅城区", "禅城"),
|
||
"南海区": ("南海区", "南海"),
|
||
"顺德区": ("顺德区", "顺德"),
|
||
"三水区": ("三水区", "三水"),
|
||
"高明区": ("高明区", "高明"),
|
||
}
|
||
|
||
|
||
def _format_summary_markdown(summary: str) -> str:
|
||
"""Render Chinese legal excerpts as Markdown-friendly text."""
|
||
if not summary:
|
||
return ""
|
||
|
||
text = summary.replace("\r\n", "\n").strip()
|
||
if not text:
|
||
return ""
|
||
|
||
text = ARTICLE_HEADING_RE.sub(lambda m: f"**{m.group(1)}**", text)
|
||
text = CN_ENUM_INLINE_RE.sub(lambda m: f"{m.group(1)}\n- （{m.group(2)}） ", text)
|
||
text = CN_ENUM_LINE_RE.sub(lambda m: f"- （{m.group(1)}） ", text)
|
||
text = ARABIC_ENUM_INLINE_RE.sub(lambda m: f"{m.group(1)}\n {m.group(2)}", text)
|
||
text = ARABIC_ENUM_LINE_RE.sub(lambda m: f" {m.group(1)}.", text)
|
||
text = NESTED_ENUM_INLINE_RE.sub(lambda m: f"{m.group(1)}\n - （{m.group(2)}）", text)
|
||
text = NESTED_ENUM_LINE_RE.sub(lambda m: f" - （{m.group(1)}）", text)
|
||
text = ARTICLE_TOKEN_RE.sub(lambda m: f"**{m.group(1)}**", text)
|
||
text = ARTICLE_NEWLINE_RE.sub(lambda m: f"\n{m.group(1)}", text)
|
||
text = COLON_NEWLINE_RE.sub("：\n", text)
|
||
text = EXTRA_NEWLINES_RE.sub("\n\n", text)
|
||
text = TRAILING_SPACE_RE.sub("\n", text)
|
||
text = re.sub(r"\n\s+\n", "\n\n", text)
|
||
return text.strip()
|
||
|
||
|
||
def _clean_text(value: Any) -> str:
|
||
"""Return a stripped string representation for Excel parsing."""
|
||
if value is None:
|
||
return ""
|
||
if isinstance(value, str):
|
||
return value.strip()
|
||
return str(value).strip()
|
||
|
||
|
||
def _to_isoformat(value: Any) -> Optional[str]:
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, datetime):
|
||
return value.isoformat()
|
||
if isinstance(value, date):
|
||
return datetime.combine(value, datetime.min.time()).isoformat()
|
||
return str(value)
|
||
|
||
|
||
def _to_optional_str(value: Any) -> Optional[str]:
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, (uuid.UUID,)):
|
||
return str(value)
|
||
return str(value)
|
||
|
||
|
||
def _normalize_permit_token(value: Any) -> str:
|
||
"""Normalize permit names for dictionary lookups (case/whitespace insensitive)."""
|
||
text = _clean_text(value)
|
||
if not text:
|
||
return ""
|
||
return text.lower()
|
||
|
||
|
||
def _permit_name_aliases(value: Any) -> Set[str]:
|
||
"""Return canonical + token aliases for a permit name."""
|
||
aliases: Set[str] = set()
|
||
canonical = _clean_text(value)
|
||
token = _normalize_permit_token(value)
|
||
if canonical:
|
||
aliases.add(canonical)
|
||
if token and token != canonical:
|
||
aliases.add(token)
|
||
return aliases
|
||
|
||
|
||
def _normalize_theme_binding_value(value: Any) -> str:
|
||
"""Normalize theme binding payload values (including virtual markers)."""
|
||
text = _clean_text(value)
|
||
if not text:
|
||
return ""
|
||
lowered = text.lower()
|
||
if lowered == ALL_THEMES_SENTINEL.lower():
|
||
return ALL_THEMES_SENTINEL
|
||
if text == ALL_THEMES_DISPLAY_NAME:
|
||
return ALL_THEMES_SENTINEL
|
||
return text
|
||
|
||
|
||
def _is_all_theme_marker(value: Any) -> bool:
|
||
"""Return True if the provided value represents the virtual 'all themes' marker."""
|
||
return _normalize_theme_binding_value(value) == ALL_THEMES_SENTINEL
|
||
|
||
|
||
def _normalize_header_label(value: Any) -> str:
|
||
"""Normalise header labels by removing spaces and lowercasing."""
|
||
if value is None:
|
||
return ""
|
||
text = _clean_text(value)
|
||
if not text:
|
||
return ""
|
||
compact = re.sub(r"[\s\u3000]+", "", text)
|
||
return compact.lower()
|
||
|
||
|
||
def _resolve_import_header(value: Any) -> List[str]:
|
||
"""Map an Excel header cell to one or more canonical field names."""
|
||
normalized = _normalize_header_label(value)
|
||
if not normalized:
|
||
return []
|
||
matches: List[str] = []
|
||
for canonical, candidates in _IMPORT_HEADER_ALIASES.items():
|
||
if normalized in candidates:
|
||
matches.append(canonical)
|
||
|
||
if not matches:
|
||
for canonical, keywords in _IMPORT_HEADER_KEYWORDS:
|
||
if any(keyword in normalized for keyword in keywords):
|
||
matches.append(canonical)
|
||
return matches
|
||
|
||
|
||
def _score_import_header(canonical: str, cell_text: str, col_idx: int) -> float:
|
||
"""Heuristic score to choose the best header cell when duplicates exist."""
|
||
score = float(len(cell_text))
|
||
text = cell_text
|
||
if canonical == "risk_content":
|
||
if "内容" in text:
|
||
score += 10
|
||
if "提示" in text:
|
||
score += 4
|
||
if "风险" in text:
|
||
score += 2
|
||
elif canonical == "permit_name":
|
||
if "事项" in text:
|
||
score += 6
|
||
if "名称" in text:
|
||
score += 3
|
||
if "许可" in text:
|
||
score += 2
|
||
elif canonical == "theme_names":
|
||
if "主题" in text:
|
||
score += 4
|
||
elif canonical == "permit_status":
|
||
if "情况" in text or "状态" in text:
|
||
score += 3
|
||
elif canonical == "summary":
|
||
if "摘要" in text:
|
||
score += 3
|
||
elif canonical == "document_no":
|
||
if "文号" in text:
|
||
score += 5
|
||
elif canonical == "remark":
|
||
if "备注" in text:
|
||
score += 3
|
||
|
||
elif canonical == "serial_number":
|
||
if "序号" in text:
|
||
score += 10
|
||
if "风险提示" == text: # Exact match for the weird template case
|
||
score += 5
|
||
if "编号" in text:
|
||
score += 3
|
||
|
||
score += col_idx * 0.1
|
||
return score
|
||
|
||
|
||
def _split_multi_value(value: Any, *, allow_dunhao: bool = False) -> List[str]:
|
||
"""Split multi-value cells using common punctuation characters.
|
||
|
||
默认不把中文顿号（、）视作分隔符，以避免误拆“文化、旅游”等合法的
|
||
许可名称。对于确实需要用顿号分隔的字段（如主题、经营范围等），调用
|
||
方可以显式传入 allow_dunhao=True。
|
||
"""
|
||
|
||
text = _clean_text(value)
|
||
if not text:
|
||
return []
|
||
pattern = TEXT_SPLIT_PATTERN_WITH_DUNHAO if allow_dunhao else TEXT_SPLIT_PATTERN
|
||
return [item.strip() for item in pattern.split(text) if item.strip()]
|
||
|
||
|
||
def _clean_empty(value: Any) -> Optional[str]:
|
||
"""Convert empty strings to None for database writes."""
|
||
text = _clean_text(value)
|
||
return text or None
|
||
|
||
|
||
def _canonicalize_region_label(label: str) -> str:
|
||
"""Return a canonical region label based on known keywords."""
|
||
text = _clean_text(label)
|
||
if not text:
|
||
return ""
|
||
for canonical, keywords in _CANONICAL_REGION_KEYWORDS.items():
|
||
for keyword in keywords:
|
||
if keyword and keyword in text:
|
||
return canonical
|
||
return text
|
||
|
||
|
||
def _normalize_sheet_token(value: Any) -> str:
|
||
"""Normalize sheet/region identifiers for override maps."""
|
||
text = _clean_text(value)
|
||
if not text:
|
||
return ""
|
||
canonical = _canonicalize_region_label(text)
|
||
token_source = canonical or text
|
||
return token_source.lower()
|
||
|
||
|
||
def _normalize_import_row(
|
||
raw_row: Dict[str, Any],
|
||
sheet_name: str,
|
||
sheet_defaults: Optional[Dict[str, Any]] = None,
|
||
) -> Optional[Dict[str, Any]]:
|
||
"""Convert a raw Excel row into canonical import structure."""
|
||
sheet_defaults = sheet_defaults or {}
|
||
|
||
permit_name = _clean_text(raw_row.get("permit_name") or sheet_defaults.get("permit_name"))
|
||
risk_content = _clean_text(raw_row.get("risk_content"))
|
||
serial_number = _clean_text(raw_row.get("serial_number"))
|
||
if not permit_name or not risk_content:
|
||
return None
|
||
|
||
row_index = raw_row.get("row_index")
|
||
legal_basis = _clean_empty(raw_row.get("legal_basis"))
|
||
document_no = _clean_empty(raw_row.get("document_no"))
|
||
summary = _clean_empty(raw_row.get("summary"))
|
||
remark = _clean_empty(raw_row.get("remark"))
|
||
permit_status = _clean_empty(raw_row.get("permit_status") or sheet_defaults.get("permit_status"))
|
||
filler_name = _clean_empty(raw_row.get("filler_name") or sheet_defaults.get("filler_name"))
|
||
unit_name = _clean_empty(raw_row.get("unit_name") or sheet_defaults.get("unit_name"))
|
||
responsible_contact = _clean_empty(
|
||
raw_row.get("responsible_contact") or sheet_defaults.get("responsible_contact")
|
||
)
|
||
jurisdiction_scope = _clean_empty(
|
||
raw_row.get("jurisdiction_scope") or sheet_defaults.get("jurisdiction_scope")
|
||
)
|
||
source_update_date = _clean_empty(
|
||
raw_row.get("source_update_date") or sheet_defaults.get("source_update_date")
|
||
)
|
||
# Validate date format YYYY-MM-DD
|
||
if source_update_date:
|
||
# Simple regex check for YYYY-MM-DD
|
||
if not re.match(r"^\d{4}-\d{2}-\d{2}$", source_update_date):
|
||
# Try to parse or warn? User said "Strictly YYYY-MM-DD"
|
||
# We will store it as is, but maybe log a warning?
|
||
# For now keep as text.
|
||
pass
|
||
|
||
subitem_names = _split_multi_value(
|
||
raw_row.get("subitem_text") or sheet_defaults.get("subitem_text"), allow_dunhao=True
|
||
)
|
||
|
||
return {
|
||
"row_index": int(row_index) if isinstance(row_index, int) else row_index,
|
||
"sheet_name": sheet_name,
|
||
"permit_name": permit_name,
|
||
"risk_content": risk_content,
|
||
"serial_number": serial_number,
|
||
"legal_basis": legal_basis,
|
||
"document_no": document_no,
|
||
"summary": summary,
|
||
"remark": remark,
|
||
"permit_status": permit_status,
|
||
"responsible_contact": responsible_contact,
|
||
"jurisdiction_scope": jurisdiction_scope,
|
||
"subitem_names": subitem_names,
|
||
"filler_name": filler_name,
|
||
"unit_name": unit_name,
|
||
"source_update_date": source_update_date,
|
||
}
|
||
|
||
|
||
def _parse_import_workbook(file_bytes: bytes, filename: str) -> Dict[str, Any]:
|
||
"""Parse the uploaded Excel workbook into structured sheet data."""
|
||
if not file_bytes:
|
||
raise ValueError("上传文件为空")
|
||
|
||
try:
|
||
workbook = load_workbook(BytesIO(file_bytes), data_only=True)
|
||
except InvalidFileException as exc:
|
||
raise ValueError(f"Excel 文件格式无法识别：{exc}") from exc
|
||
except Exception as exc:
|
||
raise ValueError(f"Excel 解析失败：{exc}") from exc
|
||
|
||
sheets: Dict[str, Dict[str, Any]] = {}
|
||
total_rows = 0
|
||
|
||
for worksheet in workbook.worksheets:
|
||
sheet_title = worksheet.title or ""
|
||
sheet_name = _clean_text(sheet_title) or f"Sheet{len(sheets) + 1}"
|
||
|
||
header_row_index: Optional[int] = None
|
||
header_values: List[Any] = []
|
||
header_map: Dict[int, str] = {}
|
||
resolved_by_name: Dict[str, Tuple[int, int, str]] = {}
|
||
metadata_rows: List[Tuple[int, Dict[int, Tuple[str, str]], Tuple[Any, ...]]] = []
|
||
max_header_row = min(worksheet.max_row or 0, 120) or 120
|
||
for row_idx, row_values in enumerate(
|
||
worksheet.iter_rows(min_row=1, max_row=max_header_row, values_only=True),
|
||
start=1,
|
||
):
|
||
if not row_values:
|
||
continue
|
||
if not any(_clean_text(cell) for cell in row_values):
|
||
continue
|
||
|
||
first_cell_text = _clean_text(row_values[0]) if len(row_values) else ""
|
||
is_section_row = bool(first_cell_text) and bool(re.fullmatch(r"[一二三四五六七八九十百零]+", first_cell_text))
|
||
|
||
row_candidate_map: Dict[str, Tuple[int, str, float]] = {}
|
||
for col_idx, header_cell in enumerate(row_values, start=1):
|
||
cell_text = _clean_text(header_cell)
|
||
if not cell_text:
|
||
continue
|
||
if len(cell_text) > 60:
|
||
continue
|
||
canonicals = _resolve_import_header(cell_text)
|
||
if not canonicals:
|
||
continue
|
||
|
||
for canonical in canonicals:
|
||
if (
|
||
is_section_row
|
||
and col_idx >= 3
|
||
and canonical in {"permit_name", "permit_status", "responsible_contact", "subitem_summary"}
|
||
):
|
||
continue
|
||
if len(cell_text) > 35 and canonical in {"permit_name", "risk_content", "summary"}:
|
||
continue
|
||
score = _score_import_header(canonical, cell_text, col_idx)
|
||
previous = row_candidate_map.get(canonical)
|
||
if not previous or score > previous[2] or (score == previous[2] and col_idx < previous[0]):
|
||
row_candidate_map[canonical] = (col_idx, cell_text, score)
|
||
|
||
candidate_map: Dict[int, Tuple[str, str]] = {
|
||
col_idx: (canonical, cell_text)
|
||
for canonical, (col_idx, cell_text, _score) in row_candidate_map.items()
|
||
}
|
||
row_canonicals: Set[str] = {canonical for canonical, _ in candidate_map.values()}
|
||
metadata_rows.append((row_idx, candidate_map, tuple(row_values)))
|
||
|
||
if candidate_map:
|
||
display_entries = []
|
||
for idx, (name, text) in sorted(candidate_map.items()):
|
||
preview = text[:30] + ("…" if len(text) > 30 else "")
|
||
display_entries.append(f"{idx}->{name}({preview})")
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Sheet %s candidate row %d resolved: %s",
|
||
sheet_name,
|
||
row_idx,
|
||
", ".join(display_entries),
|
||
)
|
||
|
||
# Update resolved columns, allowing later rows to refine mapping.
|
||
for col_idx, (canonical, cell_text) in candidate_map.items():
|
||
previous = resolved_by_name.get(canonical)
|
||
if previous and previous[0] != col_idx:
|
||
header_map.pop(previous[0], None)
|
||
resolved_by_name[canonical] = (col_idx, row_idx, cell_text)
|
||
header_map[col_idx] = canonical
|
||
|
||
if (
|
||
"risk_content" in row_canonicals
|
||
and (
|
||
{"legal_basis", "document_no", "summary"}.intersection(row_canonicals)
|
||
or ("permit_name" in row_canonicals)
|
||
or ("permit_name" in resolved_by_name)
|
||
or (any(k in row_canonicals for k in ["permit_status", "filler_name", "unit_name", "source_update_date"]))
|
||
)
|
||
):
|
||
header_row_index = row_idx
|
||
header_values = list(row_values)
|
||
break
|
||
|
||
if header_row_index is None or not header_map:
|
||
logger.warning(
|
||
"[PERMIT-IMPORT] Sheet %s skipped: 未找到包含许可名称与风险内容的表头",
|
||
sheet_name,
|
||
)
|
||
continue
|
||
|
||
sheet_defaults: Dict[str, Any] = {}
|
||
for row_idx, candidate_map, row_values in metadata_rows:
|
||
if row_idx >= header_row_index:
|
||
continue
|
||
for col_idx, (canonical, _label_text) in candidate_map.items():
|
||
value = None
|
||
# Prefer the cell to the right of the label as the value.
|
||
if col_idx < len(row_values):
|
||
value = row_values[col_idx]
|
||
if value is None and (col_idx + 1) < len(row_values):
|
||
value = row_values[col_idx + 1]
|
||
cleaned = _clean_text(value)
|
||
if cleaned and canonical not in sheet_defaults:
|
||
sheet_defaults[canonical] = value
|
||
|
||
# Keep only headers that were identified on or before the selected header row.
|
||
effective_header_map: Dict[int, str] = {}
|
||
for canonical, (col_idx, row_idx, _cell_text) in resolved_by_name.items():
|
||
if row_idx == header_row_index:
|
||
effective_header_map[col_idx] = canonical
|
||
header_map = dict(sorted(effective_header_map.items()))
|
||
has_risk_column = "risk_content" in header_map.values()
|
||
has_permit_column = "permit_name" in header_map.values()
|
||
has_permit_default = bool(_clean_text(sheet_defaults.get("permit_name")))
|
||
if not has_risk_column or (not has_permit_column and not has_permit_default):
|
||
logger.warning(
|
||
"[PERMIT-IMPORT] Sheet %s skipped: 表头缺少许可名称或风险内容列（行 %d） | header_map=%s | permit_default=%s",
|
||
sheet_name,
|
||
header_row_index,
|
||
", ".join(f"{idx}:{name}" for idx, name in sorted(header_map.items())),
|
||
_clean_text(sheet_defaults.get("permit_name") or ""),
|
||
)
|
||
continue
|
||
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Sheet %s header row %d candidates: %s",
|
||
sheet_name,
|
||
header_row_index,
|
||
", ".join(_clean_text(cell) or "<空>" for cell in header_values),
|
||
)
|
||
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Sheet %s resolved headers: %s",
|
||
sheet_name,
|
||
", ".join(f"{idx}->{name}" for idx, name in sorted(header_map.items())),
|
||
)
|
||
|
||
normalized_rows: List[Dict[str, Any]] = []
|
||
for row_idx, row_values in enumerate(
|
||
worksheet.iter_rows(min_row=header_row_index + 1, values_only=True),
|
||
start=header_row_index + 1,
|
||
):
|
||
raw_row: Dict[str, Any] = {"row_index": row_idx}
|
||
has_data = False
|
||
for col_idx, cell_value in enumerate(row_values, start=1):
|
||
canonical = header_map.get(col_idx)
|
||
if not canonical:
|
||
continue
|
||
if cell_value is None or (isinstance(cell_value, str) and not cell_value.strip()):
|
||
continue
|
||
has_data = True
|
||
raw_row[canonical] = cell_value
|
||
if not has_data:
|
||
logger.debug(
|
||
"[PERMIT-IMPORT] Sheet %s row %d ignored: 空行",
|
||
sheet_name,
|
||
row_idx,
|
||
)
|
||
continue
|
||
normalized = _normalize_import_row(raw_row, sheet_name, sheet_defaults)
|
||
if normalized:
|
||
normalized_rows.append(normalized)
|
||
else:
|
||
logger.debug(
|
||
"[PERMIT-IMPORT] Sheet %s row %d ignored: 缺少许可名称或风险内容",
|
||
sheet_name,
|
||
row_idx,
|
||
)
|
||
|
||
if not normalized_rows:
|
||
logger.warning(
|
||
"[PERMIT-IMPORT] Sheet %s skipped: 没有有效数据行",
|
||
sheet_name,
|
||
)
|
||
continue
|
||
|
||
sheets[sheet_name] = {
|
||
"sheet_name": sheet_name,
|
||
"rows": normalized_rows,
|
||
}
|
||
total_rows += len(normalized_rows)
|
||
|
||
if not sheets:
|
||
logger.error(
|
||
"[PERMIT-IMPORT] Workbook %s has no importable sheets (headers missing or rows空)",
|
||
filename,
|
||
)
|
||
raise ValueError("Excel 中未找到可导入的数据")
|
||
|
||
return {
|
||
"filename": os.path.basename(filename or ""),
|
||
"sheets": sheets,
|
||
"total_rows": total_rows,
|
||
}
|
||
|
||
|
||
def _cleanup_expired_import_sessions() -> None:
|
||
"""Remove expired import sessions to avoid unbounded growth."""
|
||
now = time.time()
|
||
expired: List[str] = []
|
||
with _PERMIT_IMPORT_LOCK:
|
||
for session_id, payload in list(_PERMIT_IMPORT_SESSIONS.items()):
|
||
created_at = payload.get("created_at", now)
|
||
if now - created_at > PERMIT_IMPORT_TTL_SECONDS:
|
||
expired.append(session_id)
|
||
for session_id in expired:
|
||
_PERMIT_IMPORT_SESSIONS.pop(session_id, None)
|
||
|
||
|
||
|
||
def _resolve_themes_for_permit(conn: pg.Connection, permit_name: str) -> List[str]:
|
||
"""Resolve theme names from permit_theme_rules."""
|
||
if not permit_name:
|
||
return []
|
||
|
||
cur = conn.cursor()
|
||
# Find theme_ids for this permit (exact match on name for now)
|
||
sql = """
|
||
SELECT t.name
|
||
FROM permit_theme_rules ptr
|
||
JOIN themes t ON t.id = ptr.theme_id
|
||
WHERE ptr.permit_name = %s
|
||
"""
|
||
cur.execute(sql, [permit_name])
|
||
return [row[0] for row in cur.fetchall()]
|
||
|
||
|
||
def start_permit_import_session(
|
||
file_bytes: bytes,
|
||
filename: str,
|
||
*,
|
||
content_type: Optional[str] = None,
|
||
uploaded_by: Optional[str] = None,
|
||
uploader_department_id: Optional[str] = None,
|
||
bound_department_id: Optional[str] = None,
|
||
binding_mode: str = "auto",
|
||
) -> Dict[str, Any]:
|
||
"""Parse the uploaded workbook and create an import session."""
|
||
if not file_bytes:
|
||
raise ValueError("上传的文件为空")
|
||
if len(file_bytes) > MAX_PERMIT_FILE_SIZE_BYTES:
|
||
raise ValueError("上传的文件超过 500KB 限制，请拆分或压缩内容后重试")
|
||
|
||
parsed = _parse_import_workbook(file_bytes, filename)
|
||
workbook_filename = parsed.get("filename") or os.path.basename(filename or "")
|
||
raw_sheet_payloads: Dict[str, Dict[str, Any]] = parsed.get("sheets", {})
|
||
|
||
canonical_sheets: Dict[str, Dict[str, Any]] = {}
|
||
sheet_name_mapping: Dict[str, str] = {}
|
||
for original_name, sheet_data in raw_sheet_payloads.items():
|
||
canonical_name = _canonicalize_region_label(original_name) or original_name
|
||
sheet_name_mapping[original_name] = canonical_name
|
||
bucket = canonical_sheets.setdefault(
|
||
canonical_name,
|
||
{
|
||
"canonical_name": canonical_name,
|
||
"original_names": [],
|
||
"rows": [],
|
||
},
|
||
)
|
||
bucket["original_names"].append(original_name)
|
||
for row in sheet_data.get("rows", []):
|
||
row_copy = dict(row)
|
||
row_copy["sheet_name"] = canonical_name
|
||
bucket["rows"].append(row_copy)
|
||
|
||
sheet_payloads: Dict[str, Dict[str, Any]] = canonical_sheets
|
||
sheet_names = list(sheet_payloads.keys())
|
||
sheet_tokens = sorted({name.lower() for name in sheet_names if name})
|
||
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Workbook %s parsed sheets: %s",
|
||
workbook_filename or filename,
|
||
", ".join(sheet_names),
|
||
)
|
||
for original_name, canonical_name in sheet_name_mapping.items():
|
||
if original_name != canonical_name:
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Sheet alias: %s -> %s",
|
||
original_name,
|
||
canonical_name,
|
||
)
|
||
|
||
region_lookup: Dict[str, Dict[str, Any]] = {}
|
||
permit_lookup: Dict[str, Dict[str, str]] = {}
|
||
|
||
# Pre-resolve department region if bound_department_id is provided
|
||
dept_region_info: Optional[Dict[str, Any]] = None
|
||
binding_mode_lower = (binding_mode or "auto").lower()
|
||
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
|
||
if bound_department_id:
|
||
try:
|
||
dept_info = _fetch_service_department(cur, str(bound_department_id))
|
||
if dept_info and dept_info.get("region_id"):
|
||
dept_region_info = {
|
||
"id": str(dept_info["region_id"]),
|
||
"uuid": dept_info["region_id"],
|
||
"name": str(dept_info.get("region_name") or ""),
|
||
}
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Resolved department region: %s (%s)",
|
||
dept_region_info["name"],
|
||
dept_region_info["id"],
|
||
)
|
||
except Exception as exc:
|
||
logger.warning("[PERMIT-IMPORT] Failed to fetch department region: %s", exc)
|
||
|
||
if sheet_tokens:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, name
|
||
FROM regions
|
||
WHERE LOWER(name) = ANY(%s)
|
||
""",
|
||
(sheet_tokens,),
|
||
)
|
||
for region_id, region_name in cur.fetchall():
|
||
key = str(region_name).lower()
|
||
region_lookup[key] = {
|
||
"id": str(region_id),
|
||
"uuid": region_id,
|
||
"name": str(region_name),
|
||
}
|
||
|
||
if region_lookup:
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Resolved regions from sheet names: %s",
|
||
", ".join(f"{entry['name']}({entry['id']})" for entry in region_lookup.values()),
|
||
)
|
||
|
||
# Build list of all active regions for this session for permit lookup
|
||
active_region_uuids = [entry["uuid"] for entry in region_lookup.values()]
|
||
if dept_region_info and dept_region_info["uuid"] not in active_region_uuids:
|
||
active_region_uuids.append(dept_region_info["uuid"])
|
||
|
||
if active_region_uuids:
|
||
cur.execute(
|
||
"""
|
||
SELECT DISTINCT rtp.region_id, p.id, p.name
|
||
FROM region_theme_permits rtp
|
||
JOIN permits p ON p.id = rtp.permit_id
|
||
WHERE rtp.region_id = ANY(%s)
|
||
""",
|
||
(active_region_uuids,),
|
||
)
|
||
for region_id, permit_id, permit_name in cur.fetchall():
|
||
rid = str(region_id)
|
||
canonical_name = _clean_text(permit_name)
|
||
normalized_permit = _normalize_permit_token(permit_name)
|
||
permit_id_str = str(permit_id)
|
||
if canonical_name:
|
||
permit_lookup.setdefault(rid, {})[canonical_name] = permit_id_str
|
||
if normalized_permit:
|
||
permit_lookup.setdefault(rid, {})[normalized_permit] = permit_id_str
|
||
|
||
sheet_summaries: List[Dict[str, Any]] = []
|
||
session_sheets: Dict[str, Dict[str, Any]] = {}
|
||
|
||
for sheet_name, sheet_data in sheet_payloads.items():
|
||
rows: List[Dict[str, Any]] = sheet_data.get("rows", [])
|
||
permit_groups: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||
for row in rows:
|
||
permit_groups[row["permit_name"]].append(row)
|
||
|
||
region_key = sheet_name.lower()
|
||
region_info = region_lookup.get(region_key)
|
||
|
||
# Override logic for department-based binding
|
||
# If binding_mode is 'department', force the department's region if we have one.
|
||
if binding_mode_lower == "department" and dept_region_info:
|
||
region_info = dept_region_info
|
||
|
||
region_id_str = region_info["id"] if region_info else None
|
||
existing_permits = permit_lookup.get(region_id_str or "", {})
|
||
|
||
duplicate_permits = sorted(
|
||
[
|
||
name
|
||
for name in permit_groups.keys()
|
||
if _normalize_permit_token(name) in existing_permits
|
||
]
|
||
)
|
||
new_permits = sorted(
|
||
[
|
||
name
|
||
for name in permit_groups.keys()
|
||
if _normalize_permit_token(name) not in existing_permits
|
||
]
|
||
)
|
||
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Sheet %s summary: permits=%d, duplicates=%d, new=%d, region=%s (%s)",
|
||
sheet_name,
|
||
len(permit_groups),
|
||
len(duplicate_permits),
|
||
len(new_permits),
|
||
region_info["name"] if region_info else "None",
|
||
region_id_str,
|
||
)
|
||
|
||
session_sheets[sheet_name] = {
|
||
"sheet_name": sheet_name,
|
||
"region_name": region_info["name"] if region_info else sheet_name,
|
||
"region_id": region_id_str,
|
||
"rows": rows,
|
||
"permit_groups": {name: group for name, group in permit_groups.items()},
|
||
"existing_permits": dict(existing_permits),
|
||
"duplicate_permits": duplicate_permits,
|
||
"new_permits": new_permits,
|
||
"original_sheet_names": sheet_data.get("original_names", [sheet_name]),
|
||
}
|
||
|
||
sheet_summaries.append(
|
||
{
|
||
"sheet_name": sheet_name,
|
||
"region_name": region_info["name"] if region_info else sheet_name,
|
||
"region_id": region_id_str or "",
|
||
"row_count": len(rows),
|
||
"permit_count": len(permit_groups),
|
||
"risk_count": len(rows),
|
||
"duplicate_permits": duplicate_permits,
|
||
"new_permits": new_permits,
|
||
"missing_region": region_info is None,
|
||
"original_sheet_names": sheet_data.get("original_names", [sheet_name]),
|
||
}
|
||
)
|
||
|
||
_cleanup_expired_import_sessions()
|
||
session_id = str(uuid.uuid4())
|
||
session_payload = {
|
||
"id": session_id,
|
||
"filename": workbook_filename,
|
||
"created_at": time.time(),
|
||
"sheets": session_sheets,
|
||
"file_bytes": bytes(file_bytes),
|
||
"file_size": len(file_bytes),
|
||
"content_type": content_type or "application/octet-stream",
|
||
"uploaded_by": uploaded_by,
|
||
"uploader_department_id": uploader_department_id,
|
||
"bound_department_id": bound_department_id,
|
||
"binding_mode": (binding_mode or "auto").lower(),
|
||
}
|
||
|
||
with _PERMIT_IMPORT_LOCK:
|
||
_PERMIT_IMPORT_SESSIONS[session_id] = session_payload
|
||
|
||
return {
|
||
"session_id": session_id,
|
||
"filename": workbook_filename,
|
||
"sheet_summaries": sheet_summaries,
|
||
"total_rows": parsed.get("total_rows", 0),
|
||
"expires_in": PERMIT_IMPORT_TTL_SECONDS,
|
||
"file_size": len(file_bytes),
|
||
"content_type": content_type or "application/octet-stream",
|
||
}
|
||
|
||
|
||
def _ensure_region(conn: pg.Connection, region_name: str) -> str:
|
||
name = _clean_text(region_name)
|
||
if not name:
|
||
raise ValueError("地区名称不能为空")
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO regions (name)
|
||
VALUES (%s)
|
||
ON CONFLICT (name)
|
||
DO UPDATE SET name = EXCLUDED.name
|
||
RETURNING id
|
||
""",
|
||
(name,),
|
||
)
|
||
region_id = cur.fetchone()[0]
|
||
return str(region_id)
|
||
|
||
|
||
def _ensure_theme(conn: pg.Connection, theme_name: str) -> str:
|
||
name = _clean_text(theme_name) or "不涉及"
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO themes (name)
|
||
VALUES (%s)
|
||
ON CONFLICT (name)
|
||
DO UPDATE SET name = EXCLUDED.name
|
||
RETURNING id
|
||
""",
|
||
(name,),
|
||
)
|
||
theme_id = cur.fetchone()[0]
|
||
return str(theme_id)
|
||
|
||
|
||
def _ensure_permit(conn: pg.Connection, permit_name: str) -> str:
|
||
name = _clean_text(permit_name)
|
||
if not name:
|
||
raise ValueError("许可名称不能为空")
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO permits (name)
|
||
VALUES (%s)
|
||
ON CONFLICT (name)
|
||
DO UPDATE SET name = EXCLUDED.name
|
||
RETURNING id
|
||
""",
|
||
(name,),
|
||
)
|
||
permit_id = cur.fetchone()[0]
|
||
return str(permit_id)
|
||
|
||
|
||
def _ensure_business_scope(conn: pg.Connection, description: str) -> Optional[str]:
|
||
text = _clean_text(description)
|
||
if not text:
|
||
return None
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO business_scopes (description)
|
||
VALUES (%s)
|
||
ON CONFLICT (description)
|
||
DO UPDATE SET description = EXCLUDED.description
|
||
RETURNING id
|
||
""",
|
||
(text,),
|
||
)
|
||
scope_id = cur.fetchone()[0]
|
||
return str(scope_id)
|
||
|
||
|
||
def _ensure_permit_subitem(conn: pg.Connection, description: str) -> Optional[str]:
|
||
text = _clean_text(description)
|
||
if not text:
|
||
return None
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO permit_subitems (description)
|
||
VALUES (%s)
|
||
ON CONFLICT (description)
|
||
DO UPDATE SET description = EXCLUDED.description
|
||
RETURNING id
|
||
""",
|
||
(text,),
|
||
)
|
||
subitem_id = cur.fetchone()[0]
|
||
return str(subitem_id)
|
||
|
||
|
||
def _ensure_risk(
|
||
conn: pg.Connection,
|
||
*,
|
||
risk_content: str,
|
||
legal_basis: Optional[str],
|
||
document_no: Optional[str],
|
||
summary: Optional[str],
|
||
remark: Optional[str] = None,
|
||
) -> str:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO risks (risk_content, legal_basis, document_no, summary, remark)
|
||
VALUES (%s, %s, %s, %s, %s)
|
||
ON CONFLICT (risk_content, legal_basis, document_no, summary, remark)
|
||
DO UPDATE SET risk_content = EXCLUDED.risk_content
|
||
RETURNING id
|
||
""",
|
||
(risk_content, legal_basis, document_no, summary, remark),
|
||
)
|
||
risk_id = cur.fetchone()[0]
|
||
return str(risk_id)
|
||
|
||
|
||
def _fetch_region_permit_name_map(conn: pg.Connection, region_id: str) -> Dict[str, str]:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT DISTINCT p.name, p.id
|
||
FROM region_theme_permits rtp
|
||
JOIN permits p ON p.id = rtp.permit_id
|
||
WHERE rtp.region_id = %s
|
||
""",
|
||
(region_id,),
|
||
)
|
||
mapping: Dict[str, str] = {}
|
||
for name, pid in cur.fetchall():
|
||
canonical = _clean_text(name)
|
||
token = _normalize_permit_token(name)
|
||
permit_id = str(pid)
|
||
if canonical:
|
||
mapping[canonical] = permit_id
|
||
if token:
|
||
mapping[token] = permit_id
|
||
return mapping
|
||
|
||
|
||
def _backup_permit_before_import(
|
||
conn: pg.Connection,
|
||
*,
|
||
region_id: str,
|
||
permit_id: str,
|
||
region_name: str,
|
||
permit_name: str,
|
||
filename: str,
|
||
sheet_name: str,
|
||
edited_by: Optional[str],
|
||
change_summary: Optional[str],
|
||
) -> Dict[str, Any]:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT risk_id
|
||
FROM region_permit_risks
|
||
WHERE region_id = %s AND permit_id = %s
|
||
ORDER BY risk_id
|
||
FOR UPDATE
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
risk_ids = [str(risk_id) for (risk_id,) in cur.fetchall()]
|
||
if not risk_ids:
|
||
return {"snapshot_count": 0, "batch_id": ""}
|
||
|
||
batch_id = str(uuid.uuid4())
|
||
base_summary = change_summary or (
|
||
f"Excel导入前快照（{filename} › {sheet_name} › {permit_name}）"
|
||
)
|
||
|
||
for idx, risk_id in enumerate(risk_ids, start=1):
|
||
detail_summary = f"{base_summary} - 风险 {idx}/{len(risk_ids)}"
|
||
_create_snapshot_with_connection(
|
||
conn,
|
||
region_id,
|
||
permit_id,
|
||
risk_id,
|
||
edited_by=edited_by,
|
||
change_summary=detail_summary,
|
||
batch_id=batch_id,
|
||
)
|
||
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Captured %d snapshots before overwriting permit %s (%s) in region %s (%s)",
|
||
len(risk_ids),
|
||
permit_id,
|
||
permit_name,
|
||
region_id,
|
||
region_name,
|
||
)
|
||
return {"snapshot_count": len(risk_ids), "batch_id": batch_id}
|
||
|
||
|
||
def _purge_region_permit_relations(conn: pg.Connection, region_id: str, permit_id: str) -> Dict[str, int]:
|
||
cur = conn.cursor()
|
||
delete_counts: Dict[str, int] = {}
|
||
|
||
overrides_available = _PERMIT_THEME_OVERRIDE_SCHEMA_READY
|
||
if overrides_available is None:
|
||
try:
|
||
_ensure_permit_theme_override_schema()
|
||
overrides_available = True
|
||
except Exception:
|
||
overrides_available = False
|
||
|
||
statements = [
|
||
("region_permit_risks", "DELETE FROM region_permit_risks WHERE region_id = %s AND permit_id = %s"),
|
||
("region_permit_scopes", "DELETE FROM region_permit_scopes WHERE region_id = %s AND permit_id = %s"),
|
||
("region_permit_subitems", "DELETE FROM region_permit_subitems WHERE region_id = %s AND permit_id = %s"),
|
||
("region_permit_details", "DELETE FROM region_permit_details WHERE region_id = %s AND permit_id = %s"),
|
||
("region_theme_permits", "DELETE FROM region_theme_permits WHERE region_id = %s AND permit_id = %s"),
|
||
]
|
||
if overrides_available:
|
||
statements.append(
|
||
(
|
||
"region_permit_theme_overrides",
|
||
"DELETE FROM region_permit_theme_overrides WHERE region_id = %s AND permit_id = %s",
|
||
)
|
||
)
|
||
|
||
for key, sql in statements:
|
||
cur.execute(sql, (region_id, permit_id))
|
||
delete_counts[key] = int(cur.rowcount or 0)
|
||
|
||
if _permit_sources_available(conn):
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM permit_sources
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
delete_counts["permit_sources"] = int(cur.rowcount or 0)
|
||
|
||
return delete_counts
|
||
|
||
|
||
def commit_permit_import_session(
|
||
session_id: str,
|
||
sheet_names: Iterable[str],
|
||
*,
|
||
overrides: Optional[Dict[str, Iterable[str]]] = None,
|
||
edited_by: Optional[str] = None,
|
||
change_summary: Optional[str] = None,
|
||
theme_bindings: Optional[Dict[str, Dict[str, Iterable[str]]]] = None,
|
||
) -> Dict[str, Any]:
|
||
if not session_id:
|
||
raise ValueError("导入会话无效")
|
||
|
||
selected_sheets: List[str] = []
|
||
seen_sheet_names: Set[str] = set()
|
||
for sheet_name in sheet_names or []:
|
||
name = _clean_text(sheet_name)
|
||
if not name or name in seen_sheet_names:
|
||
continue
|
||
seen_sheet_names.add(name)
|
||
selected_sheets.append(name)
|
||
|
||
if not selected_sheets:
|
||
raise ValueError("请选择至少一个Sheet进行导入")
|
||
|
||
with _PERMIT_IMPORT_LOCK:
|
||
session_payload = _PERMIT_IMPORT_SESSIONS.get(session_id)
|
||
|
||
if not session_payload:
|
||
raise ValueError("导入会话不存在或已过期，请重新上传Excel文件")
|
||
|
||
session_file_bytes: Optional[bytes] = session_payload.get("file_bytes")
|
||
session_file_content_type: str = session_payload.get("content_type") or "application/octet-stream"
|
||
session_uploaded_by: Optional[str] = session_payload.get("uploaded_by")
|
||
session_uploader_department: Optional[str] = session_payload.get("uploader_department_id")
|
||
session_bound_department: Optional[str] = session_payload.get("bound_department_id")
|
||
session_binding_mode: str = (session_payload.get("binding_mode") or "auto").lower()
|
||
|
||
session_sheets: Dict[str, Dict[str, Any]] = session_payload.get("sheets", {})
|
||
workbook_filename = session_payload.get("filename") or ""
|
||
|
||
overrides_map: Dict[str, Set[str]] = {}
|
||
if overrides:
|
||
for sheet_key, permit_names in overrides.items():
|
||
sheet_token = _normalize_sheet_token(sheet_key) or _clean_text(sheet_key)
|
||
if not sheet_token:
|
||
continue
|
||
bucket = overrides_map.setdefault(sheet_token, set())
|
||
for raw_name in permit_names or []:
|
||
aliases = _permit_name_aliases(raw_name)
|
||
if aliases:
|
||
bucket.update(aliases)
|
||
|
||
theme_binding_map: Dict[str, Dict[str, List[str]]] = {}
|
||
if theme_bindings:
|
||
for sheet_key, permit_map in theme_bindings.items():
|
||
sheet_token = _normalize_sheet_token(sheet_key) or _clean_text(sheet_key)
|
||
if not sheet_token:
|
||
continue
|
||
permit_binding = theme_binding_map.setdefault(sheet_token, {})
|
||
for permit_key, theme_values in (permit_map or {}).items():
|
||
canonical_permit = _clean_text(permit_key)
|
||
permit_token = _normalize_permit_token(permit_key)
|
||
if not canonical_permit and not permit_token:
|
||
continue
|
||
normalized_themes: List[str] = []
|
||
for raw_theme in theme_values or []:
|
||
normalized = _normalize_theme_binding_value(raw_theme)
|
||
if normalized:
|
||
normalized_themes.append(normalized)
|
||
if not normalized_themes:
|
||
continue
|
||
if canonical_permit:
|
||
permit_binding[canonical_permit] = normalized_themes
|
||
if permit_token and permit_token != canonical_permit:
|
||
permit_binding[permit_token] = normalized_themes
|
||
|
||
if overrides_map:
|
||
override_debug = ", ".join(f"{sheet}:{len(names)}" for sheet, names in overrides_map.items())
|
||
logger.info("[PERMIT-IMPORT] Confirmed overrides => %s", override_debug)
|
||
else:
|
||
logger.info("[PERMIT-IMPORT] No override confirmations supplied")
|
||
|
||
default_change_summary = change_summary or (f"Excel导入：{workbook_filename}" if workbook_filename else "Excel导入")
|
||
effective_bound_department_id = None
|
||
if session_binding_mode != "none":
|
||
effective_bound_department_id = session_bound_department or session_uploader_department
|
||
|
||
result: Dict[str, Any] = {
|
||
"session_id": session_id,
|
||
"filename": workbook_filename,
|
||
"processed_sheets": [],
|
||
"created_permits": [],
|
||
"overwritten_permits": [],
|
||
"skipped_permits": [],
|
||
"snapshot_count": 0,
|
||
"risk_count": 0,
|
||
}
|
||
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Committing session %s with %d sheet(s): %s",
|
||
session_id,
|
||
len(selected_sheets),
|
||
", ".join(selected_sheets),
|
||
)
|
||
|
||
stored_file_meta: Optional[Dict[str, Any]] = None
|
||
stored_file_id: Optional[str] = None
|
||
region_theme_cache: Dict[str, List[Dict[str, str]]] = {}
|
||
|
||
region_theme_cache: Dict[str, List[Dict[str, str]]] = {}
|
||
|
||
print("DEBUG: Getting DB connection...")
|
||
with _lic_pg_conn(autocommit=False) as conn:
|
||
try:
|
||
print("DEBUG: Connection acquired. Ensuring schemas...")
|
||
_ensure_service_department_schema(conn)
|
||
_ensure_permit_sources_table(conn)
|
||
_ensure_permit_theme_override_schema(conn)
|
||
print("DEBUG: Schemas ensured. Starting sheet processing...")
|
||
if session_file_bytes:
|
||
_ensure_permit_file_schema(conn)
|
||
stored_file_meta = _insert_permit_file_record(
|
||
conn,
|
||
file_bytes=session_file_bytes,
|
||
filename=workbook_filename or "许可导入.xlsx",
|
||
content_type=session_file_content_type,
|
||
uploaded_by=session_uploaded_by,
|
||
)
|
||
stored_file_id = stored_file_meta.get("file_id")
|
||
cur = conn.cursor()
|
||
|
||
for sheet_name in selected_sheets:
|
||
sheet_data = session_sheets.get(sheet_name)
|
||
if not sheet_data:
|
||
raise ValueError(f"导入会话中未找到名为 {sheet_name} 的Sheet")
|
||
|
||
region_name = sheet_data.get("region_name") or sheet_name
|
||
region_id = sheet_data.get("region_id")
|
||
if region_id:
|
||
# 确保地区仍然存在
|
||
existing_map = _fetch_region_permit_name_map(conn, region_id)
|
||
sheet_data["existing_permits"] = existing_map
|
||
else:
|
||
region_id = _ensure_region(conn, region_name)
|
||
sheet_data["region_id"] = region_id
|
||
sheet_data["existing_permits"] = {}
|
||
|
||
existing_permits = dict(sheet_data.get("existing_permits", {}))
|
||
sheet_clean_name = _clean_text(sheet_name)
|
||
sheet_token = _normalize_sheet_token(sheet_name) or sheet_clean_name
|
||
override_set = overrides_map.get(sheet_token)
|
||
if override_set is None and sheet_clean_name != sheet_token:
|
||
override_set = overrides_map.get(sheet_clean_name)
|
||
if override_set is None:
|
||
override_set = set()
|
||
binding_sheet_map = theme_binding_map.get(sheet_token)
|
||
if binding_sheet_map is None and sheet_clean_name != sheet_token:
|
||
binding_sheet_map = theme_binding_map.get(sheet_clean_name)
|
||
if binding_sheet_map is None:
|
||
binding_sheet_map = {}
|
||
permit_groups: Dict[str, List[Dict[str, Any]]] = sheet_data.get("permit_groups", {})
|
||
|
||
sheet_snapshot_count = 0
|
||
sheet_risk_count = 0
|
||
sheet_created: List[str] = []
|
||
sheet_overwritten: List[str] = []
|
||
sheet_skipped: List[str] = []
|
||
|
||
for permit_name, permit_rows in permit_groups.items():
|
||
print(f"DEBUG: Processing permit '{permit_name}' with {len(permit_rows)} rows...")
|
||
canonical_permit_name = _clean_text(permit_name)
|
||
permit_token = _normalize_permit_token(permit_name)
|
||
if not canonical_permit_name or not permit_token:
|
||
continue
|
||
|
||
permit_id = existing_permits.get(permit_token) or existing_permits.get(canonical_permit_name)
|
||
should_override = (permit_token in override_set) or (canonical_permit_name in override_set)
|
||
permit_modified = False
|
||
|
||
if permit_id and not should_override:
|
||
sheet_skipped.append(canonical_permit_name)
|
||
result["skipped_permits"].append(
|
||
{
|
||
"sheet": sheet_name,
|
||
"permit_name": canonical_permit_name,
|
||
"region_id": region_id,
|
||
"reason": "exists",
|
||
}
|
||
)
|
||
continue
|
||
|
||
if permit_id:
|
||
backup_info = _backup_permit_before_import(
|
||
conn,
|
||
region_id=region_id,
|
||
permit_id=permit_id,
|
||
region_name=region_name,
|
||
permit_name=canonical_permit_name,
|
||
filename=workbook_filename,
|
||
sheet_name=sheet_name,
|
||
edited_by=edited_by,
|
||
change_summary=default_change_summary,
|
||
)
|
||
sheet_snapshot_count += backup_info["snapshot_count"]
|
||
result["snapshot_count"] += backup_info["snapshot_count"]
|
||
_purge_region_permit_relations(conn, region_id, permit_id)
|
||
sheet_overwritten.append(canonical_permit_name)
|
||
result["overwritten_permits"].append(
|
||
{
|
||
"sheet": sheet_name,
|
||
"permit_name": canonical_permit_name,
|
||
"region_id": region_id,
|
||
"snapshot_batch_id": backup_info.get("batch_id", ""),
|
||
}
|
||
)
|
||
permit_modified = True
|
||
permit_modified = True
|
||
else:
|
||
print(f"DEBUG: Creating new permit '{canonical_permit_name}'...")
|
||
permit_id = _ensure_permit(conn, canonical_permit_name)
|
||
print(f"DEBUG: Created permit id={permit_id}")
|
||
for alias in _permit_name_aliases(canonical_permit_name) or {canonical_permit_name}:
|
||
existing_permits[alias] = permit_id
|
||
sheet_created.append(canonical_permit_name)
|
||
result["created_permits"].append(
|
||
{
|
||
"sheet": sheet_name,
|
||
"permit_name": canonical_permit_name,
|
||
"region_id": region_id,
|
||
}
|
||
)
|
||
permit_modified = True
|
||
|
||
# Insert details
|
||
if permit_rows:
|
||
print("DEBUG: Inserting permit details...")
|
||
# ... (lines 1412+)
|
||
pass
|
||
|
||
binding_override = None
|
||
for alias in _permit_name_aliases(canonical_permit_name) or {canonical_permit_name}:
|
||
binding_override = binding_sheet_map.get(alias)
|
||
if binding_override:
|
||
break
|
||
override_theme_names: Set[str] = set()
|
||
binds_all_themes = False
|
||
if binding_override:
|
||
for override_value in binding_override:
|
||
if _is_all_theme_marker(override_value):
|
||
binds_all_themes = True
|
||
else:
|
||
cleaned_override = _clean_text(override_value)
|
||
if cleaned_override:
|
||
override_theme_names.add(cleaned_override)
|
||
|
||
theme_names: Set[str] = set()
|
||
if not binding_override or binds_all_themes:
|
||
# Prioritize automated resolution from rules table
|
||
resolved_rule_themes = _resolve_themes_for_permit(conn, canonical_permit_name)
|
||
if resolved_rule_themes:
|
||
theme_names.update(resolved_rule_themes)
|
||
|
||
scope_descriptions: Set[str] = set()
|
||
subitem_names: Set[str] = set()
|
||
permit_status_val: Optional[str] = None
|
||
subitem_summary_val: Optional[str] = None
|
||
responsible_contact_val: Optional[str] = None
|
||
jurisdiction_scope_val: Optional[str] = None
|
||
|
||
for row in permit_rows:
|
||
# Workbook theme fallback removed - template has no theme column
|
||
scope_descriptions.update(row.get("scope_descriptions") or [])
|
||
subitem_names.update(row.get("subitem_names") or [])
|
||
if not permit_status_val and row.get("permit_status"):
|
||
permit_status_val = row.get("permit_status")
|
||
if not subitem_summary_val and row.get("subitem_summary"):
|
||
subitem_summary_val = row.get("subitem_summary")
|
||
if not responsible_contact_val and row.get("responsible_contact"):
|
||
responsible_contact_val = row.get("responsible_contact")
|
||
if not jurisdiction_scope_val and row.get("jurisdiction_scope"):
|
||
jurisdiction_scope_val = row.get("jurisdiction_scope")
|
||
|
||
if binding_override and not binds_all_themes and override_theme_names:
|
||
theme_names = set(override_theme_names)
|
||
|
||
if binds_all_themes and region_id:
|
||
cached_theme_options = region_theme_cache.get(region_id)
|
||
if cached_theme_options is None:
|
||
fetched_map = _fetch_region_theme_map(conn, [region_id])
|
||
cached_theme_options = fetched_map.get(region_id, [])
|
||
region_theme_cache[region_id] = cached_theme_options
|
||
for option in cached_theme_options or []:
|
||
option_label = _clean_text(option.get("name"))
|
||
if option_label:
|
||
theme_names.add(option_label)
|
||
|
||
# resolved_themes check moved up
|
||
|
||
if not theme_names:
|
||
theme_names.add("不涉及")
|
||
|
||
theme_ids: List[str] = []
|
||
for theme_name in sorted(theme_names):
|
||
if _is_all_theme_marker(theme_name):
|
||
continue
|
||
theme_id = _ensure_theme(conn, theme_name)
|
||
theme_ids.append(theme_id)
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_themes (region_id, theme_id)
|
||
VALUES (%s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, theme_id),
|
||
)
|
||
if cur.rowcount:
|
||
cache_entry = region_theme_cache.setdefault(region_id, [])
|
||
if not any(entry.get("id") == theme_id for entry in cache_entry):
|
||
cache_entry.append({"id": theme_id, "name": theme_name})
|
||
_propagate_all_theme_bindings(conn, region_id, theme_id)
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_theme_permits (region_id, theme_id, permit_id)
|
||
VALUES (%s, %s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, theme_id, permit_id),
|
||
)
|
||
|
||
if binds_all_themes:
|
||
_set_permit_bind_all_themes(
|
||
conn,
|
||
region_id,
|
||
permit_id,
|
||
created_by=edited_by or session_uploaded_by,
|
||
)
|
||
elif binding_override:
|
||
_clear_permit_bind_all_themes(conn, region_id, permit_id)
|
||
|
||
for scope_desc in sorted(scope_descriptions):
|
||
scope_id = _ensure_business_scope(conn, scope_desc)
|
||
if not scope_id:
|
||
continue
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_scopes (region_id, permit_id, scope_id)
|
||
VALUES (%s, %s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, permit_id, scope_id),
|
||
)
|
||
|
||
for subitem_desc in sorted(subitem_names):
|
||
subitem_id = _ensure_permit_subitem(conn, subitem_desc)
|
||
if not subitem_id:
|
||
continue
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_subitems (region_id, permit_id, subitem_id)
|
||
VALUES (%s, %s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, permit_id, subitem_id),
|
||
)
|
||
|
||
_ensure_contact_info_column(conn)
|
||
|
||
# Resolve contact info for the permit
|
||
contact_info_val = None
|
||
if effective_bound_department_id:
|
||
dept_data = _fetch_service_department(cur, effective_bound_department_id)
|
||
if dept_data:
|
||
contact_info_val = dept_data.get("phone")
|
||
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_details (
|
||
region_id,
|
||
permit_id,
|
||
permit_status,
|
||
subitem_summary,
|
||
responsible_contact,
|
||
jurisdiction_scope,
|
||
filler_name,
|
||
unit_name,
|
||
source_update_date,
|
||
contact_info
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON CONFLICT (region_id, permit_id)
|
||
DO UPDATE SET
|
||
permit_status = EXCLUDED.permit_status,
|
||
subitem_summary = EXCLUDED.subitem_summary,
|
||
responsible_contact = EXCLUDED.responsible_contact,
|
||
jurisdiction_scope = EXCLUDED.jurisdiction_scope,
|
||
updated_at = now(),
|
||
filler_name = EXCLUDED.filler_name,
|
||
unit_name = EXCLUDED.unit_name,
|
||
source_update_date = EXCLUDED.source_update_date,
|
||
contact_info = COALESCE(EXCLUDED.contact_info, region_permit_details.contact_info)
|
||
""",
|
||
(
|
||
region_id,
|
||
permit_id,
|
||
permit_status_val,
|
||
subitem_summary_val,
|
||
responsible_contact_val,
|
||
jurisdiction_scope_val,
|
||
row.get("filler_name"),
|
||
row.get("unit_name"),
|
||
row.get("source_update_date"),
|
||
contact_info_val,
|
||
),
|
||
)
|
||
|
||
for row in permit_rows:
|
||
risk_id = _ensure_risk(
|
||
conn,
|
||
risk_content=row.get("risk_content", ""),
|
||
legal_basis=row.get("legal_basis"),
|
||
document_no=row.get("document_no"),
|
||
summary=row.get("summary"),
|
||
remark=row.get("remark"),
|
||
)
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_risks (region_id, permit_id, risk_id, serial_number)
|
||
VALUES (%s, %s, %s, %s)
|
||
ON CONFLICT (region_id, permit_id, risk_id)
|
||
DO UPDATE SET serial_number = EXCLUDED.serial_number
|
||
""",
|
||
(region_id, permit_id, risk_id, row.get("serial_number")),
|
||
)
|
||
sheet_risk_count += len(permit_rows)
|
||
result["risk_count"] += len(permit_rows)
|
||
|
||
source_detail_payload = {
|
||
"sheet": sheet_name,
|
||
"permit": canonical_permit_name,
|
||
"risk_rows": len(permit_rows),
|
||
"imported_at": datetime.utcnow().isoformat(),
|
||
}
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO permit_sources (
|
||
region_id,
|
||
permit_id,
|
||
source_type,
|
||
source_name,
|
||
source_detail,
|
||
uploader_department_id,
|
||
bound_department_id,
|
||
created_at,
|
||
updated_at
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, now(), now())
|
||
ON CONFLICT (region_id, permit_id)
|
||
DO UPDATE SET
|
||
source_type = EXCLUDED.source_type,
|
||
source_name = EXCLUDED.source_name,
|
||
source_detail = EXCLUDED.source_detail,
|
||
uploader_department_id = EXCLUDED.uploader_department_id,
|
||
bound_department_id = EXCLUDED.bound_department_id,
|
||
updated_at = now()
|
||
""",
|
||
(
|
||
region_id,
|
||
permit_id,
|
||
"excel",
|
||
workbook_filename or sheet_name,
|
||
json.dumps(source_detail_payload, ensure_ascii=False),
|
||
session_uploader_department,
|
||
effective_bound_department_id,
|
||
),
|
||
)
|
||
if stored_file_id and permit_modified:
|
||
_link_file_to_permit(
|
||
conn,
|
||
file_id=stored_file_id,
|
||
region_id=region_id,
|
||
permit_id=permit_id,
|
||
created_by=session_uploaded_by or edited_by,
|
||
)
|
||
|
||
result["processed_sheets"].append(
|
||
{
|
||
"sheet_name": sheet_name,
|
||
"region_id": region_id,
|
||
"region_name": region_name,
|
||
"created_permits": sheet_created,
|
||
"overwritten_permits": sheet_overwritten,
|
||
"skipped_permits": sheet_skipped,
|
||
"snapshot_count": sheet_snapshot_count,
|
||
"risk_count": sheet_risk_count,
|
||
}
|
||
)
|
||
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Sheet %s processed (region=%s) -> created=%d, overwritten=%d, skipped=%d, risks=%d, snapshots=%d",
|
||
sheet_name,
|
||
region_id,
|
||
len(sheet_created),
|
||
len(sheet_overwritten),
|
||
len(sheet_skipped),
|
||
sheet_risk_count,
|
||
sheet_snapshot_count,
|
||
)
|
||
|
||
conn.commit()
|
||
except Exception:
|
||
conn.rollback()
|
||
raise
|
||
|
||
result["file_attachment"] = stored_file_meta
|
||
|
||
# Log the import operation
|
||
try:
|
||
log_operation(
|
||
operator=edited_by or session_uploaded_by or "unknown",
|
||
operation_type="IMPORT",
|
||
target_type="PERMIT_BATCH",
|
||
target_name=workbook_filename,
|
||
change_summary=change_summary or f"Imported {len(sheet_names)} sheets from {workbook_filename}",
|
||
details={
|
||
"filename": workbook_filename,
|
||
"sheets_processed": len(sheet_names),
|
||
"created_count": len(result["created_permits"]),
|
||
"overwritten_count": len(result["overwritten_permits"]),
|
||
"skipped_count": len(result["skipped_permits"]),
|
||
"snapshot_count": result["snapshot_count"]
|
||
}
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"[PERMIT-IMPORT] Failed to log operation: {e}")
|
||
|
||
with _PERMIT_IMPORT_LOCK:
|
||
_PERMIT_IMPORT_SESSIONS.pop(session_id, None)
|
||
|
||
logger.info(
|
||
"[PERMIT-IMPORT] Completed session %s: created=%d overwritten=%d skipped=%d snapshots=%d risks=%d",
|
||
session_id,
|
||
len(result["created_permits"]),
|
||
len(result["overwritten_permits"]),
|
||
len(result["skipped_permits"]),
|
||
result["snapshot_count"],
|
||
result["risk_count"],
|
||
)
|
||
|
||
return result
|
||
|
||
|
||
def describe_permit_import_session(session_id: str) -> Dict[str, Any]:
|
||
"""Return preview data for a pending permit import session."""
|
||
if not session_id:
|
||
raise ValueError("导入会话无效")
|
||
|
||
with _PERMIT_IMPORT_LOCK:
|
||
session_payload = _PERMIT_IMPORT_SESSIONS.get(session_id)
|
||
|
||
if not session_payload:
|
||
raise ValueError("导入会话不存在或已过期，请重新上传Excel文件")
|
||
|
||
session_sheets: Dict[str, Dict[str, Any]] = session_payload.get("sheets") or {}
|
||
if not session_sheets:
|
||
raise ValueError("导入会话不存在或已过期，请重新上传Excel文件")
|
||
|
||
region_ids = sorted({sheet.get("region_id") for sheet in session_sheets.values() if sheet.get("region_id")})
|
||
region_theme_map: Dict[str, List[Dict[str, str]]] = {}
|
||
all_theme_options: List[Dict[str, str]] = []
|
||
with _lic_pg_conn() as conn:
|
||
all_theme_options = _fetch_all_theme_options(conn)
|
||
if region_ids:
|
||
region_theme_map = _fetch_region_theme_map(conn, region_ids)
|
||
|
||
preview_sheets: List[Dict[str, Any]] = []
|
||
total_permits = 0
|
||
total_risks = 0
|
||
|
||
with _lic_pg_conn() as conn:
|
||
for sheet_name in sorted(session_sheets.keys()):
|
||
sheet_data = session_sheets[sheet_name]
|
||
region_id = sheet_data.get("region_id") or ""
|
||
region_name = sheet_data.get("region_name") or sheet_name
|
||
duplicate_permits = set(sheet_data.get("duplicate_permits") or [])
|
||
new_permits = set(sheet_data.get("new_permits") or [])
|
||
permit_groups: Dict[str, List[Dict[str, Any]]] = sheet_data.get("permit_groups") or {}
|
||
|
||
permit_summaries: List[Dict[str, Any]] = []
|
||
sheet_risk_total = 0
|
||
|
||
for permit_name in sorted(permit_groups.keys()):
|
||
permit_rows = permit_groups[permit_name]
|
||
sheet_risk_total += len(permit_rows)
|
||
total_risks += len(permit_rows)
|
||
|
||
permit_risks = []
|
||
common_meta = {
|
||
"responsible_contact": "",
|
||
"jurisdiction_scope": "",
|
||
"unit_name": "",
|
||
"permit_status": ""
|
||
}
|
||
|
||
for row in permit_rows:
|
||
permit_risks.append({
|
||
"serial_number": row.get("serial_number"),
|
||
"risk_content": row.get("risk_content"),
|
||
"legal_basis": row.get("legal_basis"),
|
||
"document_no": row.get("document_no"),
|
||
"summary": row.get("summary"),
|
||
"remark": row.get("remark"),
|
||
})
|
||
for key in common_meta:
|
||
if not common_meta[key] and row.get(key):
|
||
common_meta[key] = row.get(key)
|
||
|
||
# Try to resolve themes via rules (Base Table Logic)
|
||
resolved_theme_names = _resolve_themes_for_permit(conn, permit_name)
|
||
|
||
permit_summaries.append(
|
||
{
|
||
"permit_name": permit_name,
|
||
"canonical_name": _clean_text(permit_name),
|
||
"risk_count": len(permit_rows),
|
||
"is_duplicate": permit_name in duplicate_permits,
|
||
"is_new": permit_name in new_permits,
|
||
"default_theme_names": [],
|
||
"resolved_theme_names": resolved_theme_names,
|
||
"risks": permit_risks,
|
||
"common_meta": common_meta,
|
||
"sample_serial": permit_rows[0].get("serial_number") if permit_rows else None,
|
||
"sample_risk": permit_rows[0].get("risk_content") if permit_rows else "",
|
||
}
|
||
)
|
||
|
||
total_permits += len(permit_summaries)
|
||
|
||
theme_options: List[Dict[str, Any]] = []
|
||
seen_theme_labels: Set[str] = set()
|
||
|
||
theme_options.append(
|
||
{
|
||
"id": ALL_THEMES_SENTINEL,
|
||
"name": ALL_THEMES_DISPLAY_NAME,
|
||
"source": "virtual",
|
||
"description": "选择后将与当前及未来的新主题自动绑定",
|
||
"is_all": True,
|
||
}
|
||
)
|
||
seen_theme_labels.add(ALL_THEMES_SENTINEL.lower())
|
||
seen_theme_labels.add(ALL_THEMES_DISPLAY_NAME.lower())
|
||
|
||
candidate_theme_lists: List[List[Dict[str, str]]] = []
|
||
region_theme_options = region_theme_map.get(region_id)
|
||
if region_theme_options:
|
||
candidate_theme_lists.append(region_theme_options)
|
||
if all_theme_options:
|
||
candidate_theme_lists.append(all_theme_options)
|
||
|
||
for option_list in candidate_theme_lists:
|
||
for option in option_list:
|
||
label = option.get("name") or ""
|
||
normalized = label.lower()
|
||
if normalized in seen_theme_labels:
|
||
continue
|
||
seen_theme_labels.add(normalized)
|
||
theme_options.append(
|
||
{
|
||
"id": option.get("id") or "",
|
||
"name": label,
|
||
"source": "existing",
|
||
}
|
||
)
|
||
|
||
# Workbook themes are no longer supported as the template has no theme column
|
||
pass
|
||
|
||
preview_sheets.append(
|
||
{
|
||
"sheet_name": sheet_name,
|
||
"region_name": region_name,
|
||
"region_id": region_id,
|
||
"missing_region": not bool(region_id),
|
||
"theme_options": theme_options,
|
||
"permits": permit_summaries,
|
||
"duplicate_permits": list(duplicate_permits),
|
||
"new_permits": list(new_permits),
|
||
"permit_count": len(permit_summaries),
|
||
"risk_count": sheet_risk_total,
|
||
}
|
||
)
|
||
|
||
return {
|
||
"session_id": session_id,
|
||
"filename": session_payload.get("filename") or "",
|
||
"sheet_count": len(preview_sheets),
|
||
"permit_total": total_permits,
|
||
"risk_total": total_risks,
|
||
"sheets": preview_sheets,
|
||
}
|
||
|
||
|
||
def _permit_sources_available(conn: pg.Connection) -> bool:
|
||
"""Return True if permit_sources table exists (cached)."""
|
||
global _PERMIT_SOURCES_TABLE_PRESENT
|
||
if _PERMIT_SOURCES_TABLE_PRESENT is True:
|
||
return True
|
||
if _PERMIT_SOURCES_TABLE_PRESENT is False:
|
||
return False
|
||
|
||
with _PERMIT_SOURCES_TABLE_LOCK:
|
||
if _PERMIT_SOURCES_TABLE_PRESENT is not None:
|
||
return bool(_PERMIT_SOURCES_TABLE_PRESENT)
|
||
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT 1
|
||
FROM information_schema.tables
|
||
WHERE table_schema = 'public'
|
||
AND table_name = 'permit_sources'
|
||
LIMIT 1
|
||
"""
|
||
)
|
||
exists = cur.fetchone() is not None
|
||
_PERMIT_SOURCES_TABLE_PRESENT = exists
|
||
return exists
|
||
|
||
|
||
def _ensure_contact_info_column(conn: pg.Connection) -> None:
|
||
"""Ensure contact_info column exists in region_permit_details."""
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
ALTER TABLE IF EXISTS region_permit_details
|
||
ADD COLUMN IF NOT EXISTS contact_info text
|
||
"""
|
||
)
|
||
# We don't commit here as it might be part of a larger transaction
|
||
|
||
|
||
|
||
def _create_permit_sources_schema(conn: pg.Connection) -> None:
|
||
"""Create permit_sources table and ancillary indexes if missing."""
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS permit_sources (
|
||
region_id uuid NOT NULL REFERENCES regions(id) ON DELETE CASCADE,
|
||
permit_id uuid NOT NULL REFERENCES permits(id) ON DELETE CASCADE,
|
||
source_type text NOT NULL,
|
||
source_name text NOT NULL,
|
||
source_detail text,
|
||
uploader_department_id uuid REFERENCES service_departments(id),
|
||
bound_department_id uuid REFERENCES service_departments(id),
|
||
created_at timestamptz NOT NULL DEFAULT now(),
|
||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||
PRIMARY KEY (region_id, permit_id)
|
||
)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE INDEX IF NOT EXISTS permit_sources_source_name_idx
|
||
ON permit_sources (source_name)
|
||
"""
|
||
)
|
||
# 兼容已有表：补齐绑定相关字段及索引
|
||
cur.execute(
|
||
"""
|
||
ALTER TABLE IF EXISTS permit_sources
|
||
ADD COLUMN IF NOT EXISTS uploader_department_id uuid REFERENCES service_departments(id)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
ALTER TABLE IF EXISTS permit_sources
|
||
ADD COLUMN IF NOT EXISTS bound_department_id uuid REFERENCES service_departments(id)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE INDEX IF NOT EXISTS idx_permit_sources_bound_dept
|
||
ON permit_sources (bound_department_id)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE INDEX IF NOT EXISTS idx_permit_sources_uploader
|
||
ON permit_sources (uploader_department_id)
|
||
"""
|
||
)
|
||
|
||
|
||
def _ensure_permit_sources_table(conn: Optional[pg.Connection] = None) -> None:
|
||
"""Ensure the permit_sources table exists and cache the result."""
|
||
global _PERMIT_SOURCES_TABLE_PRESENT
|
||
if _PERMIT_SOURCES_TABLE_PRESENT is True:
|
||
return
|
||
|
||
with _PERMIT_SOURCES_TABLE_LOCK:
|
||
if _PERMIT_SOURCES_TABLE_PRESENT is True:
|
||
return
|
||
|
||
if conn is not None:
|
||
original_autocommit = conn.autocommit
|
||
try:
|
||
conn.autocommit = True
|
||
_create_permit_sources_schema(conn)
|
||
finally:
|
||
conn.autocommit = original_autocommit
|
||
else:
|
||
with _lic_pg_conn(autocommit=True) as ensure_conn:
|
||
_create_permit_sources_schema(ensure_conn)
|
||
|
||
_PERMIT_SOURCES_TABLE_PRESENT = True
|
||
|
||
|
||
def _create_service_department_schema(conn: pg.Connection) -> None:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS service_departments (
|
||
id uuid PRIMARY KEY,
|
||
name text NOT NULL,
|
||
code text NOT NULL UNIQUE,
|
||
phone text,
|
||
parent_id uuid REFERENCES service_departments(id) ON DELETE SET NULL,
|
||
region_id uuid REFERENCES regions(id) ON DELETE SET NULL,
|
||
description text,
|
||
created_at timestamptz NOT NULL DEFAULT now(),
|
||
updated_at timestamptz NOT NULL DEFAULT now()
|
||
)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE UNIQUE INDEX IF NOT EXISTS service_departments_name_idx
|
||
ON service_departments (name)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
ALTER TABLE IF EXISTS service_departments
|
||
ADD COLUMN IF NOT EXISTS parent_id uuid REFERENCES service_departments(id) ON DELETE SET NULL
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
ALTER TABLE IF EXISTS service_departments
|
||
ADD COLUMN IF NOT EXISTS phone text
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
ALTER TABLE IF EXISTS service_departments
|
||
ADD COLUMN IF NOT EXISTS grade int DEFAULT 0
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE INDEX IF NOT EXISTS service_departments_parent_idx
|
||
ON service_departments (parent_id)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE INDEX IF NOT EXISTS service_departments_grade_idx
|
||
ON service_departments (grade)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS service_department_permits (
|
||
department_id uuid NOT NULL REFERENCES service_departments(id) ON DELETE CASCADE,
|
||
region_id uuid NOT NULL REFERENCES regions(id) ON DELETE CASCADE,
|
||
permit_id uuid NOT NULL REFERENCES permits(id) ON DELETE CASCADE,
|
||
created_at timestamptz NOT NULL DEFAULT now(),
|
||
created_by text,
|
||
PRIMARY KEY (department_id, region_id, permit_id)
|
||
)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE INDEX IF NOT EXISTS service_dept_permits_region_idx
|
||
ON service_department_permits (region_id, permit_id)
|
||
"""
|
||
)
|
||
|
||
|
||
def _ensure_service_department_schema(conn: Optional[pg.Connection] = None) -> None:
|
||
global _SERVICE_DEPARTMENT_SCHEMA_READY
|
||
if _SERVICE_DEPARTMENT_SCHEMA_READY:
|
||
return
|
||
with _SERVICE_DEPARTMENT_SCHEMA_LOCK:
|
||
if _SERVICE_DEPARTMENT_SCHEMA_READY:
|
||
return
|
||
if conn is not None:
|
||
original_autocommit = conn.autocommit
|
||
try:
|
||
conn.autocommit = True
|
||
_create_service_department_schema(conn)
|
||
finally:
|
||
conn.autocommit = original_autocommit
|
||
else:
|
||
with _lic_pg_conn(autocommit=True) as ensure_conn:
|
||
_create_service_department_schema(ensure_conn)
|
||
_SERVICE_DEPARTMENT_SCHEMA_READY = True
|
||
|
||
|
||
_SERVICE_DEPARTMENT_SELECT = """
|
||
SELECT
|
||
sd.id,
|
||
sd.name,
|
||
sd.code,
|
||
sd.phone,
|
||
sd.parent_id,
|
||
parent.name AS parent_name,
|
||
sd.region_id,
|
||
r.name AS region_name,
|
||
sd.description,
|
||
sd.grade,
|
||
sd.unit_level,
|
||
sd.created_at,
|
||
sd.updated_at
|
||
FROM service_departments sd
|
||
LEFT JOIN service_departments parent ON parent.id = sd.parent_id
|
||
LEFT JOIN regions r ON r.id = sd.region_id
|
||
"""
|
||
|
||
|
||
def _serialize_service_department_row(record: Dict[str, Any]) -> Dict[str, Any]:
|
||
return {
|
||
"id": _to_optional_str(record.get("id")),
|
||
"name": record.get("name"),
|
||
"code": record.get("code"),
|
||
"phone": record.get("phone"),
|
||
"parent_id": _to_optional_str(record.get("parent_id")),
|
||
"parent_name": record.get("parent_name"),
|
||
"region_id": _to_optional_str(record.get("region_id")),
|
||
"region_name": record.get("region_name"),
|
||
"description": record.get("description"),
|
||
"grade": record.get("grade", 0),
|
||
"unit_level": record.get("unit_level", "unit"),
|
||
"created_at": _to_isoformat(record.get("created_at")),
|
||
"updated_at": _to_isoformat(record.get("updated_at")),
|
||
}
|
||
|
||
|
||
def _fetch_service_department(cur: pg.Cursor, department_id: str) -> Optional[Dict[str, Any]]:
|
||
cur.execute(
|
||
_SERVICE_DEPARTMENT_SELECT + " WHERE sd.id = %s LIMIT 1",
|
||
(department_id,),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return None
|
||
columns = tuple(col[0] for col in cur.description)
|
||
record = {columns[idx]: row[idx] for idx in range(len(columns))}
|
||
return _serialize_service_department_row(record)
|
||
|
||
|
||
def list_service_departments(region_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
||
"""List service departments, optionally filtered by region.
|
||
|
||
Args:
|
||
region_id: If provided, only return departments associated with this region
|
||
|
||
Returns:
|
||
List of department dictionaries
|
||
"""
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_service_department_schema(conn)
|
||
cur = conn.cursor()
|
||
|
||
if region_id:
|
||
# Get departments associated with the specific region through service_departments.region_id
|
||
# This includes both direct region match and hierarchical matching (child regions)
|
||
sql = _SERVICE_DEPARTMENT_SELECT + """
|
||
WHERE sd.region_id = %s
|
||
ORDER BY sd.created_at ASC
|
||
"""
|
||
cur.execute(sql, (region_id,))
|
||
else:
|
||
# Get all departments
|
||
cur.execute(_SERVICE_DEPARTMENT_SELECT + " ORDER BY sd.created_at ASC")
|
||
rows = cur.fetchall()
|
||
columns = tuple(col[0] for col in cur.description)
|
||
|
||
departments: List[Dict[str, Any]] = []
|
||
for row in rows:
|
||
record = {columns[idx]: row[idx] for idx in range(len(columns))}
|
||
departments.append(_serialize_service_department_row(record))
|
||
return departments
|
||
|
||
|
||
def build_service_department_tree() -> List[Dict[str, Any]]:
|
||
"""
|
||
Build a tree structure of service departments with parent-child relationships.
|
||
Returns a list of root-level departments with nested children.
|
||
"""
|
||
departments = list_service_departments()
|
||
|
||
# Create a dictionary for fast lookup
|
||
dept_dict = {dept["id"]: dept.copy() for dept in departments}
|
||
|
||
# Initialize children arrays
|
||
for dept_id in dept_dict:
|
||
dept_dict[dept_id]["children"] = []
|
||
|
||
# Build tree structure
|
||
tree: List[Dict[str, Any]] = []
|
||
for dept_id, dept in dept_dict.items():
|
||
if dept.get("parent_id"):
|
||
# Has a parent, add to parent's children
|
||
parent_id = dept["parent_id"]
|
||
if parent_id in dept_dict:
|
||
dept_dict[parent_id]["children"].append(dept)
|
||
else:
|
||
# Root level department
|
||
tree.append(dept)
|
||
|
||
# Sort tree recursively
|
||
def sort_tree(nodes: List[Dict[str, Any]]) -> None:
|
||
nodes.sort(key=lambda x: x.get("name", ""))
|
||
for node in nodes:
|
||
if node.get("children"):
|
||
sort_tree(node["children"])
|
||
|
||
sort_tree(tree)
|
||
return tree
|
||
|
||
|
||
def get_subordinate_departments(department_id: str, level: Optional[str] = None) -> List[str]:
|
||
"""获取指定单位的所有下级单位ID列表。
|
||
|
||
Args:
|
||
department_id: 父单位ID
|
||
level: 指定下级单位的unit_level（None表示所有级别）
|
||
|
||
Returns:
|
||
下级单位ID列表
|
||
"""
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
if level:
|
||
cur.execute("""
|
||
SELECT id
|
||
FROM service_departments
|
||
WHERE parent_id = %s AND unit_level = %s
|
||
""", (department_id, level))
|
||
else:
|
||
cur.execute("""
|
||
SELECT id
|
||
FROM service_departments
|
||
WHERE parent_id = %s
|
||
""", (department_id,))
|
||
return [str(row[0]) for row in cur.fetchall()]
|
||
|
||
|
||
def _fetch_department_descendants(cur: pg.Cursor, root_id: str) -> List[str]:
|
||
"""返回包含自身在内的下级单位ID列表（递归）。"""
|
||
if not root_id:
|
||
return []
|
||
cur.execute(
|
||
"""
|
||
WITH RECURSIVE sub AS (
|
||
SELECT id FROM service_departments WHERE id = %s
|
||
UNION ALL
|
||
SELECT sd.id
|
||
FROM service_departments sd
|
||
JOIN sub ON sd.parent_id = sub.id
|
||
)
|
||
SELECT id FROM sub
|
||
""",
|
||
(root_id,),
|
||
)
|
||
return [str(row[0]) for row in cur.fetchall()]
|
||
|
||
|
||
def create_service_department(
|
||
name: str,
|
||
*,
|
||
code: Optional[str] = None,
|
||
phone: Optional[str] = None,
|
||
parent_id: Optional[str] = None,
|
||
region_id: Optional[str] = None,
|
||
description: Optional[str] = None,
|
||
grade: Optional[int] = None, # grade参数保留但不再使用，自动根据层级计算
|
||
unit_level: Optional[str] = None, # 新增：单位级别
|
||
operator: str = "admin",
|
||
) -> Dict[str, Any]:
|
||
normalized_name = _clean_text(name)
|
||
if not normalized_name:
|
||
raise ValueError("服务部门名称不能为空")
|
||
normalized_code = (_clean_text(code).upper() if code else "") or uuid.uuid4().hex[:8].upper()
|
||
parent_token = _clean_text(parent_id) or None
|
||
region_token = _clean_text(region_id) or None
|
||
description_text = description.strip() if isinstance(description, str) else description
|
||
unit_level_token = unit_level.strip().lower() if unit_level else None
|
||
|
||
# 自动根据父节点计算权限等级，不再使用传入的grade参数
|
||
grade_value = _calculate_grade_by_parent(parent_token)
|
||
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_service_department_schema(conn)
|
||
cur = conn.cursor()
|
||
dept_id = uuid.uuid4()
|
||
# 确定unit_level值：未指定时，根节点视为市级，其余视为末级
|
||
if unit_level_token:
|
||
final_unit_level = unit_level_token
|
||
else:
|
||
final_unit_level = 'municipal' if parent_token is None else 'unit'
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO service_departments (id, name, code, phone, parent_id, region_id, description, grade, unit_level)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||
""",
|
||
(dept_id, normalized_name, normalized_code, phone, parent_token, region_token, description_text, grade_value, final_unit_level),
|
||
)
|
||
conn.commit()
|
||
result = _fetch_service_department(cur, str(dept_id))
|
||
if not result:
|
||
raise RuntimeError("创建服务部门失败")
|
||
|
||
# Log the operation
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="CREATE",
|
||
target_type="DEPARTMENT",
|
||
target_id=str(result["id"]),
|
||
target_name=result["name"],
|
||
change_summary=f"Created department: {result['name']}",
|
||
details=result
|
||
)
|
||
|
||
return result
|
||
|
||
|
||
def _calculate_grade_by_parent(parent_id: Optional[str]) -> int:
|
||
"""
|
||
根据父节点计算权限等级
|
||
|
||
Args:
|
||
parent_id: 父节点ID，如果为None表示是根级节点
|
||
|
||
Returns:
|
||
int: 权限等级数值
|
||
"""
|
||
# 根级节点（无父节点）= 超级权限
|
||
if not parent_id:
|
||
return 90
|
||
|
||
# 获取父节点的层级信息
|
||
parent_level = _get_department_level(parent_id)
|
||
# 子节点的层级 = 父节点层级 + 1
|
||
child_level = parent_level + 1
|
||
|
||
# 根据层级映射权限等级
|
||
grade = 60 # 默认四级及以下
|
||
if child_level == 1:
|
||
grade = 80 # 二级部门 - 高级权限
|
||
elif child_level == 2:
|
||
grade = 70 # 三级部门 - 中级权限
|
||
|
||
return grade
|
||
|
||
|
||
def _get_department_level(department_id: str) -> int:
|
||
"""
|
||
获取部门在组织架构中的层级（根节点为0）
|
||
|
||
Args:
|
||
department_id: 部门ID
|
||
|
||
Returns:
|
||
int: 部门层级
|
||
"""
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
level = 0
|
||
current_id = department_id
|
||
|
||
while current_id:
|
||
cur.execute(
|
||
"SELECT parent_id FROM service_departments WHERE id = %s",
|
||
(current_id,)
|
||
)
|
||
result = cur.fetchone()
|
||
if result and result[0]:
|
||
level += 1
|
||
current_id = result[0]
|
||
else:
|
||
break
|
||
|
||
return level
|
||
|
||
|
||
def update_service_department(
|
||
department_id: str,
|
||
*,
|
||
name: Optional[str] = None,
|
||
phone: Optional[str] = None,
|
||
parent_id: Optional[str] = None,
|
||
region_id: Optional[str] = None,
|
||
description: Optional[str] = None,
|
||
grade: Optional[int] = None,
|
||
unit_level: Optional[str] = None, # 新增：单位级别
|
||
operator: str = "admin",
|
||
) -> Optional[Dict[str, Any]]:
|
||
dept_token = _clean_text(department_id)
|
||
if not dept_token:
|
||
raise ValueError("department_id 不能为空")
|
||
updates: List[str] = []
|
||
values: List[Any] = []
|
||
if name is not None:
|
||
normalized_name = _clean_text(name)
|
||
if not normalized_name:
|
||
raise ValueError("服务部门名称不能为空")
|
||
updates.append("name = %s")
|
||
values.append(normalized_name)
|
||
if phone is not None:
|
||
updates.append("phone = %s")
|
||
values.append(phone.strip() if isinstance(phone, str) else phone)
|
||
if parent_id is not None:
|
||
parent_token = _clean_text(parent_id) or None
|
||
if parent_token and parent_token == dept_token:
|
||
raise ValueError("服务部门不能设置为自己的上级")
|
||
updates.append("parent_id = %s")
|
||
values.append(parent_token)
|
||
if region_id is not None:
|
||
updates.append("region_id = %s")
|
||
values.append(_clean_text(region_id) or None)
|
||
if description is not None:
|
||
updates.append("description = %s")
|
||
values.append(description.strip() if isinstance(description, str) else description)
|
||
|
||
# 如果修改了parent_id，则自动计算grade（根据新层级计算）
|
||
# grade字段不再接受手动设置，完全基于层级自动计算
|
||
if parent_id is not None:
|
||
new_grade = _calculate_grade_by_parent(parent_id)
|
||
updates.append("grade = %s")
|
||
values.append(new_grade)
|
||
elif grade is not None:
|
||
# 如果手动设置grade（向后兼容），仍然允许但会记录警告
|
||
updates.append("grade = %s")
|
||
values.append(int(grade))
|
||
|
||
# 添加unit_level字段处理
|
||
if unit_level is not None:
|
||
unit_level_value = unit_level.strip().lower() if unit_level else None
|
||
if unit_level_value and unit_level_value not in ('admin', 'municipal', 'district', 'unit'):
|
||
raise ValueError(f"无效的unit_level值: {unit_level_value}")
|
||
updates.append("unit_level = %s")
|
||
values.append(unit_level_value)
|
||
|
||
if not updates:
|
||
return _get_service_department_by_id(dept_token)
|
||
|
||
updates.append("updated_at = now()")
|
||
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_service_department_schema(conn)
|
||
cur = conn.cursor()
|
||
query = f"UPDATE service_departments SET {', '.join(updates)} WHERE id = %s"
|
||
values.append(dept_token)
|
||
cur.execute(query, tuple(values))
|
||
conn.commit()
|
||
|
||
result = _get_service_department_by_id(dept_token)
|
||
# Log the operation
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="UPDATE",
|
||
target_type="DEPARTMENT",
|
||
target_id=dept_token,
|
||
target_name=result.get("name") if result else None,
|
||
change_summary=f"Updated department: {result.get('name')}" if result else f"Updated department {dept_token}",
|
||
details=result
|
||
)
|
||
return result
|
||
|
||
|
||
def _get_service_department_by_id(department_id: str) -> Optional[Dict[str, Any]]:
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
return _fetch_service_department(cur, department_id)
|
||
|
||
|
||
def delete_service_department(department_id: str, operator: str = "admin") -> Dict[str, Any]:
|
||
dept_token = _clean_text(department_id)
|
||
if not dept_token:
|
||
raise ValueError("department_id 不能为空")
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_service_department_schema(conn)
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"SELECT COUNT(*) FROM auth_users WHERE service_department_id = %s",
|
||
(dept_token,),
|
||
)
|
||
bound_count = int(cur.fetchone()[0] or 0)
|
||
if bound_count > 0:
|
||
raise ValueError("仍有账号绑定该服务部门，请先解除绑定后再删除")
|
||
cur.execute(
|
||
"UPDATE service_departments SET parent_id = NULL WHERE parent_id = %s",
|
||
(dept_token,),
|
||
)
|
||
cur.execute("SELECT name FROM service_departments WHERE id = %s", (dept_token,))
|
||
dept_name_row = cur.fetchone()
|
||
dept_name = dept_name_row[0] if dept_name_row else dept_token
|
||
|
||
cur.execute(
|
||
"DELETE FROM service_departments WHERE id = %s RETURNING id",
|
||
(dept_token,),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
conn.rollback()
|
||
return {"deleted": False}
|
||
conn.commit()
|
||
|
||
# Log the operation
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="DELETE",
|
||
target_type="DEPARTMENT",
|
||
target_id=dept_token,
|
||
target_name=dept_name,
|
||
change_summary=f"Deleted department: {dept_name}"
|
||
)
|
||
|
||
return {"deleted": True}
|
||
|
||
|
||
_THEME_SUMMARY_SELECT = """
|
||
SELECT
|
||
t.id,
|
||
t.name,
|
||
COUNT(DISTINCT rtp.permit_id) AS permit_count,
|
||
COUNT(DISTINCT rtp.region_id) AS region_count,
|
||
STRING_AGG(DISTINCT r.name, ',') AS region_names
|
||
FROM themes t
|
||
LEFT JOIN region_theme_permits rtp ON rtp.theme_id = t.id
|
||
LEFT JOIN regions r ON rtp.region_id = r.id
|
||
"""
|
||
|
||
|
||
def _serialize_theme_row(record: Dict[str, Any]) -> Dict[str, Any]:
|
||
region_names_str = record.get("region_names") or ""
|
||
levels = set()
|
||
r_names = [n.strip() for n in region_names_str.split(',') if n.strip()]
|
||
|
||
for r_name in r_names:
|
||
if r_name == "市级" or r_name == "佛山市":
|
||
levels.add("市级")
|
||
else:
|
||
# Assuming any other region is District level (e.g. 禅城区, 南海区)
|
||
levels.add("区级")
|
||
|
||
impl_level_list = []
|
||
if "市级" in levels:
|
||
impl_level_list.append("市级")
|
||
if "区级" in levels:
|
||
impl_level_list.append("区级")
|
||
|
||
impl_level = "、".join(impl_level_list) if impl_level_list else "-"
|
||
|
||
return {
|
||
"id": _to_optional_str(record.get("id")),
|
||
"name": record.get("name"),
|
||
"permit_count": int(record.get("permit_count") or 0),
|
||
"region_count": int(record.get("region_count") or 0),
|
||
"implementation_level": impl_level,
|
||
}
|
||
|
||
|
||
def _fetch_theme_summary(cur: pg.Cursor, theme_id: str) -> Optional[Dict[str, Any]]:
|
||
cur.execute(
|
||
_THEME_SUMMARY_SELECT + " WHERE t.id = %s GROUP BY t.id, t.name",
|
||
(theme_id,),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return None
|
||
columns = tuple(col[0] for col in cur.description)
|
||
record = {columns[idx]: row[idx] for idx in range(len(columns))}
|
||
return _serialize_theme_row(record)
|
||
|
||
|
||
def list_all_themes(
|
||
start_date: Optional[str] = None,
|
||
end_date: Optional[str] = None,
|
||
department_ids: Optional[List[str]] = None,
|
||
) -> List[Dict[str, Any]]:
|
||
|
||
# Base query components
|
||
select_fields = """
|
||
t.id,
|
||
t.name,
|
||
COUNT(DISTINCT CASE WHEN {filter_condition} THEN rtp.permit_id END) AS permit_count,
|
||
COUNT(DISTINCT CASE WHEN {filter_condition} THEN rtp.region_id END) AS region_count,
|
||
STRING_AGG(DISTINCT r.name, ',') AS region_names
|
||
"""
|
||
|
||
# Base joins
|
||
joins = [
|
||
"LEFT JOIN region_theme_permits rtp ON rtp.theme_id = t.id",
|
||
"LEFT JOIN regions r ON rtp.region_id = r.id"
|
||
]
|
||
|
||
conditions = []
|
||
params = []
|
||
|
||
# Conditionally add joins and conditions
|
||
if start_date or end_date:
|
||
joins.append("LEFT JOIN region_permit_details rpd ON rpd.permit_id = rtp.permit_id AND rpd.region_id = rtp.region_id")
|
||
|
||
if department_ids:
|
||
joins.append("LEFT JOIN permit_sources ps ON ps.permit_id = rtp.permit_id AND ps.region_id = rtp.region_id")
|
||
|
||
if start_date:
|
||
conditions.append("rpd.updated_at >= %s")
|
||
params.append(start_date)
|
||
|
||
if end_date:
|
||
conditions.append("rpd.updated_at <= %s")
|
||
params.append(end_date)
|
||
|
||
if department_ids:
|
||
expanded_ids = _expand_department_family(department_ids)
|
||
if expanded_ids:
|
||
placeholders = ','.join(['%s'] * len(expanded_ids))
|
||
conditions.append(f"(ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))")
|
||
params.extend(expanded_ids * 2)
|
||
|
||
filter_condition = " AND ".join(conditions) if conditions else "TRUE"
|
||
join_clause = "\n".join(joins)
|
||
|
||
sql = f"""
|
||
SELECT
|
||
{select_fields.format(filter_condition=filter_condition)}
|
||
FROM themes t
|
||
{join_clause}
|
||
GROUP BY t.id, t.name
|
||
ORDER BY t.name ASC
|
||
"""
|
||
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, params)
|
||
rows = cur.fetchall()
|
||
columns = tuple(col[0] for col in cur.description)
|
||
|
||
items: List[Dict[str, Any]] = []
|
||
for row in rows:
|
||
record = {columns[idx]: row[idx] for idx in range(len(columns))}
|
||
items.append(_serialize_theme_row(record))
|
||
return items
|
||
|
||
|
||
def list_unbound_permits(
|
||
visibility: Optional[str] = None,
|
||
search_text: Optional[str] = None,
|
||
department_ids: Optional[List[str]] = None,
|
||
region_id: Optional[str] = None,
|
||
) -> List[Dict[str, Any]]:
|
||
"""Return all permits that are in a region but not bound to any theme in that region.
|
||
|
||
Args:
|
||
visibility: 'visible', 'hidden', or None (all)
|
||
search_text: Filter by permit name (keyword)
|
||
department_ids: Filter by uploader/bound department
|
||
region_id: Filter by region
|
||
"""
|
||
filters = ["rtp.theme_id IS NULL"]
|
||
params = []
|
||
|
||
if visibility == 'visible':
|
||
filters.append("(rpd.is_v2_visible IS TRUE OR rpd.is_v2_visible IS NULL)")
|
||
elif visibility == 'hidden':
|
||
filters.append("rpd.is_v2_visible IS FALSE")
|
||
|
||
if search_text:
|
||
filters.append("p.name ILIKE %s")
|
||
params.append(f"%{search_text}%")
|
||
|
||
if department_ids:
|
||
expanded_ids = _expand_department_family(department_ids)
|
||
if expanded_ids:
|
||
placeholders = ','.join(['%s'] * len(expanded_ids))
|
||
filters.append(f"(ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))")
|
||
params.extend(expanded_ids * 2)
|
||
|
||
if region_id:
|
||
filters.append("rpd.region_id = %s")
|
||
params.append(region_id)
|
||
|
||
ps_join = ""
|
||
if department_ids:
|
||
ps_join = """
|
||
LEFT JOIN permit_sources ps
|
||
ON ps.permit_id = rpd.permit_id
|
||
AND ps.region_id = rpd.region_id
|
||
"""
|
||
|
||
where_clause = " AND ".join(filters)
|
||
|
||
sql = f"""
|
||
SELECT
|
||
r.id AS region_id,
|
||
r.name AS region_name,
|
||
p.id AS permit_id,
|
||
p.name AS permit_name,
|
||
rpd.unit_name,
|
||
rpd.updated_at,
|
||
COALESCE(rpd.is_v2_visible, true) AS is_v2_visible
|
||
FROM region_permit_details rpd
|
||
JOIN regions r ON r.id = rpd.region_id
|
||
JOIN permits p ON p.id = rpd.permit_id
|
||
LEFT JOIN region_theme_permits rtp
|
||
ON rtp.region_id = rpd.region_id
|
||
AND rtp.permit_id = rpd.permit_id
|
||
{ps_join}
|
||
WHERE {where_clause}
|
||
ORDER BY r.name, p.name
|
||
"""
|
||
items: List[Dict[str, Any]] = []
|
||
try:
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, params)
|
||
rows = cur.fetchall()
|
||
columns = tuple(col[0] for col in cur.description)
|
||
for row in rows:
|
||
record = {columns[idx]: row[idx] for idx in range(len(columns))}
|
||
# Serialize UUIDs and timestamps
|
||
record["region_id"] = str(record["region_id"])
|
||
record["permit_id"] = str(record["permit_id"])
|
||
if record["updated_at"]:
|
||
record["updated_at"] = record["updated_at"].isoformat()
|
||
items.append(record)
|
||
except Exception as e:
|
||
logger.error(f"Error listing unbound permits: {e}")
|
||
return items
|
||
|
||
|
||
def create_theme(name: str, operator: str = "admin") -> Dict[str, Any]:
|
||
normalized = _clean_text(name)
|
||
if not normalized:
|
||
raise ValueError("主题名称不能为空")
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO themes (name)
|
||
VALUES (%s)
|
||
ON CONFLICT (name) DO NOTHING
|
||
RETURNING id
|
||
""",
|
||
(normalized,),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
cur.execute("SELECT id FROM themes WHERE name = %s LIMIT 1", (normalized,))
|
||
row = cur.fetchone()
|
||
conn.commit()
|
||
if not row:
|
||
raise RuntimeError("创建主题失败")
|
||
theme_id = str(row[0])
|
||
summary = _fetch_theme_summary(cur, theme_id)
|
||
if not summary:
|
||
raise RuntimeError("无法加载主题信息")
|
||
|
||
# Log the operation
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="CREATE",
|
||
target_type="THEME",
|
||
target_id=str(summary["id"]),
|
||
target_name=summary["name"],
|
||
change_summary=f"Created theme: {summary['name']}"
|
||
)
|
||
|
||
return summary
|
||
|
||
|
||
def rename_theme(theme_id: str, new_name: str, operator: str = "admin") -> Dict[str, Any]:
|
||
theme_token = _clean_text(theme_id)
|
||
if not theme_token:
|
||
raise ValueError("theme_id 不能为空")
|
||
normalized = _clean_text(new_name)
|
||
if not normalized:
|
||
raise ValueError("主题名称不能为空")
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"UPDATE themes SET name = %s WHERE id = %s RETURNING id",
|
||
(normalized, theme_token),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
conn.rollback()
|
||
raise ValueError("主题不存在或已被删除")
|
||
conn.commit()
|
||
summary = _fetch_theme_summary(cur, theme_token)
|
||
if not summary:
|
||
raise RuntimeError("无法刷新主题信息")
|
||
|
||
# Log the operation
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="UPDATE",
|
||
target_type="THEME",
|
||
target_id=theme_token,
|
||
target_name=normalized,
|
||
change_summary=f"Renamed theme to: {normalized}"
|
||
)
|
||
|
||
return summary
|
||
|
||
|
||
def delete_theme(theme_id: str, operator: str = "admin") -> Dict[str, Any]:
|
||
theme_token = _clean_text(theme_id)
|
||
if not theme_token:
|
||
raise ValueError("theme_id 不能为空")
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute("SELECT name FROM themes WHERE id = %s", (theme_token,))
|
||
theme_name_row = cur.fetchone()
|
||
theme_name = theme_name_row[0] if theme_name_row else theme_token
|
||
|
||
cur.execute(
|
||
"DELETE FROM region_theme_permits WHERE theme_id = %s",
|
||
(theme_token,),
|
||
)
|
||
rtp_deleted = int(cur.rowcount or 0)
|
||
cur.execute(
|
||
"DELETE FROM region_themes WHERE theme_id = %s",
|
||
(theme_token,),
|
||
)
|
||
rt_deleted = int(cur.rowcount or 0)
|
||
cur.execute(
|
||
"DELETE FROM themes WHERE id = %s RETURNING id",
|
||
(theme_token,),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
conn.rollback()
|
||
return {"deleted": False, "message": "主题不存在"}
|
||
conn.commit()
|
||
|
||
# Log the operation
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="DELETE",
|
||
target_type="THEME",
|
||
target_id=theme_token,
|
||
target_name=theme_name,
|
||
change_summary=f"Deleted theme: {theme_name}"
|
||
)
|
||
|
||
return {
|
||
"deleted": True,
|
||
"region_theme_permits_deleted": rtp_deleted,
|
||
"region_themes_deleted": rt_deleted,
|
||
}
|
||
|
||
|
||
def _create_permit_file_schema(conn: pg.Connection) -> None:
|
||
"""Create permit file storage tables on demand."""
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS permit_files (
|
||
id uuid PRIMARY KEY,
|
||
filename text NOT NULL,
|
||
content_type text,
|
||
file_size integer NOT NULL,
|
||
file_data bytea NOT NULL,
|
||
checksum text,
|
||
uploaded_by text,
|
||
created_at timestamptz NOT NULL DEFAULT now()
|
||
)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS permit_file_links (
|
||
id uuid PRIMARY KEY,
|
||
region_id uuid NOT NULL REFERENCES regions(id) ON DELETE CASCADE,
|
||
permit_id uuid NOT NULL REFERENCES permits(id) ON DELETE CASCADE,
|
||
file_id uuid NOT NULL REFERENCES permit_files(id) ON DELETE CASCADE,
|
||
created_by text,
|
||
created_at timestamptz NOT NULL DEFAULT now()
|
||
)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE UNIQUE INDEX IF NOT EXISTS permit_file_links_region_permit_idx
|
||
ON permit_file_links (region_id, permit_id)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE INDEX IF NOT EXISTS permit_file_links_permit_idx
|
||
ON permit_file_links (permit_id)
|
||
"""
|
||
)
|
||
|
||
|
||
def _ensure_permit_file_schema(conn: Optional[pg.Connection] = None) -> None:
|
||
"""Ensure permit file tables exist (lazy creation, thread safe)."""
|
||
global _PERMIT_FILE_SCHEMA_READY
|
||
if _PERMIT_FILE_SCHEMA_READY:
|
||
return
|
||
|
||
with _PERMIT_FILE_SCHEMA_LOCK:
|
||
if _PERMIT_FILE_SCHEMA_READY:
|
||
return
|
||
if conn is not None:
|
||
original_autocommit = conn.autocommit
|
||
try:
|
||
conn.autocommit = True
|
||
_create_permit_file_schema(conn)
|
||
finally:
|
||
conn.autocommit = original_autocommit
|
||
else:
|
||
with _lic_pg_conn(autocommit=True) as ensure_conn:
|
||
_create_permit_file_schema(ensure_conn)
|
||
_PERMIT_FILE_SCHEMA_READY = True
|
||
|
||
|
||
def _create_permit_theme_override_schema(conn: pg.Connection) -> None:
|
||
"""Create region-level theme override table for all-theme bindings."""
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS region_permit_theme_overrides (
|
||
region_id uuid NOT NULL REFERENCES regions(id) ON DELETE CASCADE,
|
||
permit_id uuid NOT NULL REFERENCES permits(id) ON DELETE CASCADE,
|
||
binds_all_themes boolean NOT NULL DEFAULT false,
|
||
created_at timestamptz NOT NULL DEFAULT now(),
|
||
created_by text,
|
||
PRIMARY KEY (region_id, permit_id)
|
||
)
|
||
"""
|
||
)
|
||
|
||
|
||
def _ensure_permit_theme_override_schema(conn: Optional[pg.Connection] = None) -> None:
|
||
"""Ensure the override table exists; it is required for 'all theme' bindings."""
|
||
global _PERMIT_THEME_OVERRIDE_SCHEMA_READY
|
||
if _PERMIT_THEME_OVERRIDE_SCHEMA_READY:
|
||
return
|
||
|
||
with _PERMIT_THEME_OVERRIDE_SCHEMA_LOCK:
|
||
if _PERMIT_THEME_OVERRIDE_SCHEMA_READY:
|
||
return
|
||
if conn is not None:
|
||
original_autocommit = conn.autocommit
|
||
try:
|
||
conn.autocommit = True
|
||
_create_permit_theme_override_schema(conn)
|
||
finally:
|
||
conn.autocommit = original_autocommit
|
||
else:
|
||
with _lic_pg_conn(autocommit=True) as ensure_conn:
|
||
_create_permit_theme_override_schema(ensure_conn)
|
||
_PERMIT_THEME_OVERRIDE_SCHEMA_READY = True
|
||
|
||
|
||
def _create_permit_approval_departments_schema(conn: pg.Connection) -> None:
|
||
"""Create table for mapping permits to approval departments."""
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS permit_approval_departments (
|
||
id uuid PRIMARY KEY,
|
||
permit_name text NOT NULL,
|
||
department_name text NOT NULL,
|
||
created_at timestamptz NOT NULL DEFAULT now(),
|
||
updated_at timestamptz NOT NULL DEFAULT now()
|
||
)
|
||
"""
|
||
)
|
||
cur.execute(
|
||
"""
|
||
CREATE UNIQUE INDEX IF NOT EXISTS permit_approval_departments_name_idx
|
||
ON permit_approval_departments (permit_name)
|
||
"""
|
||
)
|
||
|
||
|
||
def _ensure_permit_approval_departments_schema(conn: Optional[pg.Connection] = None) -> None:
|
||
"""Ensure approval departments table exists."""
|
||
global _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY
|
||
if _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY:
|
||
return
|
||
|
||
with _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_LOCK:
|
||
if _PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY:
|
||
return
|
||
if conn is not None:
|
||
original_autocommit = conn.autocommit
|
||
try:
|
||
conn.autocommit = True
|
||
_create_permit_approval_departments_schema(conn)
|
||
finally:
|
||
conn.autocommit = original_autocommit
|
||
else:
|
||
with _lic_pg_conn(autocommit=True) as ensure_conn:
|
||
_create_permit_approval_departments_schema(ensure_conn)
|
||
_PERMIT_APPROVAL_DEPARTMENTS_SCHEMA_READY = True
|
||
|
||
|
||
def _create_operation_log_schema(conn: pg.Connection) -> None:
|
||
"""Create a table for auditing user operations."""
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS operation_logs (
|
||
id uuid PRIMARY KEY,
|
||
operator text NOT NULL,
|
||
operation_type text NOT NULL,
|
||
target_type text,
|
||
target_id text,
|
||
target_name text,
|
||
change_summary text,
|
||
details jsonb,
|
||
created_at timestamptz NOT NULL DEFAULT now()
|
||
)
|
||
"""
|
||
)
|
||
cur.execute("CREATE INDEX IF NOT EXISTS operation_logs_operator_idx ON operation_logs (operator)")
|
||
cur.execute("CREATE INDEX IF NOT EXISTS operation_logs_created_at_idx ON operation_logs (created_at)")
|
||
|
||
|
||
def _ensure_operation_log_schema(conn: Optional[pg.Connection] = None) -> None:
|
||
"""Ensure the operation_logs table exists."""
|
||
global _OPERATION_LOG_SCHEMA_READY
|
||
if _OPERATION_LOG_SCHEMA_READY:
|
||
return
|
||
|
||
with _OPERATION_LOG_SCHEMA_LOCK:
|
||
if _OPERATION_LOG_SCHEMA_READY:
|
||
return
|
||
if conn is not None:
|
||
_create_operation_log_schema(conn)
|
||
else:
|
||
with _lic_pg_conn(autocommit=True) as ensure_conn:
|
||
_create_operation_log_schema(ensure_conn)
|
||
_OPERATION_LOG_SCHEMA_READY = True
|
||
|
||
|
||
def log_operation(
|
||
operator: str,
|
||
operation_type: str,
|
||
*,
|
||
target_type: Optional[str] = None,
|
||
target_id: Optional[str] = None,
|
||
target_name: Optional[str] = None,
|
||
change_summary: Optional[str] = None,
|
||
details: Optional[Dict[str, Any]] = None
|
||
) -> str:
|
||
"""Persist an audit log entry for a user operation."""
|
||
_ensure_operation_log_schema()
|
||
|
||
log_id = uuid.uuid4()
|
||
with _lic_pg_conn(autocommit=True) as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO operation_logs (
|
||
id, operator, operation_type, target_type, target_id,
|
||
target_name, change_summary, details
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||
""",
|
||
(
|
||
str(log_id),
|
||
operator,
|
||
operation_type,
|
||
target_type,
|
||
target_id,
|
||
target_name,
|
||
change_summary,
|
||
json.dumps(details, ensure_ascii=False) if details else None
|
||
)
|
||
)
|
||
return str(log_id)
|
||
|
||
|
||
def list_operation_logs(
|
||
*,
|
||
operator: Optional[str] = None,
|
||
operation_type: Optional[str] = None,
|
||
limit: int = 50,
|
||
offset: int = 0
|
||
) -> List[Dict[str, Any]]:
|
||
"""Return recent operation logs with optional filters."""
|
||
_ensure_operation_log_schema()
|
||
|
||
filters = []
|
||
params = []
|
||
|
||
if operator:
|
||
filters.append("operator = %s")
|
||
params.append(operator)
|
||
if operation_type:
|
||
filters.append("operation_type = %s")
|
||
params.append(operation_type)
|
||
|
||
where_clause = f"WHERE {' AND '.join(filters)}" if filters else ""
|
||
|
||
sql = f"""
|
||
SELECT id, operator, operation_type, target_type, target_id,
|
||
target_name, change_summary, details, created_at
|
||
FROM operation_logs
|
||
{where_clause}
|
||
ORDER BY created_at DESC
|
||
LIMIT %s OFFSET %s
|
||
"""
|
||
params.extend([limit, offset])
|
||
|
||
logs = []
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, tuple(params))
|
||
for row in cur.fetchall():
|
||
l_id, op, o_type, t_type, t_id, t_name, summary, details_raw, created_at = row
|
||
|
||
# Details handle
|
||
details = None
|
||
if details_raw:
|
||
if isinstance(details_raw, dict):
|
||
details = details_raw
|
||
else:
|
||
try:
|
||
details = json.loads(details_raw)
|
||
except:
|
||
details = str(details_raw)
|
||
|
||
logs.append({
|
||
"id": str(l_id),
|
||
"operator": op,
|
||
"operation_type": o_type,
|
||
"target_type": t_type,
|
||
"target_id": t_id,
|
||
"target_name": t_name,
|
||
"change_summary": summary,
|
||
"details": details,
|
||
"created_at": _convert_snapshot_value(created_at)
|
||
})
|
||
return logs
|
||
|
||
|
||
def _insert_permit_file_record(
|
||
conn: pg.Connection,
|
||
*,
|
||
file_bytes: bytes,
|
||
filename: str,
|
||
content_type: Optional[str],
|
||
uploaded_by: Optional[str],
|
||
) -> Dict[str, Any]:
|
||
"""Persist an uploaded file and return its metadata."""
|
||
normalized_name = filename or "许可导入.xlsx"
|
||
content_type = content_type or "application/octet-stream"
|
||
file_id = uuid.uuid4()
|
||
checksum = hashlib.sha256(file_bytes).hexdigest()
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO permit_files (
|
||
id,
|
||
filename,
|
||
content_type,
|
||
file_size,
|
||
file_data,
|
||
checksum,
|
||
uploaded_by
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||
""",
|
||
(
|
||
file_id,
|
||
normalized_name,
|
||
content_type,
|
||
len(file_bytes),
|
||
file_bytes,
|
||
checksum,
|
||
uploaded_by,
|
||
),
|
||
)
|
||
return {
|
||
"file_id": str(file_id),
|
||
"filename": normalized_name,
|
||
"content_type": content_type,
|
||
"file_size": len(file_bytes),
|
||
"checksum": checksum,
|
||
"uploaded_by": uploaded_by,
|
||
}
|
||
|
||
|
||
def _link_file_to_permit(
|
||
conn: pg.Connection,
|
||
*,
|
||
file_id: str,
|
||
region_id: str,
|
||
permit_id: str,
|
||
created_by: Optional[str],
|
||
) -> None:
|
||
"""Associate a stored file with a region-permit pair."""
|
||
if not (file_id and region_id and permit_id):
|
||
return
|
||
rid = str(region_id)
|
||
pid = str(permit_id)
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO permit_file_links (id, region_id, permit_id, file_id, created_by)
|
||
VALUES (%s, %s, %s, %s, %s)
|
||
ON CONFLICT (region_id, permit_id)
|
||
DO UPDATE SET
|
||
file_id = EXCLUDED.file_id,
|
||
created_by = EXCLUDED.created_by,
|
||
created_at = now()
|
||
""",
|
||
(uuid.uuid4(), rid, pid, file_id, created_by),
|
||
)
|
||
|
||
|
||
def _set_permit_bind_all_themes(
|
||
conn: pg.Connection,
|
||
region_id: str,
|
||
permit_id: str,
|
||
*,
|
||
created_by: Optional[str] = None,
|
||
) -> None:
|
||
"""Mark a permit so it always links to every theme under the region."""
|
||
if not (region_id and permit_id):
|
||
return
|
||
_ensure_permit_theme_override_schema(conn)
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_theme_overrides (region_id, permit_id, binds_all_themes, created_by)
|
||
VALUES (%s, %s, TRUE, %s)
|
||
ON CONFLICT (region_id, permit_id)
|
||
DO UPDATE SET
|
||
binds_all_themes = TRUE,
|
||
created_by = COALESCE(EXCLUDED.created_by, region_permit_theme_overrides.created_by),
|
||
created_at = CASE
|
||
WHEN region_permit_theme_overrides.binds_all_themes IS DISTINCT FROM EXCLUDED.binds_all_themes
|
||
THEN now()
|
||
ELSE region_permit_theme_overrides.created_at
|
||
END
|
||
""",
|
||
(region_id, permit_id, created_by),
|
||
)
|
||
global _PERMIT_THEME_OVERRIDE_SCHEMA_READY
|
||
_PERMIT_THEME_OVERRIDE_SCHEMA_READY = True
|
||
|
||
|
||
def _clear_permit_bind_all_themes(conn: pg.Connection, region_id: str, permit_id: str) -> None:
|
||
"""Remove the 'all themes' override for a permit."""
|
||
if not (region_id and permit_id):
|
||
return
|
||
if not _PERMIT_THEME_OVERRIDE_SCHEMA_READY:
|
||
try:
|
||
_ensure_permit_theme_override_schema()
|
||
except Exception:
|
||
return
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM region_permit_theme_overrides
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
|
||
|
||
def _fetch_permit_all_theme_flags(
|
||
conn: pg.Connection, region_id: str, permit_ids: Iterable[str]
|
||
) -> Dict[str, bool]:
|
||
"""Return permit_id -> True when the permit is flagged for all themes."""
|
||
global _PERMIT_THEME_OVERRIDE_SCHEMA_READY
|
||
permit_list = [str(pid) for pid in permit_ids if pid]
|
||
if not permit_list:
|
||
return {}
|
||
if _PERMIT_THEME_OVERRIDE_SCHEMA_READY is False:
|
||
return {}
|
||
cur = conn.cursor()
|
||
try:
|
||
cur.execute(
|
||
"""
|
||
SELECT permit_id
|
||
FROM region_permit_theme_overrides
|
||
WHERE region_id = %s
|
||
AND permit_id = ANY(%s)
|
||
AND binds_all_themes
|
||
""",
|
||
(region_id, permit_list),
|
||
)
|
||
except pg.DatabaseError as exc: # type: ignore[attr-defined]
|
||
sqlstate = getattr(exc, "sqlstate", "")
|
||
if sqlstate == "42P01":
|
||
try:
|
||
conn.rollback()
|
||
except Exception:
|
||
pass
|
||
_PERMIT_THEME_OVERRIDE_SCHEMA_READY = False
|
||
return {}
|
||
raise
|
||
|
||
rows = cur.fetchall()
|
||
_PERMIT_THEME_OVERRIDE_SCHEMA_READY = True
|
||
return {str(permit_id): True for (permit_id,) in rows}
|
||
|
||
|
||
def update_permit_v2_visibility(
|
||
region_id: str, permit_id: str, is_visible: bool, operator: str = "admin"
|
||
) -> bool:
|
||
"""Toggle the visibility of a permit in V2 API retrieval for a specific region."""
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_v2_visibility_column(conn)
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
UPDATE region_permit_details
|
||
SET is_v2_visible = %s, updated_at = now()
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(is_visible, region_id, permit_id),
|
||
)
|
||
success = cur.rowcount > 0
|
||
if success:
|
||
conn.commit()
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="UPDATE",
|
||
target_type="PERMIT_VISIBILITY",
|
||
target_id=permit_id,
|
||
target_name=f"Visibility set to {is_visible}",
|
||
change_summary=f"Updated v2_visibility for permit {permit_id} in region {region_id} to {is_visible}",
|
||
details={"region_id": region_id, "permit_id": permit_id, "is_v2_visible": is_visible},
|
||
)
|
||
return success
|
||
|
||
|
||
def _permit_binds_all_themes(conn: pg.Connection, region_id: str, permit_id: str) -> bool:
|
||
"""Check override flag for a single region-permit pair."""
|
||
global _PERMIT_THEME_OVERRIDE_SCHEMA_READY
|
||
if not (region_id and permit_id):
|
||
return False
|
||
if _PERMIT_THEME_OVERRIDE_SCHEMA_READY is False:
|
||
return False
|
||
cur = conn.cursor()
|
||
try:
|
||
cur.execute(
|
||
"""
|
||
SELECT binds_all_themes
|
||
FROM region_permit_theme_overrides
|
||
WHERE region_id = %s AND permit_id = %s
|
||
LIMIT 1
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
except pg.DatabaseError as exc: # type: ignore[attr-defined]
|
||
sqlstate = getattr(exc, "sqlstate", "")
|
||
if sqlstate == "42P01":
|
||
try:
|
||
conn.rollback()
|
||
except Exception:
|
||
pass
|
||
_PERMIT_THEME_OVERRIDE_SCHEMA_READY = False
|
||
return False
|
||
raise
|
||
row = cur.fetchone()
|
||
if row is None:
|
||
return False
|
||
_PERMIT_THEME_OVERRIDE_SCHEMA_READY = True
|
||
return bool(row[0])
|
||
|
||
|
||
def _propagate_all_theme_bindings(conn: pg.Connection, region_id: str, theme_id: str) -> None:
|
||
"""Ensure all-theme permits also link to a newly attached theme."""
|
||
if not (region_id and theme_id):
|
||
return
|
||
if _PERMIT_THEME_OVERRIDE_SCHEMA_READY is False:
|
||
return
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_theme_permits (region_id, theme_id, permit_id)
|
||
SELECT %s, %s, rpto.permit_id
|
||
FROM region_permit_theme_overrides rpto
|
||
WHERE rpto.region_id = %s
|
||
AND rpto.binds_all_themes
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, theme_id, region_id),
|
||
)
|
||
|
||
|
||
def _load_permit_file_metadata(
|
||
conn: pg.Connection,
|
||
region_id: str,
|
||
permit_ids: Iterable[str],
|
||
) -> Dict[str, Dict[str, Any]]:
|
||
"""Load file metadata for a batch of permits."""
|
||
ids = [str(pid) for pid in permit_ids if pid]
|
||
if not ids:
|
||
return {}
|
||
|
||
_ensure_permit_file_schema(conn)
|
||
rows = _select_permit_files(conn, region_id, ids)
|
||
out: Dict[str, Dict[str, Any]] = {}
|
||
for row in rows:
|
||
(
|
||
permit_id,
|
||
file_id,
|
||
filename,
|
||
content_type,
|
||
file_size,
|
||
created_at,
|
||
uploaded_by,
|
||
) = row
|
||
out[str(permit_id)] = {
|
||
"file_id": str(file_id),
|
||
"filename": filename or "",
|
||
"content_type": content_type or "",
|
||
"file_size": int(file_size or 0),
|
||
"created_at": created_at.isoformat() if created_at else None,
|
||
"uploaded_by": uploaded_by or "",
|
||
}
|
||
return out
|
||
|
||
|
||
def _select_permit_files(conn: pg.Connection, region_id: str, permit_ids: Iterable[str]):
|
||
"""Execute the permit file metadata query, recreating tables if missing."""
|
||
sql = """
|
||
SELECT
|
||
pfl.permit_id,
|
||
pf.id,
|
||
pf.filename,
|
||
pf.content_type,
|
||
pf.file_size,
|
||
pf.created_at,
|
||
pf.uploaded_by
|
||
FROM permit_file_links pfl
|
||
JOIN permit_files pf ON pf.id = pfl.file_id
|
||
WHERE pfl.region_id = %s
|
||
AND pfl.permit_id = ANY(%s)
|
||
"""
|
||
attempts = 0
|
||
while attempts < 2:
|
||
try:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (region_id, permit_ids))
|
||
return cur.fetchall()
|
||
except pg.DatabaseError as exc: # type: ignore[attr-defined]
|
||
sqlstate = getattr(exc, "sqlstate", "")
|
||
if sqlstate != "42P01":
|
||
raise
|
||
attempts += 1
|
||
try:
|
||
conn.rollback()
|
||
except Exception:
|
||
pass
|
||
global _PERMIT_FILE_SCHEMA_READY
|
||
_PERMIT_FILE_SCHEMA_READY = None
|
||
_ensure_permit_file_schema(conn)
|
||
if attempts >= 2:
|
||
logger.warning("[PERMIT-FILES] permit_file_links table missing after recreate attempt, skipping metadata fetch")
|
||
return []
|
||
return []
|
||
|
||
|
||
def _lic_pg_conn(autocommit: bool = False) -> pg.Connection:
|
||
host = os.getenv("LIC_PG_HOST", "172.24.240.1")
|
||
port = int(os.getenv("LIC_PG_PORT", os.getenv("PG_PORT", "5432")))
|
||
user = os.getenv("LIC_PG_USER", os.getenv("PG_USER", "postgres"))
|
||
password = os.getenv("LIC_PG_PASSWORD", "")
|
||
database = os.getenv("LIC_PG_DATABASE", LIC_DEFAULT_DB)
|
||
conn = pg.connect(host=host, port=port, user=user, password=password, database=database)
|
||
conn.autocommit = autocommit
|
||
return conn
|
||
|
||
|
||
def list_region_theme_options() -> List[Dict[str, str]]:
|
||
"""Return all region-theme pairs usable for LLM selection."""
|
||
sql = """
|
||
SELECT
|
||
rt.region_id,
|
||
r.name AS region_name,
|
||
rt.theme_id,
|
||
t.name AS theme_name
|
||
FROM region_themes rt
|
||
JOIN regions r ON r.id = rt.region_id
|
||
JOIN themes t ON t.id = rt.theme_id
|
||
WHERE EXISTS (
|
||
SELECT 1 FROM region_theme_permits rtp
|
||
JOIN region_permit_details rpd ON rpd.region_id = rtp.region_id AND rpd.permit_id = rtp.permit_id
|
||
WHERE rtp.region_id = rt.region_id AND rtp.theme_id = rt.theme_id
|
||
AND COALESCE(rpd.is_v2_visible, true) = true
|
||
)
|
||
ORDER BY r.name, t.name
|
||
"""
|
||
out: List[Dict[str, str]] = []
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_v2_visibility_column(conn)
|
||
cur = conn.cursor()
|
||
cur.execute(sql)
|
||
for region_id, region_name, theme_id, theme_name in cur.fetchall():
|
||
rid = str(region_id)
|
||
tid = str(theme_id)
|
||
out.append(
|
||
{
|
||
"option_id": f"{rid}:{tid}",
|
||
"region_id": rid,
|
||
"region_name": str(region_name),
|
||
"theme_id": tid,
|
||
"theme_name": str(theme_name),
|
||
"display_name": f"{region_name} · {theme_name}",
|
||
}
|
||
)
|
||
return out
|
||
|
||
|
||
def load_business_scopes(region_id: str) -> List[Dict[str, str]]:
|
||
"""List business scopes bound to a region."""
|
||
sql = """
|
||
SELECT bs.id, bs.description
|
||
FROM region_scopes rs
|
||
JOIN business_scopes bs ON bs.id = rs.scope_id
|
||
WHERE rs.region_id = %s
|
||
ORDER BY bs.description
|
||
"""
|
||
scopes: List[Dict[str, str]] = []
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (region_id,))
|
||
for scope_id, description in cur.fetchall():
|
||
scopes.append({"id": str(scope_id), "description": str(description)})
|
||
return scopes
|
||
|
||
|
||
def get_visible_permits(
|
||
current_user: Optional[Dict[str, Any]] = None,
|
||
filters: Optional[Dict[str, Any]] = None
|
||
) -> List[Dict[str, str]]:
|
||
"""根据部门绑定关系返回可见的许可列表（仅依赖部门树，不再按区域授予权限）。
|
||
|
||
Args:
|
||
current_user: 当前认证用户
|
||
filters: 筛选条件（municipal_dept_id, district_dept_id, region, search_text）
|
||
|
||
Returns:
|
||
根据用户权限可见的许可列表
|
||
"""
|
||
filters = filters or {}
|
||
username = current_user.get("username", "anonymous") if current_user else "anonymous"
|
||
user_department = current_user.get("department", {}) if current_user else {}
|
||
user_dept_id = user_department.get("id")
|
||
|
||
if not current_user or not user_dept_id:
|
||
logger.warning("Permission denied: User %s has no department binding", username)
|
||
return []
|
||
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_service_department_schema(conn)
|
||
_ensure_permit_sources_table(conn)
|
||
cur = conn.cursor()
|
||
|
||
# 取用户部门及其下级列表
|
||
cur.execute(
|
||
"""
|
||
SELECT id, region_id
|
||
FROM service_departments
|
||
WHERE id = %s
|
||
""",
|
||
(user_dept_id,),
|
||
)
|
||
dept_row = cur.fetchone()
|
||
if not dept_row:
|
||
logger.warning("Permission denied: User %s department not found", username)
|
||
return []
|
||
|
||
dept_id_value, dept_region_id = dept_row
|
||
accessible_dept_ids = _fetch_department_descendants(cur, str(dept_id_value))
|
||
if not accessible_dept_ids:
|
||
logger.warning("Permission denied: User %s has no accessible departments", username)
|
||
return []
|
||
|
||
# 解析筛选部门（用部门树收缩范围）
|
||
def _resolve_target_departments(dept_token: Optional[str]) -> List[str]:
|
||
if not dept_token:
|
||
return []
|
||
try:
|
||
target_uuid = str(dept_token).strip()
|
||
except Exception:
|
||
return []
|
||
return _fetch_department_descendants(cur, target_uuid)
|
||
|
||
filter_dept_sets: List[List[str]] = []
|
||
if filters.get("municipal_dept_id"):
|
||
filter_dept_sets.append(_resolve_target_departments(filters.get("municipal_dept_id")))
|
||
if filters.get("district_dept_id"):
|
||
filter_dept_sets.append(_resolve_target_departments(filters.get("district_dept_id")))
|
||
|
||
sql = """
|
||
SELECT DISTINCT p.id, p.name
|
||
FROM permit_sources ps
|
||
JOIN permits p ON p.id = ps.permit_id
|
||
LEFT JOIN regions r ON r.id = ps.region_id
|
||
"""
|
||
where_clauses: List[str] = []
|
||
params: List[Any] = []
|
||
|
||
# 用户可见范围：自身及下级部门上传/绑定的许可
|
||
where_clauses.append(
|
||
"(ps.uploader_department_id = ANY(%s) OR ps.bound_department_id = ANY(%s))"
|
||
)
|
||
params.extend([accessible_dept_ids, accessible_dept_ids])
|
||
|
||
# 额外的部门筛选（前端传入）
|
||
for target_set in filter_dept_sets:
|
||
if not target_set:
|
||
continue
|
||
where_clauses.append(
|
||
"(ps.uploader_department_id = ANY(%s) OR ps.bound_department_id = ANY(%s))"
|
||
)
|
||
params.extend([target_set, target_set])
|
||
|
||
if filters.get("region"):
|
||
where_clauses.append(
|
||
"(ps.region_id::text = %s OR LOWER(r.name) = LOWER(%s))"
|
||
)
|
||
region = filters["region"]
|
||
params.extend([region, region])
|
||
|
||
if filters.get("search_text"):
|
||
where_clauses.append("LOWER(p.name) LIKE LOWER(%s)")
|
||
params.append(f"%{filters['search_text']}%")
|
||
|
||
if where_clauses:
|
||
sql += " WHERE " + " AND ".join(where_clauses)
|
||
|
||
sql += " ORDER BY p.name"
|
||
|
||
permits: List[Dict[str, str]] = []
|
||
cur.execute(sql, params)
|
||
for permit_id, permit_name in cur.fetchall():
|
||
permits.append({"id": str(permit_id), "name": str(permit_name)})
|
||
|
||
logger.info("User %s can view %d permits after filtering", username, len(permits))
|
||
return permits
|
||
|
||
|
||
def list_permits_for_region(region: str, current_user: Optional[Dict[str, Any]] = None) -> List[Dict[str, str]]:
|
||
"""Return all permits available within a region (accepts id or name).
|
||
|
||
Args:
|
||
region: Region ID or name
|
||
current_user: Current authenticated user (optional). If provided,
|
||
will filter permits based on user's department tree
|
||
(region仅作为附加过滤，不授予权限)
|
||
|
||
Returns:
|
||
List of permits visible to the user based on their department tree
|
||
"""
|
||
# 使用新的权限模型
|
||
filters = {"region": region} if region else {}
|
||
return get_visible_permits(current_user, filters)
|
||
|
||
|
||
def list_region_permit_catalog(region_id: str) -> List[Dict[str, Any]]:
|
||
"""Return permit entries for a region (deduplicated per permit) with risk totals."""
|
||
sql = """
|
||
SELECT
|
||
rtp.permit_id,
|
||
p.name AS permit_name,
|
||
rtp.theme_id,
|
||
COALESCE(t.name, '') AS theme_name,
|
||
COUNT(rpr.risk_id) OVER (PARTITION BY rtp.permit_id) AS risk_total,
|
||
COALESCE(rpd.is_v2_visible, true) AS is_v2_visible
|
||
FROM region_theme_permits rtp
|
||
JOIN permits p ON p.id = rtp.permit_id
|
||
LEFT JOIN themes t ON t.id = rtp.theme_id
|
||
LEFT JOIN region_permit_details rpd ON rpd.region_id = rtp.region_id AND rpd.permit_id = rtp.permit_id
|
||
LEFT JOIN region_permit_risks rpr
|
||
ON rpr.region_id = rtp.region_id
|
||
AND rpr.permit_id = rtp.permit_id
|
||
WHERE rtp.region_id = %s
|
||
ORDER BY LOWER(p.name), LOWER(COALESCE(t.name, ''))
|
||
"""
|
||
catalog_map: "OrderedDict[str, Dict[str, Any]]" = OrderedDict()
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_v2_visibility_column(conn)
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (region_id,))
|
||
for permit_id, permit_name, theme_id, theme_name, risk_total, v2_visible in cur.fetchall():
|
||
pid = str(permit_id)
|
||
entry = catalog_map.setdefault(
|
||
pid,
|
||
{
|
||
"id": pid,
|
||
"name": str(permit_name),
|
||
"risk_count": int(risk_total or 0),
|
||
"is_v2_visible": bool(v2_visible),
|
||
"theme": {"id": "", "name": ""},
|
||
"themes": [],
|
||
},
|
||
)
|
||
if entry["theme"].get("id") or entry["theme"].get("name"):
|
||
pass
|
||
else:
|
||
entry["theme"] = {
|
||
"id": str(theme_id) if theme_id else "",
|
||
"name": str(theme_name) if theme_name else "",
|
||
}
|
||
if theme_id or theme_name:
|
||
theme_payload = {
|
||
"id": str(theme_id) if theme_id else "",
|
||
"name": str(theme_name) if theme_name else "",
|
||
}
|
||
if not any(
|
||
candidate.get("id") == theme_payload["id"]
|
||
and candidate.get("name") == theme_payload["name"]
|
||
for candidate in entry["themes"]
|
||
):
|
||
entry["themes"].append(theme_payload)
|
||
return list(catalog_map.values())
|
||
|
||
|
||
def _fetch_all_theme_options(conn: pg.Connection) -> List[Dict[str, str]]:
|
||
"""Return the list of all theme records for cross-region binding."""
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT id, name
|
||
FROM themes
|
||
ORDER BY name
|
||
"""
|
||
)
|
||
return [{"id": str(theme_id), "name": str(theme_name)} for theme_id, theme_name in cur.fetchall()]
|
||
|
||
|
||
def _fetch_region_theme_map(
|
||
conn: pg.Connection, region_ids: Iterable[str]
|
||
) -> Dict[str, List[Dict[str, str]]]:
|
||
"""Return mapping of region_id -> theme metadata list."""
|
||
uuid_list: List[uuid.UUID] = []
|
||
for region_id in region_ids:
|
||
if not region_id:
|
||
continue
|
||
try:
|
||
uuid_list.append(uuid.UUID(str(region_id)))
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if not uuid_list:
|
||
return {}
|
||
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT rt.region_id, t.id, t.name
|
||
FROM region_themes rt
|
||
JOIN themes t ON t.id = rt.theme_id
|
||
WHERE rt.region_id = ANY(%s)
|
||
ORDER BY t.name
|
||
""",
|
||
(uuid_list,),
|
||
)
|
||
region_theme_map: Dict[str, List[Dict[str, str]]] = {}
|
||
for region_uuid, theme_id, theme_name in cur.fetchall():
|
||
rid = str(region_uuid)
|
||
region_theme_map.setdefault(rid, []).append(
|
||
{"id": str(theme_id), "name": str(theme_name)}
|
||
)
|
||
return region_theme_map
|
||
|
||
|
||
def resolve_region_permit_theme(region_id: str, permit_id: str) -> Optional[Dict[str, str]]:
|
||
"""Return the first theme associated with a region-permit pair (if any)."""
|
||
sql = """
|
||
SELECT rtp.theme_id, t.name
|
||
FROM region_theme_permits rtp
|
||
LEFT JOIN themes t ON t.id = rtp.theme_id
|
||
WHERE rtp.region_id = %s
|
||
AND rtp.permit_id = %s
|
||
ORDER BY t.name NULLS LAST
|
||
LIMIT 1
|
||
"""
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (region_id, permit_id))
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return None
|
||
theme_id, theme_name = row
|
||
return {
|
||
"id": str(theme_id) if theme_id else "",
|
||
"name": str(theme_name) if theme_name else "",
|
||
}
|
||
|
||
|
||
def _load_permit_scopes_for_region(
|
||
conn: pg.Connection, region_id: str, permit_ids: List[str]
|
||
) -> Dict[str, List[Dict[str, str]]]:
|
||
"""Return mapping of permit_id -> business scopes for that permit within region."""
|
||
scope_map: Dict[str, List[Dict[str, str]]] = {pid: [] for pid in permit_ids}
|
||
if not permit_ids:
|
||
return scope_map
|
||
|
||
sql = """
|
||
SELECT rps.permit_id, bs.id, bs.description
|
||
FROM region_permit_scopes rps
|
||
JOIN business_scopes bs ON bs.id = rps.scope_id
|
||
WHERE rps.region_id = %s
|
||
ORDER BY rps.permit_id, bs.description
|
||
"""
|
||
cur = conn.cursor()
|
||
try:
|
||
cur.execute(sql, (region_id,))
|
||
except pg.ProgrammingError as exc:
|
||
# 42P01 => undefined_table; allow fallback when migration not yet applied.
|
||
sqlstate = getattr(exc, "sqlstate", "")
|
||
if sqlstate == "42P01":
|
||
return scope_map
|
||
raise
|
||
|
||
for permit_id, scope_id, description in cur.fetchall():
|
||
pid = str(permit_id)
|
||
if pid not in scope_map:
|
||
continue
|
||
scope_map[pid].append({"id": str(scope_id), "description": str(description)})
|
||
return scope_map
|
||
|
||
|
||
def _load_permit_sources_for_region(
|
||
conn: pg.Connection, region_id: str, permit_ids: Iterable[str]
|
||
) -> Dict[str, Dict[str, Any]]:
|
||
"""Return mapping of permit_id -> source metadata for the permit."""
|
||
permit_ids_list = [str(pid) for pid in permit_ids]
|
||
if not permit_ids_list:
|
||
return {}
|
||
if not _permit_sources_available(conn):
|
||
return {}
|
||
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT permit_id, source_type, source_name, source_detail, updated_at
|
||
FROM permit_sources
|
||
WHERE region_id = %s AND permit_id = ANY(%s)
|
||
""",
|
||
(region_id, permit_ids_list),
|
||
)
|
||
|
||
sources: Dict[str, Dict[str, Any]] = {}
|
||
for permit_id, source_type, source_name, source_detail, updated_at in cur.fetchall():
|
||
pid = str(permit_id)
|
||
sources[pid] = {
|
||
"source_type": source_type or "",
|
||
"source_name": source_name or "",
|
||
"source_detail": source_detail or "",
|
||
"updated_at": _convert_snapshot_value(updated_at),
|
||
}
|
||
return sources
|
||
|
||
|
||
def load_permits_and_risks(
|
||
region_id: str,
|
||
theme_id: Optional[str] = None,
|
||
permit_id: Optional[str] = None,
|
||
only_visible: bool = False
|
||
) -> List[Dict[str, object]]:
|
||
"""Return permits with attached risk entries for a region (optionally filtered by theme)."""
|
||
# Ensure optional permit file tables exist before running user queries.
|
||
try:
|
||
_ensure_permit_file_schema()
|
||
except Exception as exc:
|
||
logger.warning("[PERMIT-FILES] Failed to ensure permit file schema before loading permits: %s", exc)
|
||
|
||
try:
|
||
_ensure_permit_approval_departments_schema()
|
||
except Exception as exc:
|
||
logger.warning("[PERMIT-APPROVAL] Failed to ensure approval departments schema: %s", exc)
|
||
|
||
sql = """
|
||
SELECT
|
||
rtp.theme_id,
|
||
t.name AS theme_name,
|
||
p.id AS permit_id,
|
||
p.name AS permit_name,
|
||
rk.id AS risk_id,
|
||
rk.risk_content,
|
||
rk.legal_basis,
|
||
rk.document_no,
|
||
rk.summary,
|
||
rk.remark,
|
||
rpr.serial_number,
|
||
rpd.permit_status,
|
||
rpd.subitem_summary,
|
||
rpd.responsible_contact,
|
||
rpd.jurisdiction_scope,
|
||
rpd.filler_name,
|
||
COALESCE(pad.department_name, rpd.unit_name) AS unit_name,
|
||
rpd.source_update_date,
|
||
rpd.contact_info,
|
||
COALESCE(rpd.is_v2_visible, true) AS is_v2_visible
|
||
FROM region_permit_details rpd
|
||
JOIN permits p ON p.id = rpd.permit_id
|
||
LEFT JOIN permit_approval_departments pad
|
||
ON (p.name = pad.permit_name OR p.name LIKE CONCAT(pad.permit_name, '%'))
|
||
LEFT JOIN region_theme_permits rtp
|
||
ON rtp.region_id = rpd.region_id
|
||
AND rtp.permit_id = rpd.permit_id
|
||
LEFT JOIN themes t ON t.id = rtp.theme_id
|
||
LEFT JOIN region_permit_risks rpr
|
||
ON rpr.region_id = rpd.region_id
|
||
AND rpr.permit_id = rpd.permit_id
|
||
LEFT JOIN risks rk ON rk.id = rpr.risk_id
|
||
WHERE rpd.region_id = %s
|
||
"""
|
||
params: List[Any] = [region_id]
|
||
theme_filter = theme_id if (theme_id and not _is_all_theme_marker(theme_id)) else None
|
||
if theme_filter:
|
||
sql += " AND (rtp.theme_id = %s OR t.name = '所有主题事项')"
|
||
params.append(theme_filter)
|
||
if permit_id is not None:
|
||
sql += " AND rpd.permit_id = %s"
|
||
params.append(permit_id)
|
||
if only_visible:
|
||
sql += " AND COALESCE(rpd.is_v2_visible, true) = true"
|
||
|
||
sql += """
|
||
ORDER BY p.name, LENGTH(rpr.serial_number), rpr.serial_number, rk.risk_content
|
||
"""
|
||
permits: Dict[str, Dict[str, object]] = {}
|
||
risk_seen_map: Dict[str, Set[str]] = {} # pid -> set of risk_ids
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_v2_visibility_column(conn)
|
||
_ensure_contact_info_column(conn)
|
||
cur = conn.cursor()
|
||
cur.execute(sql, tuple(params))
|
||
for row in cur.fetchall():
|
||
(
|
||
row_theme_id,
|
||
row_theme_name,
|
||
permit_id,
|
||
permit_name,
|
||
risk_id,
|
||
risk_content,
|
||
legal_basis,
|
||
document_no,
|
||
summary,
|
||
remark,
|
||
serial_number,
|
||
permit_status,
|
||
subitem_summary,
|
||
responsible_contact,
|
||
jurisdiction_scope,
|
||
filler_name,
|
||
unit_name,
|
||
source_update_date,
|
||
contact_info,
|
||
v2_visible,
|
||
) = row
|
||
pid = str(permit_id)
|
||
theme_id_value = str(row_theme_id) if row_theme_id else ""
|
||
theme_name_value = str(row_theme_name) if row_theme_name else ""
|
||
entry = permits.setdefault(
|
||
pid,
|
||
{
|
||
"id": pid,
|
||
"name": str(permit_name),
|
||
"business_scopes": [],
|
||
"risks": [],
|
||
"permit_status": None,
|
||
"subitem_summary": None,
|
||
"responsible_contact": None,
|
||
"jurisdiction_scope": None,
|
||
"filler_name": None,
|
||
"unit_name": None,
|
||
"source_update_date": None,
|
||
"contact_info": None,
|
||
"theme": {
|
||
"id": theme_id_value,
|
||
"name": theme_name_value,
|
||
},
|
||
"themes": [],
|
||
"binds_all_themes": False,
|
||
},
|
||
)
|
||
# ... (theme logic skipped for brevity if identical, but I must match exact lines to be safe)
|
||
if theme_id_value and not entry["theme"].get("id"):
|
||
entry["theme"]["id"] = theme_id_value
|
||
if theme_name_value and not entry["theme"].get("name"):
|
||
entry["theme"]["name"] = theme_name_value
|
||
if theme_id_value or theme_name_value:
|
||
# Same theme duplication logic
|
||
theme_list = entry.get("themes") or []
|
||
duplicate = False
|
||
for theme_entry in theme_list:
|
||
if theme_id_value:
|
||
if theme_entry.get("id") == theme_id_value:
|
||
duplicate = True
|
||
break
|
||
else:
|
||
if not theme_entry.get("id") and theme_entry.get("name") == theme_name_value:
|
||
duplicate = True
|
||
break
|
||
if not duplicate:
|
||
theme_list.append(
|
||
{
|
||
"id": theme_id_value,
|
||
"name": theme_name_value,
|
||
}
|
||
)
|
||
entry["themes"] = theme_list
|
||
|
||
if entry["permit_status"] is None and permit_status:
|
||
entry["permit_status"] = permit_status.strip() or None
|
||
if entry["subitem_summary"] is None and subitem_summary:
|
||
entry["subitem_summary"] = subitem_summary.strip() or None
|
||
if entry["responsible_contact"] is None and responsible_contact:
|
||
entry["responsible_contact"] = responsible_contact.strip() or None
|
||
if entry["jurisdiction_scope"] is None and jurisdiction_scope:
|
||
entry["jurisdiction_scope"] = jurisdiction_scope.strip() or None
|
||
if entry["filler_name"] is None and filler_name:
|
||
entry["filler_name"] = filler_name.strip() or None
|
||
if entry["unit_name"] is None and unit_name:
|
||
entry["unit_name"] = unit_name.strip() or None
|
||
if entry["source_update_date"] is None and source_update_date:
|
||
entry["source_update_date"] = source_update_date.strip() or None
|
||
if entry["contact_info"] is None and contact_info:
|
||
entry["contact_info"] = contact_info.strip() or None
|
||
if risk_id is not None:
|
||
risk_id_str = str(risk_id)
|
||
# Avoid duplicates when a permit has multiple themes
|
||
seen_risk_ids = risk_seen_map.setdefault(pid, set())
|
||
if risk_id_str not in seen_risk_ids:
|
||
seen_risk_ids.add(risk_id_str)
|
||
summary_markdown = _format_summary_markdown(summary or "")
|
||
remark_markdown = _format_summary_markdown(remark or "")
|
||
entry["risks"].append(
|
||
{
|
||
"id": risk_id_str,
|
||
"risk_content": risk_content or "",
|
||
"legal_basis": legal_basis or "",
|
||
"document_no": document_no or "",
|
||
"summary": summary_markdown,
|
||
"remark": remark_markdown,
|
||
"serial_number": serial_number,
|
||
}
|
||
)
|
||
|
||
permit_ids = list(permits.keys())
|
||
scope_map = _load_permit_scopes_for_region(conn, region_id, permit_ids)
|
||
source_map = _load_permit_sources_for_region(conn, region_id, permit_ids)
|
||
try:
|
||
file_meta_map = _load_permit_file_metadata(conn, region_id, permit_ids)
|
||
except pg.DatabaseError as exc: # type: ignore[attr-defined]
|
||
sqlstate = getattr(exc, "sqlstate", "")
|
||
if sqlstate == "42P01":
|
||
logger.warning("[PERMIT-FILES] permit_file_links missing while loading permits, recreating schema lazily")
|
||
global _PERMIT_FILE_SCHEMA_READY
|
||
_PERMIT_FILE_SCHEMA_READY = None
|
||
_ensure_permit_file_schema()
|
||
file_meta_map = {}
|
||
else:
|
||
raise
|
||
all_theme_flags = _fetch_permit_all_theme_flags(conn, region_id, permit_ids)
|
||
for pid in permit_ids:
|
||
permits[pid]["business_scopes"] = scope_map.get(pid, [])
|
||
if pid in source_map:
|
||
permits[pid]["permit_source"] = source_map[pid]
|
||
else:
|
||
permits[pid]["permit_source"] = {
|
||
"source_type": "",
|
||
"source_name": "",
|
||
"source_detail": "",
|
||
"updated_at": None,
|
||
}
|
||
if pid in file_meta_map:
|
||
permits[pid]["permit_file"] = file_meta_map[pid]
|
||
else:
|
||
permits[pid]["permit_file"] = {
|
||
"file_id": "",
|
||
"filename": "",
|
||
"content_type": "",
|
||
"file_size": 0,
|
||
"created_at": None,
|
||
"uploaded_by": "",
|
||
}
|
||
if "themes" not in permits[pid] or permits[pid]["themes"] is None:
|
||
permits[pid]["themes"] = []
|
||
if all_theme_flags.get(pid):
|
||
permits[pid]["binds_all_themes"] = True
|
||
theme_list = permits[pid].get("themes") or []
|
||
has_marker = any(_is_all_theme_marker(theme.get("id")) or _is_all_theme_marker(theme.get("name")) for theme in theme_list)
|
||
if not has_marker:
|
||
theme_list.append(
|
||
{
|
||
"id": ALL_THEMES_SENTINEL,
|
||
"name": ALL_THEMES_DISPLAY_NAME,
|
||
"is_virtual": True,
|
||
}
|
||
)
|
||
permits[pid]["themes"] = theme_list
|
||
theme_meta = permits[pid].get("theme") or {}
|
||
if not (theme_meta.get("id") or theme_meta.get("name")):
|
||
permits[pid]["theme"] = {
|
||
"id": ALL_THEMES_SENTINEL,
|
||
"name": ALL_THEMES_DISPLAY_NAME,
|
||
}
|
||
else:
|
||
permits[pid]["binds_all_themes"] = False
|
||
return list(permits.values())
|
||
|
||
|
||
def fetch_permit_file(region_id: str, permit_id: str) -> Optional[Dict[str, Any]]:
|
||
"""Return file payload for a region-permit pair if available."""
|
||
if not region_id or not permit_id:
|
||
return None
|
||
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_permit_file_schema(conn)
|
||
try:
|
||
row = _select_permit_file_blob(conn, region_id, permit_id)
|
||
except pg.DatabaseError as exc: # type: ignore[attr-defined]
|
||
sqlstate = getattr(exc, "sqlstate", "")
|
||
if sqlstate == "42P01":
|
||
logger.warning(
|
||
"[PERMIT-FILES] permit_file_links missing when downloading file (region=%s permit=%s); recreating schema",
|
||
region_id,
|
||
permit_id,
|
||
)
|
||
global _PERMIT_FILE_SCHEMA_READY
|
||
_PERMIT_FILE_SCHEMA_READY = None
|
||
_ensure_permit_file_schema()
|
||
return None
|
||
raise
|
||
if not row:
|
||
return None
|
||
|
||
(
|
||
file_id,
|
||
filename,
|
||
content_type,
|
||
file_size,
|
||
file_data,
|
||
created_at,
|
||
uploaded_by,
|
||
) = row
|
||
return {
|
||
"file_id": str(file_id),
|
||
"filename": filename or "",
|
||
"content_type": content_type or "application/octet-stream",
|
||
"file_size": int(file_size or 0),
|
||
"file_data": bytes(file_data) if file_data is not None else b"",
|
||
"created_at": created_at.isoformat() if created_at else None,
|
||
"uploaded_by": uploaded_by or "",
|
||
}
|
||
|
||
|
||
def list_stored_permit_files(
|
||
limit: int = 20,
|
||
offset: int = 0,
|
||
keyword: Optional[str] = None,
|
||
) -> Dict[str, Any]:
|
||
"""Return paginated permit file metadata with optional keyword filtering."""
|
||
try:
|
||
limit = int(limit)
|
||
except (TypeError, ValueError):
|
||
limit = 20
|
||
limit = max(1, min(limit, 100))
|
||
try:
|
||
offset = int(offset)
|
||
except (TypeError, ValueError):
|
||
offset = 0
|
||
offset = max(0, offset)
|
||
normalized_keyword = _clean_text(keyword) or None
|
||
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_permit_file_schema(conn)
|
||
cur = conn.cursor()
|
||
|
||
where_clauses: List[str] = []
|
||
params: List[Any] = []
|
||
if normalized_keyword:
|
||
like_pattern = f"%{normalized_keyword}%"
|
||
where_clauses.append(
|
||
"""
|
||
(
|
||
pf.filename ILIKE %s
|
||
OR EXISTS (
|
||
SELECT 1
|
||
FROM permit_file_links sub
|
||
JOIN permits sp ON sp.id = sub.permit_id
|
||
WHERE sub.file_id = pf.id
|
||
AND sp.name ILIKE %s
|
||
)
|
||
)
|
||
"""
|
||
)
|
||
params.extend([like_pattern, like_pattern])
|
||
where_sql = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
|
||
|
||
count_sql = f"SELECT COUNT(*) FROM permit_files pf {where_sql}"
|
||
cur.execute(count_sql, tuple(params))
|
||
total_rows = int(cur.fetchone()[0] or 0)
|
||
|
||
data_sql = f"""
|
||
SELECT
|
||
pf.id,
|
||
pf.filename,
|
||
pf.content_type,
|
||
pf.file_size,
|
||
pf.created_at,
|
||
pf.uploaded_by,
|
||
COALESCE(
|
||
json_agg(
|
||
jsonb_build_object(
|
||
'region_id', pfl.region_id::text,
|
||
'region_name', COALESCE(r.name, ''),
|
||
'permit_id', pfl.permit_id::text,
|
||
'permit_name', COALESCE(p.name, '')
|
||
)
|
||
) FILTER (WHERE pfl.permit_id IS NOT NULL),
|
||
'[]'::json
|
||
) AS link_payload
|
||
FROM permit_files pf
|
||
LEFT JOIN permit_file_links pfl ON pfl.file_id = pf.id
|
||
LEFT JOIN regions r ON r.id = pfl.region_id
|
||
LEFT JOIN permits p ON p.id = pfl.permit_id
|
||
{where_sql}
|
||
GROUP BY pf.id, pf.filename, pf.content_type, pf.file_size, pf.created_at, pf.uploaded_by
|
||
ORDER BY pf.created_at DESC, pf.id DESC
|
||
LIMIT %s OFFSET %s
|
||
"""
|
||
data_params = list(params) + [limit, offset]
|
||
cur.execute(data_sql, data_params)
|
||
rows = cur.fetchall()
|
||
|
||
files: List[Dict[str, Any]] = []
|
||
for row in rows:
|
||
(
|
||
file_id,
|
||
filename,
|
||
content_type,
|
||
file_size,
|
||
created_at,
|
||
uploaded_by,
|
||
link_payload,
|
||
) = row
|
||
link_items: List[Dict[str, str]] = []
|
||
if link_payload:
|
||
if isinstance(link_payload, str):
|
||
try:
|
||
parsed = json.loads(link_payload)
|
||
except (TypeError, ValueError):
|
||
parsed = []
|
||
else:
|
||
parsed = link_payload if isinstance(link_payload, list) else []
|
||
if isinstance(parsed, list):
|
||
for entry in parsed:
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
link_items.append(
|
||
{
|
||
"region_id": str(entry.get("region_id") or ""),
|
||
"region_name": str(entry.get("region_name") or ""),
|
||
"permit_id": str(entry.get("permit_id") or ""),
|
||
"permit_name": str(entry.get("permit_name") or ""),
|
||
}
|
||
)
|
||
files.append(
|
||
{
|
||
"file_id": str(file_id),
|
||
"filename": filename or "",
|
||
"content_type": content_type or "",
|
||
"file_size": int(file_size or 0),
|
||
"created_at": created_at.isoformat() if created_at else None,
|
||
"uploaded_by": uploaded_by or "",
|
||
"links": link_items,
|
||
}
|
||
)
|
||
|
||
return {
|
||
"files": files,
|
||
"pagination": {
|
||
"limit": limit,
|
||
"offset": offset,
|
||
"total": total_rows,
|
||
},
|
||
}
|
||
|
||
|
||
def delete_stored_permit_file(file_id: str) -> bool:
|
||
"""Delete a stored permit file (and cascading links)."""
|
||
normalized = _clean_text(file_id)
|
||
if not normalized:
|
||
raise ValueError("file_id 不能为空")
|
||
|
||
with _lic_pg_conn(autocommit=True) as conn:
|
||
_ensure_permit_file_schema(conn)
|
||
cur = conn.cursor()
|
||
cur.execute("DELETE FROM permit_files WHERE id = %s", (normalized,))
|
||
return cur.rowcount > 0
|
||
|
||
|
||
def start_import_session_from_file(
|
||
file_id: str,
|
||
*,
|
||
requested_by: Optional[str] = None,
|
||
uploader_department_id: Optional[str] = None,
|
||
bound_department_id: Optional[str] = None,
|
||
binding_mode: str = "auto",
|
||
) -> Dict[str, Any]:
|
||
"""Create a fresh import session using an archived permit file."""
|
||
normalized = _clean_text(file_id)
|
||
if not normalized:
|
||
raise ValueError("file_id 不能为空")
|
||
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_permit_file_schema(conn)
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT filename, content_type, file_data, uploaded_by
|
||
FROM permit_files
|
||
WHERE id = %s
|
||
""",
|
||
(normalized,),
|
||
)
|
||
row = cur.fetchone()
|
||
|
||
if not row:
|
||
raise ValueError("文件不存在或已删除")
|
||
|
||
filename, content_type, file_data, uploaded_by = row
|
||
if isinstance(file_data, memoryview):
|
||
file_bytes = file_data.tobytes()
|
||
elif isinstance(file_data, bytearray):
|
||
file_bytes = bytes(file_data)
|
||
else:
|
||
file_bytes = bytes(file_data or b"")
|
||
effective_uploader = requested_by or uploaded_by
|
||
|
||
return start_permit_import_session(
|
||
file_bytes=file_bytes,
|
||
filename=filename or "许可导入.xlsx",
|
||
content_type=content_type or "application/octet-stream",
|
||
uploaded_by=effective_uploader,
|
||
uploader_department_id=uploader_department_id,
|
||
bound_department_id=bound_department_id,
|
||
binding_mode=binding_mode,
|
||
)
|
||
|
||
|
||
def _select_permit_file_blob(conn: pg.Connection, region_id: str, permit_id: str):
|
||
"""Fetch a single permit file with binary content, recreating tables if needed."""
|
||
sql = """
|
||
SELECT
|
||
pf.id,
|
||
pf.filename,
|
||
pf.content_type,
|
||
pf.file_size,
|
||
pf.file_data,
|
||
pf.created_at,
|
||
pf.uploaded_by
|
||
FROM permit_file_links pfl
|
||
JOIN permit_files pf ON pf.id = pfl.file_id
|
||
WHERE pfl.region_id = %s
|
||
AND pfl.permit_id = %s
|
||
LIMIT 1
|
||
"""
|
||
attempts = 0
|
||
while attempts < 2:
|
||
try:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (region_id, permit_id))
|
||
return cur.fetchone()
|
||
except pg.DatabaseError as exc: # type: ignore[attr-defined]
|
||
sqlstate = getattr(exc, "sqlstate", "")
|
||
if sqlstate != "42P01":
|
||
raise
|
||
attempts += 1
|
||
try:
|
||
conn.rollback()
|
||
except Exception:
|
||
pass
|
||
global _PERMIT_FILE_SCHEMA_READY
|
||
_PERMIT_FILE_SCHEMA_READY = None
|
||
_ensure_permit_file_schema(conn)
|
||
if attempts >= 2:
|
||
logger.warning(
|
||
"[PERMIT-FILES] permit_file_links table missing after recreate attempt when downloading file (region=%s permit=%s)",
|
||
region_id,
|
||
permit_id,
|
||
)
|
||
return None
|
||
return None
|
||
|
||
|
||
def find_permit_contexts_by_name(permit_name: str) -> List[Dict[str, str]]:
|
||
"""Return region/theme contexts for permits with an exact name match or prefix match."""
|
||
if not permit_name:
|
||
return []
|
||
|
||
sql = """
|
||
SELECT
|
||
rpd.region_id,
|
||
r.name AS region_name,
|
||
rtp.theme_id,
|
||
t.name AS theme_name,
|
||
p.id AS permit_id,
|
||
p.name AS permit_name,
|
||
COALESCE(rpd.is_v2_visible, true) AS is_v2_visible
|
||
FROM region_permit_details rpd
|
||
JOIN permits p ON p.id = rpd.permit_id
|
||
JOIN regions r ON r.id = rpd.region_id
|
||
LEFT JOIN region_theme_permits rtp ON rtp.region_id = rpd.region_id AND rtp.permit_id = rpd.permit_id
|
||
LEFT JOIN themes t ON t.id = rtp.theme_id
|
||
WHERE p.name = %s
|
||
ORDER BY r.name, t.name NULLS LAST
|
||
"""
|
||
ordered: OrderedDict[Tuple[str, str], Dict[str, str]] = OrderedDict()
|
||
with _lic_pg_conn() as conn:
|
||
_ensure_v2_visibility_column(conn)
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (permit_name,))
|
||
rows = cur.fetchall()
|
||
|
||
# If no exact match, try prefix match (e.g. "药品经营许可" matching "药品经营许可证")
|
||
if not rows and len(permit_name) >= 2:
|
||
sql_fuzzy = """
|
||
SELECT
|
||
rpd.region_id,
|
||
r.name AS region_name,
|
||
rtp.theme_id,
|
||
t.name AS theme_name,
|
||
p.id AS permit_id,
|
||
p.name AS permit_name,
|
||
COALESCE(rpd.is_v2_visible, true) AS is_v2_visible
|
||
FROM region_permit_details rpd
|
||
JOIN permits p ON p.id = rpd.permit_id
|
||
JOIN regions r ON r.id = rpd.region_id
|
||
LEFT JOIN region_theme_permits rtp ON rtp.region_id = rpd.region_id AND rtp.permit_id = rpd.permit_id
|
||
LEFT JOIN themes t ON t.id = rtp.theme_id
|
||
WHERE p.name LIKE %s
|
||
ORDER BY r.name, t.name NULLS LAST
|
||
"""
|
||
cur.execute(sql_fuzzy, (permit_name + "%",))
|
||
rows = cur.fetchall()
|
||
|
||
for row in rows:
|
||
region_id, region_name, theme_id, theme_name, permit_id, canonical_name, v2_visible = row
|
||
rid = str(region_id)
|
||
pid = str(permit_id)
|
||
tid = str(theme_id) if theme_id else ""
|
||
tname = str(theme_name) if theme_name else ""
|
||
key = (rid, pid)
|
||
if key in ordered:
|
||
continue
|
||
ordered[key] = {
|
||
"region_id": rid,
|
||
"region_name": str(region_name),
|
||
"theme_id": tid,
|
||
"theme_name": tname,
|
||
"permit_id": pid,
|
||
"permit_name": str(canonical_name),
|
||
"is_v2_visible": bool(v2_visible),
|
||
}
|
||
return [item for item in ordered.values() if item.get("is_v2_visible")]
|
||
|
||
|
||
def load_theme_payload(region_id: str, theme_id: str, only_visible: bool = False) -> Dict[str, object]:
|
||
"""Assemble full data bundle for a region-theme selection."""
|
||
info_sql = """
|
||
SELECT r.id, r.name, t.id, t.name
|
||
FROM regions r
|
||
JOIN region_themes rt ON rt.region_id = r.id
|
||
JOIN themes t ON t.id = rt.theme_id
|
||
WHERE r.id = %s AND t.id = %s
|
||
LIMIT 1
|
||
"""
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(info_sql, (region_id, theme_id))
|
||
row = cur.fetchone()
|
||
if not row:
|
||
raise ValueError("Region/theme combination not found")
|
||
region_uuid, region_name, theme_uuid, theme_name = row
|
||
|
||
permits = load_permits_and_risks(region_id, theme_id, only_visible=only_visible)
|
||
return {
|
||
"region": {"id": str(region_uuid), "name": str(region_name)},
|
||
"theme": {"id": str(theme_uuid), "name": str(theme_name)},
|
||
"permits": permits,
|
||
}
|
||
|
||
|
||
def _get_checkpoints_dir() -> str:
|
||
"""Get the directory for storing checkpoint files."""
|
||
base_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data")
|
||
checkpoints_dir = os.path.join(base_dir, "checkpoints")
|
||
os.makedirs(checkpoints_dir, exist_ok=True)
|
||
return checkpoints_dir
|
||
|
||
|
||
def _get_all_tables() -> List[str]:
|
||
"""Get list of all tables in the licensing_risks database."""
|
||
sql = """
|
||
SELECT table_name
|
||
FROM information_schema.tables
|
||
WHERE table_schema = 'public'
|
||
AND table_type = 'BASE TABLE'
|
||
ORDER BY table_name
|
||
"""
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql)
|
||
return [row[0] for row in cur.fetchall()]
|
||
|
||
|
||
def _get_table_dependencies(conn: pg.Connection) -> Dict[str, List[str]]:
|
||
"""
|
||
获取表依赖关系图。
|
||
返回: {被引用表名: [引用它的表列表]}
|
||
例如: {'regions': ['region_themes', 'region_scopes'], 'themes': ['region_themes']}
|
||
"""
|
||
sql = """
|
||
SELECT
|
||
ccu.table_name AS referenced_table,
|
||
tc.table_name AS dependent_table
|
||
FROM information_schema.table_constraints tc
|
||
JOIN information_schema.key_column_usage kcu
|
||
ON tc.constraint_name = kcu.constraint_name
|
||
AND tc.table_schema = kcu.table_schema
|
||
JOIN information_schema.constraint_column_usage ccu
|
||
ON tc.constraint_name = ccu.constraint_name
|
||
AND tc.table_schema = ccu.table_schema
|
||
WHERE tc.constraint_type = 'FOREIGN KEY'
|
||
AND tc.table_schema = 'public'
|
||
ORDER BY ccu.table_name, tc.table_name
|
||
"""
|
||
cur = conn.cursor()
|
||
cur.execute(sql)
|
||
|
||
dependencies = {}
|
||
for row in cur.fetchall():
|
||
referenced_table, dependent_table = row
|
||
if referenced_table not in dependencies:
|
||
dependencies[referenced_table] = []
|
||
if dependent_table not in dependencies[referenced_table]:
|
||
dependencies[referenced_table].append(dependent_table)
|
||
|
||
return dependencies
|
||
|
||
|
||
def _topological_sort_tables(all_tables: List[str], dependencies: Dict[str, List[str]]) -> List[str]:
|
||
"""
|
||
使用拓扑排序确定表恢复顺序。
|
||
先返回父表（无外键依赖），再返回子表（引用其他表）。
|
||
这确保恢复时不会违反外键约束。
|
||
"""
|
||
from collections import deque
|
||
|
||
# 计算每个表的入度（被多少表引用/依赖）
|
||
in_degree = {table: 0 for table in all_tables}
|
||
for parent_table, children in dependencies.items():
|
||
for child in children:
|
||
if child in all_tables:
|
||
in_degree[child] += 1
|
||
|
||
# 入度为0的表是父表，可以先恢复
|
||
queue = deque([table for table in all_tables if in_degree[table] == 0])
|
||
result = []
|
||
|
||
while queue:
|
||
table = queue.popleft()
|
||
result.append(table)
|
||
|
||
# 减少依赖该表的子表的入度
|
||
for parent, children in dependencies.items():
|
||
if parent == table:
|
||
for child in children:
|
||
if child in all_tables and in_degree[child] > 0:
|
||
in_degree[child] -= 1
|
||
if in_degree[child] == 0:
|
||
queue.append(child)
|
||
|
||
# 如果有循环依赖或孤立节点，将剩余表添加到末尾
|
||
for table in all_tables:
|
||
if table not in result:
|
||
result.append(table)
|
||
|
||
return result
|
||
|
||
|
||
def _reset_all_sequences(conn: pg.Connection, tables: List[str]) -> None:
|
||
"""
|
||
Reset sequences for all tables to the max(id) + 1.
|
||
This prevents 'duplicate key value violates unique constraint' errors after restore.
|
||
"""
|
||
cur = conn.cursor()
|
||
for table in tables:
|
||
# Check if table has an 'id' column or serial column that is a sequence
|
||
# Heuristic: verify if 'id' column exists
|
||
try:
|
||
# Check if 'id' column exists
|
||
cur.execute(
|
||
"""
|
||
SELECT column_name, data_type
|
||
FROM information_schema.columns
|
||
WHERE table_name = %s AND column_name = 'id'
|
||
""",
|
||
(table,)
|
||
)
|
||
col = cur.fetchone()
|
||
if not col:
|
||
continue # No ID column, skip
|
||
|
||
# Try to reset sequence.
|
||
# If the column is not a serial/identity, setval might fail or do nothing if no sequence attached.
|
||
# A robust way is to find the sequence name dynamically.
|
||
|
||
# This query finds the sequence associated with a column (if any)
|
||
cur.execute(
|
||
"""
|
||
SELECT pg_get_serial_sequence(%s, 'id')
|
||
""",
|
||
(table,)
|
||
)
|
||
seq_row = cur.fetchone()
|
||
if seq_row and seq_row[0]:
|
||
seq_name = seq_row[0]
|
||
# Get max id
|
||
cur.execute(f"SELECT MAX(id) FROM {table}")
|
||
max_id = cur.fetchone()[0]
|
||
if max_id is not None:
|
||
# Reset sequence
|
||
# Using setval with is_called=true ensures next value is max+1
|
||
cur.execute(f"SELECT setval(%s, %s, true)", (seq_name, max_id))
|
||
logger.info(f"[CHECKPOINT] Reset sequence {seq_name} for table {table} to {max_id}")
|
||
else:
|
||
# Table empty, reset to 1 (is_called=false means next will be 1)
|
||
cur.execute(f"SELECT setval(%s, 1, false)", (seq_name,))
|
||
logger.info(f"[CHECKPOINT] Reset sequence {seq_name} for table {table} to 1 (empty)")
|
||
except Exception as e:
|
||
# Log but don't fail the entire restore for one sequence issue
|
||
logger.warning(f"[CHECKPOINT] Failed to reset sequence for table {table}: {e}")
|
||
|
||
|
||
def _backup_table(conn: pg.Connection, table_name: str) -> Tuple[List[Dict[str, Any]], int]:
|
||
"""Backup a single table and return its data and row count."""
|
||
logger.info(f"[CHECKPOINT] Backing up table: {table_name}")
|
||
sql = f"SELECT * FROM {table_name}"
|
||
cur = conn.cursor()
|
||
cur.execute(sql)
|
||
|
||
rows = cur.fetchall()
|
||
colnames = [desc[0] for desc in cur.description]
|
||
|
||
data = []
|
||
for row in rows:
|
||
row_dict = {}
|
||
for i, col in enumerate(colnames):
|
||
value = row[i]
|
||
if value is not None:
|
||
# Convert UUID and other non-serializable types to strings
|
||
if hasattr(value, 'isoformat'): # UUID, datetime, etc.
|
||
row_dict[col] = str(value)
|
||
else:
|
||
row_dict[col] = value
|
||
data.append(row_dict)
|
||
|
||
row_count = len(data)
|
||
logger.info(f"[CHECKPOINT] Backup complete: {table_name} - {row_count} rows, {len(colnames)} columns")
|
||
|
||
return data, row_count
|
||
|
||
|
||
def _restore_table(conn: pg.Connection, table_name: str, data: List[Dict[str, Any]], batch_size: int = 1000) -> int:
|
||
"""Restore a table from backup data. Returns number of rows restored."""
|
||
if not data:
|
||
logger.info(f"[CHECKPOINT] Skipping empty table: {table_name}")
|
||
return 0
|
||
|
||
conn.autocommit = False
|
||
try:
|
||
cur = conn.cursor()
|
||
|
||
logger.info(f"[CHECKPOINT] Truncating table: {table_name}")
|
||
|
||
# Use TRUNCATE with CASCADE to handle foreign key dependencies
|
||
# This will automatically remove dependent records
|
||
truncate_sql = f"TRUNCATE TABLE {table_name} CASCADE"
|
||
cur.execute(truncate_sql)
|
||
|
||
columns = list(data[0].keys())
|
||
placeholders = ", ".join(["%s"] * len(columns))
|
||
|
||
logger.info(f"[CHECKPOINT] Restoring {len(data)} rows into {table_name} (columns: {', '.join(columns)})")
|
||
|
||
# 🚀 优化: 使用批量插入而不是逐行插入
|
||
# pg8000 使用 executemany 进行批量插入
|
||
batch_size = 1000 # 每批1000行
|
||
|
||
if len(data) <= batch_size:
|
||
# 小数据量，直接批量插入
|
||
# 小数据量，直接批量插入
|
||
values_list = []
|
||
for row in data:
|
||
processed_row = []
|
||
for col in columns:
|
||
val = row.get(col)
|
||
# Handle Base64 decoding for bytea fields (specifically file_data in permit_files)
|
||
if table_name == 'permit_files' and col == 'file_data' and isinstance(val, str):
|
||
try:
|
||
import base64
|
||
val = base64.b64decode(val)
|
||
except Exception:
|
||
pass
|
||
processed_row.append(val)
|
||
values_list.append(processed_row)
|
||
|
||
cur.executemany(f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({placeholders})", values_list)
|
||
logger.info(f"[CHECKPOINT] Bulk insert complete: {table_name} - {len(data)} rows inserted")
|
||
else:
|
||
# 大数据量，分批插入
|
||
total_rows = len(data)
|
||
for i in range(0, total_rows, batch_size):
|
||
batch_end = min(i + batch_size, total_rows)
|
||
batch_data = data[i:batch_end]
|
||
values_list = []
|
||
for row in batch_data:
|
||
processed_row = []
|
||
for col in columns:
|
||
val = row.get(col)
|
||
# Handle Base64 decoding for bytea fields (specifically file_data in permit_files)
|
||
if table_name == 'permit_files' and col == 'file_data' and isinstance(val, str):
|
||
try:
|
||
import base64
|
||
val = base64.b64decode(val)
|
||
except Exception:
|
||
pass # Keep as is if decode fails
|
||
processed_row.append(val)
|
||
values_list.append(processed_row)
|
||
cur.executemany(f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({placeholders})", values_list)
|
||
logger.info(f"[CHECKPOINT] Progress: {table_name} - {batch_end}/{total_rows} rows inserted")
|
||
|
||
conn.commit()
|
||
logger.info(f"[CHECKPOINT] Restore complete: {table_name} - {len(data)} rows successfully inserted")
|
||
|
||
return len(data)
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logger.error(f"[CHECKPOINT] Restore FAILED: {table_name} - {str(e)}")
|
||
raise e
|
||
finally:
|
||
conn.autocommit = False
|
||
|
||
|
||
def create_checkpoint(description: str = "", operator: str = "admin") -> Dict[str, Any]:
|
||
"""
|
||
安全创建checkpoint，所有表操作在一个事务中。
|
||
|
||
Args:
|
||
description: checkpoint的描述信息
|
||
|
||
Returns:
|
||
包含checkpoint信息的字典
|
||
"""
|
||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
checkpoint_id = f"checkpoint_{timestamp}"
|
||
|
||
logger.info("=" * 80)
|
||
logger.info(f"[CHECKPOINT] Starting checkpoint creation: {checkpoint_id}")
|
||
if description:
|
||
logger.info(f"[CHECKPOINT] Description: {description}")
|
||
logger.info("=" * 80)
|
||
|
||
tables = _get_all_tables()
|
||
logger.info(f"[CHECKPOINT] Found {len(tables)} tables to backup: {', '.join(tables)}")
|
||
|
||
checkpoint_data = {
|
||
"checkpoint_id": checkpoint_id,
|
||
"timestamp": timestamp,
|
||
"description": description,
|
||
"tables": {}
|
||
}
|
||
|
||
total_rows = 0
|
||
table_counts = {}
|
||
|
||
with _lic_pg_conn(autocommit=False) as conn:
|
||
try:
|
||
for i, table in enumerate(tables, 1):
|
||
logger.info(f"[CHECKPOINT] [{i}/{len(tables)}] Processing table: {table}")
|
||
data, row_count = _backup_table(conn, table)
|
||
checkpoint_data["tables"][table] = data
|
||
table_counts[table] = row_count
|
||
total_rows += row_count
|
||
logger.info(f"[CHECKPOINT] [{i}/{len(tables)}] Table {table} backed up: {row_count} rows")
|
||
|
||
# 全部成功后才提交
|
||
logger.info("[CHECKPOINT] All tables backed up successfully, committing transaction...")
|
||
conn.commit()
|
||
logger.info("[CHECKPOINT] Transaction committed")
|
||
except Exception as e:
|
||
# 任何失败都回滚
|
||
logger.error(f"[CHECKPOINT] ERROR during backup, rolling back: {str(e)}")
|
||
conn.rollback()
|
||
raise e
|
||
finally:
|
||
conn.autocommit = False
|
||
|
||
checkpoint_data["table_counts"] = table_counts
|
||
checkpoint_data["total_rows"] = total_rows
|
||
|
||
checkpoints_dir = _get_checkpoints_dir()
|
||
checkpoint_file = os.path.join(checkpoints_dir, f"{checkpoint_id}.json")
|
||
|
||
logger.info(f"[CHECKPOINT] Saving checkpoint file: {checkpoint_file}")
|
||
|
||
def json_serializer(obj):
|
||
"""Convert non-JSON serializable objects to strings."""
|
||
try:
|
||
import uuid
|
||
if isinstance(obj, uuid.UUID):
|
||
return str(obj)
|
||
except ImportError:
|
||
pass
|
||
|
||
if isinstance(obj, (bytes, bytearray, memoryview)):
|
||
import base64
|
||
# For data integrity in JSON, base64 is standard
|
||
return base64.b64encode(obj).decode('ascii')
|
||
|
||
if hasattr(obj, 'isoformat'):
|
||
return str(obj)
|
||
raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
|
||
|
||
with open(checkpoint_file, "w", encoding="utf-8") as f:
|
||
json.dump(checkpoint_data, f, ensure_ascii=False, indent=2, default=json_serializer)
|
||
|
||
logger.info("=" * 80)
|
||
|
||
# Log the operation
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="CHECKPOINT",
|
||
target_type="DATABASE",
|
||
target_id=checkpoint_id,
|
||
target_name=checkpoint_id,
|
||
change_summary=f"Created database checkpoint: {checkpoint_id}. {description}",
|
||
details={
|
||
"checkpoint_id": checkpoint_id,
|
||
"total_rows": total_rows,
|
||
"table_counts": table_counts
|
||
}
|
||
)
|
||
|
||
return {
|
||
"checkpoint_id": checkpoint_id,
|
||
"timestamp": timestamp,
|
||
"description": description,
|
||
"total_rows": total_rows,
|
||
"table_counts": table_counts
|
||
}
|
||
|
||
|
||
def list_checkpoints() -> List[Dict[str, Any]]:
|
||
"""List all available checkpoints."""
|
||
checkpoints_dir = _get_checkpoints_dir()
|
||
checkpoints = []
|
||
|
||
if not os.path.exists(checkpoints_dir):
|
||
return checkpoints
|
||
|
||
for filename in os.listdir(checkpoints_dir):
|
||
if filename.endswith(".json"):
|
||
filepath = os.path.join(checkpoints_dir, filename)
|
||
try:
|
||
with open(filepath, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
checkpoints.append({
|
||
"checkpoint_id": data["checkpoint_id"],
|
||
"timestamp": data["timestamp"],
|
||
"description": data.get("description", ""),
|
||
"total_rows": data.get("total_rows", 0),
|
||
"table_counts": data.get("table_counts", {}),
|
||
"filename": filename
|
||
})
|
||
except Exception as e:
|
||
print(f"Error reading checkpoint {filename}: {e}")
|
||
|
||
return sorted(checkpoints, key=lambda x: x["timestamp"], reverse=True)
|
||
|
||
|
||
def restore_checkpoint(
|
||
checkpoint_id: str,
|
||
create_auto_backup: bool = True,
|
||
batch_size: int = 1000,
|
||
operator: str = "admin",
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
安全恢复数据库从checkpoint。
|
||
|
||
⚠️ 危险操作: 会覆盖现有数据!
|
||
|
||
Args:
|
||
checkpoint_id: 要恢复的checkpoint ID
|
||
create_auto_backup: 是否在恢复前自动备份当前状态
|
||
batch_size: 批量插入的批次大小 (默认1000行/批)
|
||
|
||
Returns:
|
||
包含恢复结果的字典
|
||
"""
|
||
checkpoints_dir = _get_checkpoints_dir()
|
||
checkpoint_file = os.path.join(checkpoints_dir, f"{checkpoint_id}.json")
|
||
|
||
logger.warning("=" * 80)
|
||
logger.warning(f"[CHECKPOINT] WARNING: Starting restore operation: {checkpoint_id}")
|
||
logger.warning("[CHECKPOINT] This will OVERWRITE all existing data in the database!")
|
||
logger.warning("=" * 80)
|
||
|
||
if not os.path.exists(checkpoint_file):
|
||
error_msg = f"Checkpoint {checkpoint_id} not found"
|
||
logger.error(f"[CHECKPOINT] {error_msg}")
|
||
raise ValueError(error_msg)
|
||
|
||
with open(checkpoint_file, "r", encoding="utf-8") as f:
|
||
checkpoint_data = json.load(f)
|
||
|
||
# 自动备份当前状态（可选但推荐）
|
||
auto_backup_info = None
|
||
if create_auto_backup:
|
||
logger.info("[CHECKPOINT] Creating auto-backup before restore... (THIS MAY TAKE TIME)")
|
||
try:
|
||
import time
|
||
start_time = time.time()
|
||
auto_backup_info = create_checkpoint(f"auto_backup_before_restore_{checkpoint_id}")
|
||
elapsed = time.time() - start_time
|
||
logger.info(f"[CHECKPOINT] Auto-backup created in {elapsed:.2f}s: {auto_backup_info['checkpoint_id']}")
|
||
logger.info(f"[CHECKPOINT] Auto-backup contains {auto_backup_info['total_rows']} rows")
|
||
except Exception as e:
|
||
logger.error(f"[CHECKPOINT] WARNING: Failed to create auto-backup: {e}")
|
||
logger.warning("[CHECKPOINT] Continuing without auto-backup...")
|
||
else:
|
||
logger.info("[CHECKPOINT] Auto-backup DISABLED by user")
|
||
|
||
tables = checkpoint_data.get("tables", {})
|
||
total_rows_in_checkpoint = sum(len(data) for data in tables.values())
|
||
|
||
logger.info("=" * 80)
|
||
logger.info(f"[CHECKPOINT] Checkpoint details:")
|
||
logger.info(f"[CHECKPOINT] ID: {checkpoint_id}")
|
||
logger.info(f"[CHECKPOINT] Tables: {len(tables)}")
|
||
logger.info(f"[CHECKPOINT] Total rows: {total_rows_in_checkpoint}")
|
||
logger.info(f"[CHECKPOINT] Auto-backup: {'Yes (' + auto_backup_info['checkpoint_id'] + ')' if auto_backup_info else 'No'}")
|
||
logger.info("=" * 80)
|
||
|
||
restore_summary = {
|
||
"checkpoint_id": checkpoint_id,
|
||
"auto_backup": auto_backup_info.get("checkpoint_id") if auto_backup_info else None,
|
||
"tables_restored": 0,
|
||
"total_rows_restored": 0,
|
||
"table_details": {},
|
||
"errors": []
|
||
}
|
||
|
||
with _lic_pg_conn(autocommit=False) as conn:
|
||
try:
|
||
# 1. 构建表依赖关系图
|
||
logger.info("[CHECKPOINT] Building table dependency graph...")
|
||
dependencies = _get_table_dependencies(conn)
|
||
all_tables = list(tables.keys())
|
||
logger.info(f"[CHECKPOINT] Found {len(dependencies)} table dependencies")
|
||
|
||
# 2. 拓扑排序获取恢复顺序
|
||
logger.info("[CHECKPOINT] Calculating restore order...")
|
||
restore_order = _topological_sort_tables(all_tables, dependencies)
|
||
logger.info(f"[CHECKPOINT] Restore order: {' -> '.join(restore_order)}")
|
||
|
||
# 3. 锁定所有表（防止并发写入）
|
||
logger.info("[CHECKPOINT] Acquiring exclusive locks on all tables...")
|
||
cur = conn.cursor()
|
||
for table in restore_order:
|
||
cur.execute(f"LOCK TABLE {table} IN EXCLUSIVE MODE")
|
||
logger.info("[CHECKPOINT] All tables locked exclusively")
|
||
|
||
# 4. 按依赖顺序恢复表
|
||
logger.info("=" * 80)
|
||
logger.info("[CHECKPOINT] Starting restore process...")
|
||
logger.info("=" * 80)
|
||
|
||
import time
|
||
restore_start_time = time.time()
|
||
|
||
for i, table_name in enumerate(restore_order, 1):
|
||
data = tables.get(table_name, [])
|
||
table_start_time = time.time()
|
||
logger.info(f"[CHECKPOINT] [{i}/{len(restore_order)}] Preparing to restore table: {table_name}")
|
||
|
||
try:
|
||
rows_restored = _restore_table(conn, table_name, data, batch_size=batch_size)
|
||
table_elapsed = time.time() - table_start_time
|
||
restore_summary["tables_restored"] += 1
|
||
restore_summary["total_rows_restored"] += rows_restored
|
||
restore_summary["table_details"][table_name] = rows_restored
|
||
logger.info(f"[CHECKPOINT] [{i}/{len(restore_order)}] Table {table_name} restored: {rows_restored} rows in {table_elapsed:.2f}s")
|
||
except Exception as e:
|
||
error_msg = f"Failed to restore table {table_name}: {str(e)}"
|
||
logger.error(f"[CHECKPOINT] ERROR: {error_msg}")
|
||
restore_summary["errors"].append(error_msg)
|
||
raise e
|
||
|
||
restore_elapsed = time.time() - restore_start_time
|
||
total_tables = len(restore_order)
|
||
logger.info(f"[CHECKPOINT] All {total_tables} tables restored in {restore_elapsed:.2f}s")
|
||
|
||
restore_elapsed = time.time() - restore_start_time
|
||
total_tables = len(restore_order)
|
||
logger.info(f"[CHECKPOINT] All {total_tables} tables restored in {restore_elapsed:.2f}s")
|
||
|
||
# 5. 重置所有表的序列 (Sequences)
|
||
logger.info("[CHECKPOINT] Resetting sequences for all tables...")
|
||
_reset_all_sequences(conn, all_tables)
|
||
logger.info("[CHECKPOINT] Sequences reset successfully")
|
||
|
||
# 6. 提交事务
|
||
logger.info("=" * 80)
|
||
logger.info("[CHECKPOINT] All tables restored successfully, committing transaction...")
|
||
conn.commit()
|
||
logger.info("[CHECKPOINT] Transaction committed successfully")
|
||
logger.info("=" * 80)
|
||
|
||
logger.warning("=" * 80)
|
||
|
||
# Log the operation
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="RESTORE",
|
||
target_type="DATABASE",
|
||
target_id=checkpoint_id,
|
||
target_name=checkpoint_id,
|
||
change_summary=f"Restored database from checkpoint: {checkpoint_id}",
|
||
details=restore_summary
|
||
)
|
||
|
||
return {
|
||
"status": "success",
|
||
"message": f"Successfully restored {restore_summary['tables_restored']} tables, "
|
||
f"{restore_summary['total_rows_restored']} total rows",
|
||
"summary": restore_summary
|
||
}
|
||
|
||
except Exception as e:
|
||
# 回滚事务
|
||
logger.error(f"[CHECKPOINT] ERROR during restore, rolling back: {str(e)}")
|
||
conn.rollback()
|
||
logger.warning("[CHECKPOINT] Transaction rolled back, changes reverted")
|
||
|
||
error_info = {
|
||
"status": "error",
|
||
"message": f"Restore failed: {str(e)}",
|
||
"summary": restore_summary,
|
||
"auto_backup_available": bool(auto_backup_info)
|
||
}
|
||
# 如果有自动备份，提供恢复建议
|
||
if auto_backup_info:
|
||
error_info["recovery_suggestion"] = (
|
||
f"Use auto-backup to restore current state: "
|
||
f"restore_checkpoint('{auto_backup_info['checkpoint_id']}')"
|
||
)
|
||
logger.warning(f"[CHECKPOINT] Recovery suggestion: Use auto-backup '{auto_backup_info['checkpoint_id']}'")
|
||
|
||
logger.error("=" * 80)
|
||
logger.error(f"[CHECKPOINT] RESTORE FAILED: {str(e)}")
|
||
if auto_backup_info:
|
||
logger.error(f"[CHECKPOINT] You can restore from auto-backup: {auto_backup_info['checkpoint_id']}")
|
||
logger.error("=" * 80)
|
||
|
||
return error_info
|
||
|
||
|
||
def delete_checkpoint(checkpoint_id: str) -> bool:
|
||
"""Delete a checkpoint file."""
|
||
checkpoints_dir = _get_checkpoints_dir()
|
||
checkpoint_file = os.path.join(checkpoints_dir, f"{checkpoint_id}.json")
|
||
|
||
if os.path.exists(checkpoint_file):
|
||
os.remove(checkpoint_file)
|
||
return True
|
||
return False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Permit risk snapshot helpers (region_permit_risk_vw)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _convert_snapshot_value(value: Any) -> Any:
|
||
"""Convert database values into JSON-serialisable primitives."""
|
||
if isinstance(value, (datetime, date)):
|
||
return value.isoformat()
|
||
if isinstance(value, Decimal):
|
||
return float(value)
|
||
if isinstance(value, uuid.UUID):
|
||
return str(value)
|
||
if isinstance(value, memoryview):
|
||
return bytes(value).decode("utf-8", errors="replace")
|
||
if isinstance(value, (bytes, bytearray)):
|
||
return value.decode("utf-8", errors="replace")
|
||
if isinstance(value, (list, tuple)):
|
||
return [_convert_snapshot_value(v) for v in value]
|
||
return value
|
||
|
||
|
||
def _normalize_snapshot_payload(record: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""Return a JSON-safe copy of the permit risk view record."""
|
||
return {key: _convert_snapshot_value(val) for key, val in record.items()}
|
||
|
||
|
||
def _fetch_permit_risk_row(
|
||
conn: pg.Connection, region_id: str, permit_id: str, risk_id: str
|
||
) -> Dict[str, Any]:
|
||
"""Fetch a single row from the consolidation view."""
|
||
if _permit_sources_available(conn):
|
||
sql = """
|
||
SELECT
|
||
vw.region_id,
|
||
vw.region_name,
|
||
vw.permit_id,
|
||
vw.permit_name,
|
||
vw.risk_id,
|
||
vw.risk_content,
|
||
vw.legal_basis,
|
||
vw.document_no,
|
||
vw.summary,
|
||
vw.theme_ids,
|
||
vw.theme_names,
|
||
vw.scope_ids,
|
||
vw.scope_descriptions,
|
||
vw.subitem_ids,
|
||
vw.permit_status,
|
||
vw.subitem_summary,
|
||
vw.responsible_contact,
|
||
vw.jurisdiction_scope,
|
||
vw.permit_detail_updated_at,
|
||
vw.permit_risk_key,
|
||
ps.source_type AS permit_source_type,
|
||
ps.source_name AS permit_source_name,
|
||
ps.source_detail AS permit_source_detail,
|
||
ps.updated_at AS permit_source_updated_at,
|
||
rpd.contact_info
|
||
FROM region_permit_risk_vw vw
|
||
LEFT JOIN permit_sources ps
|
||
ON ps.region_id = vw.region_id
|
||
AND ps.permit_id = vw.permit_id
|
||
LEFT JOIN region_permit_details rpd
|
||
ON rpd.region_id = vw.region_id
|
||
AND rpd.permit_id = vw.permit_id
|
||
WHERE vw.region_id = %s AND vw.permit_id = %s AND vw.risk_id = %s
|
||
LIMIT 1
|
||
"""
|
||
else:
|
||
sql = """
|
||
SELECT
|
||
vw.region_id,
|
||
vw.region_name,
|
||
vw.permit_id,
|
||
vw.permit_name,
|
||
vw.risk_id,
|
||
vw.risk_content,
|
||
vw.legal_basis,
|
||
vw.document_no,
|
||
vw.summary,
|
||
vw.theme_ids,
|
||
vw.theme_names,
|
||
vw.scope_ids,
|
||
vw.scope_descriptions,
|
||
vw.subitem_ids,
|
||
vw.permit_status,
|
||
vw.subitem_summary,
|
||
vw.responsible_contact,
|
||
vw.jurisdiction_scope,
|
||
vw.permit_detail_updated_at,
|
||
vw.permit_risk_key,
|
||
rpd.contact_info
|
||
FROM region_permit_risk_vw vw
|
||
LEFT JOIN region_permit_details rpd
|
||
ON rpd.region_id = vw.region_id
|
||
AND rpd.permit_id = vw.permit_id
|
||
WHERE vw.region_id = %s AND vw.permit_id = %s AND vw.risk_id = %s
|
||
LIMIT 1
|
||
"""
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (region_id, permit_id, risk_id))
|
||
row = cur.fetchone()
|
||
if not row:
|
||
raise ValueError("Permit risk combination not found in consolidation view")
|
||
columns = [desc[0] for desc in cur.description]
|
||
return {columns[i]: row[i] for i in range(len(columns))}
|
||
|
||
|
||
def _insert_permit_risk_snapshot(
|
||
conn: pg.Connection,
|
||
payload: Dict[str, Any],
|
||
*,
|
||
edited_by: Optional[str],
|
||
change_summary: Optional[str],
|
||
batch_id: str,
|
||
) -> Dict[str, Any]:
|
||
"""Insert a snapshot row and return metadata."""
|
||
permit_risk_key = str(payload["permit_risk_key"])
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT version
|
||
FROM permit_risk_snapshots
|
||
WHERE permit_risk_key = %s
|
||
ORDER BY version DESC
|
||
LIMIT 1
|
||
FOR UPDATE
|
||
""",
|
||
(permit_risk_key,),
|
||
)
|
||
row = cur.fetchone()
|
||
next_version = (int(row[0]) + 1) if row else 1
|
||
|
||
payload_with_batch = dict(payload)
|
||
payload_with_batch["snapshot_batch_id"] = batch_id
|
||
payload_json = json.dumps(payload_with_batch, ensure_ascii=False)
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO permit_risk_snapshots (
|
||
region_id,
|
||
permit_id,
|
||
risk_id,
|
||
permit_risk_key,
|
||
version,
|
||
payload,
|
||
edited_by,
|
||
change_summary
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||
RETURNING snapshot_id, created_at
|
||
""",
|
||
(
|
||
payload["region_id"],
|
||
payload["permit_id"],
|
||
payload["risk_id"],
|
||
permit_risk_key,
|
||
next_version,
|
||
payload_json,
|
||
edited_by,
|
||
change_summary,
|
||
),
|
||
)
|
||
snapshot_id, created_at = cur.fetchone()
|
||
|
||
return {
|
||
"snapshot_id": str(snapshot_id),
|
||
"region_id": payload["region_id"],
|
||
"permit_id": payload["permit_id"],
|
||
"risk_id": payload["risk_id"],
|
||
"permit_risk_key": permit_risk_key,
|
||
"version": next_version,
|
||
"created_at": _convert_snapshot_value(created_at),
|
||
"edited_by": edited_by,
|
||
"change_summary": change_summary or "",
|
||
"snapshot_batch_id": batch_id,
|
||
"payload": payload_with_batch,
|
||
}
|
||
|
||
|
||
def _create_snapshot_with_connection(
|
||
conn: pg.Connection,
|
||
region_id: str,
|
||
permit_id: str,
|
||
risk_id: str,
|
||
*,
|
||
edited_by: Optional[str],
|
||
change_summary: Optional[str],
|
||
batch_id: Optional[str] = None,
|
||
) -> Dict[str, Any]:
|
||
"""Create a snapshot using an existing DB connection (no commit)."""
|
||
view_record = _fetch_permit_risk_row(conn, region_id, permit_id, risk_id)
|
||
payload = _normalize_snapshot_payload(view_record)
|
||
try:
|
||
payload["binds_all_themes"] = _permit_binds_all_themes(conn, region_id, permit_id)
|
||
except Exception:
|
||
payload["binds_all_themes"] = False
|
||
snapshot_batch_id = batch_id or str(uuid.uuid4())
|
||
metadata = _insert_permit_risk_snapshot(
|
||
conn,
|
||
payload,
|
||
edited_by=edited_by,
|
||
change_summary=change_summary,
|
||
batch_id=snapshot_batch_id,
|
||
)
|
||
theme_names = payload.get("theme_names") or []
|
||
scopes = payload.get("scope_ids") or []
|
||
subitems = payload.get("subitem_ids") or []
|
||
permit_status = payload.get("permit_status") or ""
|
||
preview_text = str(payload.get("risk_content") or "").strip()
|
||
preview_flat = re.sub(r"\s+", " ", preview_text)
|
||
if len(preview_flat) > 120:
|
||
preview_flat = f"{preview_flat[:117]}..."
|
||
logger.info(
|
||
"[CHECKPOINT] Snapshot created: %s version %s",
|
||
metadata["permit_risk_key"],
|
||
metadata["version"],
|
||
)
|
||
logger.info(
|
||
"[CHECKPOINT] Snapshot context: region=%s(%s) permit=%s(%s) risk=%s | themes=%s | scopes=%d | subitems=%d | status=%s",
|
||
payload.get("region_id"),
|
||
payload.get("region_name"),
|
||
payload.get("permit_id"),
|
||
payload.get("permit_name"),
|
||
payload.get("risk_id"),
|
||
"、".join(str(name) for name in theme_names) if theme_names else "—",
|
||
len(scopes),
|
||
len(subitems),
|
||
permit_status or "—",
|
||
)
|
||
source_name = payload.get("permit_source_name")
|
||
if source_name:
|
||
logger.info(
|
||
"[CHECKPOINT] Snapshot permit source: %s",
|
||
source_name,
|
||
)
|
||
if preview_flat:
|
||
logger.info(
|
||
"[CHECKPOINT] Snapshot risk preview: %s",
|
||
preview_flat,
|
||
)
|
||
return metadata
|
||
|
||
|
||
def create_permit_risk_snapshot(
|
||
region_id: str,
|
||
permit_id: str,
|
||
risk_id: str,
|
||
*,
|
||
edited_by: Optional[str] = None,
|
||
change_summary: Optional[str] = None,
|
||
batch_id: Optional[str] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Capture the current state of a region/permit/risk record as a versioned snapshot.
|
||
|
||
Returns metadata about the created snapshot including version number.
|
||
"""
|
||
with _lic_pg_conn(autocommit=False) as conn:
|
||
try:
|
||
snapshot_meta = _create_snapshot_with_connection(
|
||
conn,
|
||
region_id,
|
||
permit_id,
|
||
risk_id,
|
||
edited_by=edited_by,
|
||
change_summary=change_summary,
|
||
batch_id=batch_id,
|
||
)
|
||
conn.commit()
|
||
return snapshot_meta
|
||
except Exception:
|
||
conn.rollback()
|
||
raise
|
||
|
||
|
||
def list_permit_risk_snapshots(
|
||
region_id: Optional[str] = None,
|
||
permit_id: Optional[str] = None,
|
||
risk_id: Optional[str] = None,
|
||
*,
|
||
permit_risk_key: Optional[str] = None,
|
||
limit: int = 20,
|
||
offset: int = 0,
|
||
) -> List[Dict[str, Any]]:
|
||
"""
|
||
List snapshots for a region/permit/risk combination ordered by version descending.
|
||
|
||
At least one identifier (permit_risk_key or region/permit/risk) must be provided.
|
||
"""
|
||
filters: List[str] = []
|
||
params: List[Any] = []
|
||
|
||
if permit_risk_key:
|
||
filters.append("permit_risk_key = %s")
|
||
params.append(permit_risk_key)
|
||
else:
|
||
if region_id:
|
||
filters.append("region_id = %s")
|
||
params.append(region_id)
|
||
if permit_id:
|
||
filters.append("permit_id = %s")
|
||
params.append(permit_id)
|
||
if risk_id:
|
||
filters.append("risk_id = %s")
|
||
params.append(risk_id)
|
||
|
||
if not filters:
|
||
raise ValueError("At least one identifier must be provided to list snapshots")
|
||
|
||
filters_clause = " AND ".join(filters)
|
||
sql = f"""
|
||
SELECT snapshot_id, version, permit_risk_key, edited_by, change_summary, created_at
|
||
FROM permit_risk_snapshots
|
||
WHERE {filters_clause}
|
||
ORDER BY version DESC
|
||
LIMIT %s OFFSET %s
|
||
"""
|
||
params.extend([limit, offset])
|
||
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, tuple(params))
|
||
rows = cur.fetchall()
|
||
|
||
snapshots: List[Dict[str, Any]] = []
|
||
for snapshot_id, version, key, editor, summary, created_at in rows:
|
||
snapshots.append(
|
||
{
|
||
"snapshot_id": str(snapshot_id),
|
||
"permit_risk_key": key,
|
||
"version": int(version),
|
||
"created_at": _convert_snapshot_value(created_at),
|
||
"edited_by": editor,
|
||
"change_summary": summary or "",
|
||
}
|
||
)
|
||
return snapshots
|
||
|
||
|
||
def get_permit_risk_snapshot(snapshot_id: str) -> Optional[Dict[str, Any]]:
|
||
"""Fetch a snapshot payload by its identifier."""
|
||
sql = """
|
||
SELECT
|
||
snapshot_id,
|
||
region_id,
|
||
permit_id,
|
||
risk_id,
|
||
permit_risk_key,
|
||
version,
|
||
payload,
|
||
edited_by,
|
||
change_summary,
|
||
created_at
|
||
FROM permit_risk_snapshots
|
||
WHERE snapshot_id = %s
|
||
"""
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (snapshot_id,))
|
||
row = cur.fetchone()
|
||
|
||
if not row:
|
||
return None
|
||
|
||
(
|
||
snap_id,
|
||
region_id,
|
||
permit_id,
|
||
risk_id,
|
||
permit_risk_key,
|
||
version,
|
||
payload,
|
||
edited_by,
|
||
change_summary,
|
||
created_at,
|
||
) = row
|
||
|
||
if isinstance(payload, (bytes, bytearray, memoryview)):
|
||
payload_obj = json.loads(payload)
|
||
else:
|
||
payload_obj = payload if isinstance(payload, dict) else json.loads(payload)
|
||
|
||
return {
|
||
"snapshot_id": str(snap_id),
|
||
"region_id": str(region_id),
|
||
"permit_id": str(permit_id),
|
||
"risk_id": str(risk_id),
|
||
"permit_risk_key": permit_risk_key,
|
||
"version": int(version),
|
||
"created_at": _convert_snapshot_value(created_at),
|
||
"edited_by": edited_by,
|
||
"change_summary": change_summary or "",
|
||
"payload": payload_obj,
|
||
}
|
||
|
||
|
||
def list_permit_risk_snapshot_summaries(
|
||
*,
|
||
region_id: Optional[str] = None,
|
||
permit_id: Optional[str] = None,
|
||
edited_by: Optional[str] = None,
|
||
limit: int = 20,
|
||
offset: int = 0,
|
||
) -> List[Dict[str, Any]]:
|
||
"""
|
||
Return snapshot summaries for checkpoint history views.
|
||
Groups results by snapshot_batch_id to return one entry per batch operation.
|
||
"""
|
||
filters: List[str] = []
|
||
params: List[Any] = []
|
||
|
||
if region_id:
|
||
filters.append("region_id = %s")
|
||
params.append(region_id)
|
||
if permit_id:
|
||
filters.append("permit_id = %s")
|
||
params.append(permit_id)
|
||
if edited_by:
|
||
filters.append("edited_by = %s")
|
||
params.append(edited_by)
|
||
|
||
where_clause = f"WHERE {' AND '.join(filters)}" if filters else ""
|
||
|
||
# Group by batch_id so that we show one history entry per operation (which may contain multiple risk items).
|
||
# If snapshot_batch_id is missing (legacy data), fall back to snapshot_id to treat each as unique.
|
||
sql = f"""
|
||
SELECT
|
||
MAX(snapshot_id::text) as snapshot_id,
|
||
MAX(region_id::text) as region_id,
|
||
MAX(permit_id::text) as permit_id,
|
||
MAX(risk_id::text) as risk_id,
|
||
MAX(permit_risk_key) as permit_risk_key,
|
||
MAX(version) as version,
|
||
MAX(edited_by) as edited_by,
|
||
MAX(change_summary) as change_summary,
|
||
MAX(created_at) as created_at,
|
||
|
||
MAX(payload ->> 'region_name') AS region_name,
|
||
MAX(payload ->> 'permit_name') AS permit_name,
|
||
MAX(payload ->> 'risk_content') AS risk_content,
|
||
MAX(payload ->> 'legal_basis') AS legal_basis,
|
||
MAX(payload ->> 'document_no') AS document_no,
|
||
MAX(payload ->> 'permit_status') AS permit_status,
|
||
COALESCE(MAX(payload ->> 'snapshot_batch_id'), MAX(snapshot_id::text)) AS snapshot_batch_id,
|
||
MAX(payload ->> 'permit_source_name') AS permit_source_name,
|
||
MAX(payload ->> 'permit_source_type') AS permit_source_type,
|
||
COUNT(*) as item_count
|
||
FROM permit_risk_snapshots
|
||
{where_clause}
|
||
GROUP BY COALESCE(payload ->> 'snapshot_batch_id', snapshot_id::text)
|
||
ORDER BY MAX(created_at) DESC
|
||
LIMIT %s OFFSET %s
|
||
"""
|
||
params.extend([limit, offset])
|
||
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, tuple(params))
|
||
rows = cur.fetchall()
|
||
|
||
summaries: List[Dict[str, Any]] = []
|
||
for (
|
||
snapshot_id,
|
||
region_uuid,
|
||
permit_uuid,
|
||
risk_uuid,
|
||
permit_risk_key,
|
||
version,
|
||
editor,
|
||
summary_text,
|
||
created_at,
|
||
region_name,
|
||
permit_name,
|
||
risk_content,
|
||
legal_basis,
|
||
document_no,
|
||
permit_status,
|
||
snapshot_batch_id,
|
||
permit_source_name,
|
||
permit_source_type,
|
||
item_count,
|
||
) in rows:
|
||
summaries.append(
|
||
{
|
||
"snapshot_id": str(snapshot_id),
|
||
"region_id": str(region_uuid),
|
||
"permit_id": str(permit_uuid),
|
||
"risk_id": str(risk_uuid),
|
||
"permit_risk_key": permit_risk_key,
|
||
"version": int(version) if version else 0,
|
||
"created_at": _convert_snapshot_value(created_at),
|
||
"edited_by": editor,
|
||
"change_summary": summary_text or "",
|
||
"region_name": region_name or "",
|
||
"permit_name": permit_name or "",
|
||
"risk_content": risk_content or "",
|
||
"legal_basis": legal_basis or "",
|
||
"document_no": document_no or "",
|
||
"permit_status": permit_status or "",
|
||
"snapshot_batch_id": snapshot_batch_id or "",
|
||
"permit_source_name": permit_source_name or "",
|
||
"permit_source_type": permit_source_type or "",
|
||
"item_count": int(item_count),
|
||
}
|
||
)
|
||
return summaries
|
||
|
||
|
||
def count_permit_risk_snapshots(
|
||
*,
|
||
region_id: Optional[str] = None,
|
||
permit_id: Optional[str] = None,
|
||
edited_by: Optional[str] = None,
|
||
) -> int:
|
||
"""Return total snapshot batches matching the optional filters."""
|
||
filters: List[str] = []
|
||
params: List[Any] = []
|
||
|
||
if region_id:
|
||
filters.append("region_id = %s")
|
||
params.append(region_id)
|
||
if permit_id:
|
||
filters.append("permit_id = %s")
|
||
params.append(permit_id)
|
||
if edited_by:
|
||
filters.append("edited_by = %s")
|
||
params.append(edited_by)
|
||
|
||
where_clause = f"WHERE {' AND '.join(filters)}" if filters else ""
|
||
# Count distinct batches (using same COALESCE logic as list query)
|
||
sql = f"SELECT COUNT(DISTINCT COALESCE(payload ->> 'snapshot_batch_id', snapshot_id::text)) FROM permit_risk_snapshots {where_clause}"
|
||
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, tuple(params))
|
||
row = cur.fetchone()
|
||
|
||
return int(row[0]) if row else 0
|
||
|
||
|
||
def update_permit_risk_record(
|
||
region_id: str,
|
||
permit_id: str,
|
||
risk_id: str,
|
||
*,
|
||
risk_content: Any = _UNSET,
|
||
legal_basis: Any = _UNSET,
|
||
document_no: Any = _UNSET,
|
||
summary: Any = _UNSET,
|
||
permit_status: Any = _UNSET,
|
||
subitem_summary: Any = _UNSET,
|
||
responsible_contact: Any = _UNSET,
|
||
jurisdiction_scope: Any = _UNSET,
|
||
contact_info: Any = _UNSET,
|
||
edited_by: Optional[str] = None,
|
||
change_summary: Optional[str] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Update the permit risk record while capturing a checkpoint snapshot beforehand.
|
||
|
||
Returns the snapshot metadata (pre-change) and the refreshed view row (post-change).
|
||
"""
|
||
update_flags = [
|
||
risk_content,
|
||
legal_basis,
|
||
document_no,
|
||
summary,
|
||
permit_status,
|
||
subitem_summary,
|
||
responsible_contact,
|
||
jurisdiction_scope,
|
||
contact_info,
|
||
]
|
||
if all(flag is _UNSET for flag in update_flags):
|
||
raise ValueError("No fields provided to update.")
|
||
|
||
with _lic_pg_conn(autocommit=False) as conn:
|
||
try:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT 1
|
||
FROM region_permit_risks
|
||
WHERE region_id = %s AND permit_id = %s AND risk_id = %s
|
||
FOR UPDATE
|
||
""",
|
||
(region_id, permit_id, risk_id),
|
||
)
|
||
if cur.fetchone() is None:
|
||
raise ValueError("Permit risk combination not found.")
|
||
|
||
snapshot_meta = _create_snapshot_with_connection(
|
||
conn,
|
||
region_id,
|
||
permit_id,
|
||
risk_id,
|
||
edited_by=edited_by,
|
||
change_summary=change_summary,
|
||
)
|
||
|
||
risk_updates: List[str] = []
|
||
risk_params: List[Any] = []
|
||
risk_fields = (
|
||
("risk_content", risk_content),
|
||
("legal_basis", legal_basis),
|
||
("document_no", document_no),
|
||
("summary", summary),
|
||
)
|
||
for column, value in risk_fields:
|
||
if value is not _UNSET:
|
||
risk_updates.append(f"{column} = %s")
|
||
risk_params.append(value)
|
||
|
||
if risk_updates:
|
||
risk_params.append(risk_id)
|
||
cur.execute(
|
||
f"UPDATE risks SET {', '.join(risk_updates)} WHERE id = %s",
|
||
tuple(risk_params),
|
||
)
|
||
|
||
detail_columns: List[str] = []
|
||
detail_values: List[Any] = []
|
||
detail_fields = (
|
||
("permit_status", permit_status),
|
||
("subitem_summary", subitem_summary),
|
||
("responsible_contact", responsible_contact),
|
||
("jurisdiction_scope", jurisdiction_scope),
|
||
("contact_info", contact_info),
|
||
)
|
||
for column, value in detail_fields:
|
||
if value is not _UNSET:
|
||
detail_columns.append(column)
|
||
detail_values.append(value)
|
||
|
||
details_updated = False
|
||
if detail_columns:
|
||
insert_cols = ", ".join(["region_id", "permit_id"] + detail_columns)
|
||
insert_placeholders = ", ".join(["%s"] * (2 + len(detail_values)))
|
||
update_assignments = ", ".join(
|
||
[f"{col} = EXCLUDED.{col}" for col in detail_columns]
|
||
)
|
||
_ensure_contact_info_column(conn)
|
||
sql = f"""
|
||
INSERT INTO region_permit_details ({insert_cols})
|
||
VALUES ({insert_placeholders})
|
||
ON CONFLICT (region_id, permit_id)
|
||
DO UPDATE SET
|
||
{update_assignments},
|
||
updated_at = now()
|
||
"""
|
||
cur.execute(
|
||
sql,
|
||
(region_id, permit_id, *detail_values),
|
||
)
|
||
details_updated = True
|
||
|
||
updated_record = _normalize_snapshot_payload(
|
||
_fetch_permit_risk_row(conn, region_id, permit_id, risk_id)
|
||
)
|
||
conn.commit()
|
||
|
||
logger.info(
|
||
"[CHECKPOINT] Permit risk updated: %s version %s -> new snapshot ready",
|
||
snapshot_meta["permit_risk_key"],
|
||
snapshot_meta["version"],
|
||
)
|
||
|
||
return {
|
||
"snapshot": snapshot_meta,
|
||
"current": updated_record,
|
||
"risk_updated": bool(risk_updates),
|
||
"details_updated": details_updated,
|
||
}
|
||
except Exception:
|
||
conn.rollback()
|
||
raise
|
||
|
||
|
||
def delete_region_permit(
|
||
region_id: str,
|
||
theme_id: str,
|
||
permit_id: str,
|
||
*,
|
||
edited_by: Optional[str] = None,
|
||
change_summary: Optional[str] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
删除指定区划下的许可，同时为所有关联风险生成快照，并清理依赖关系。
|
||
|
||
返回删除摘要、快照列表以及主题剩余许可数量。
|
||
"""
|
||
with _lic_pg_conn(autocommit=False) as conn:
|
||
cur = conn.cursor()
|
||
try:
|
||
cur.execute(
|
||
"""
|
||
SELECT r.name, t.name, p.name
|
||
FROM region_theme_permits rtp
|
||
JOIN regions r ON r.id = rtp.region_id
|
||
JOIN themes t ON t.id = rtp.theme_id
|
||
JOIN permits p ON p.id = rtp.permit_id
|
||
WHERE rtp.region_id = %s
|
||
AND rtp.theme_id = %s
|
||
AND rtp.permit_id = %s
|
||
FOR UPDATE
|
||
""",
|
||
(region_id, theme_id, permit_id),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
raise ValueError("地区-主题-许可组合不存在，无法删除")
|
||
region_name, theme_name, permit_name = (str(val) for val in row)
|
||
|
||
cur.execute(
|
||
"""
|
||
SELECT risk_id
|
||
FROM region_permit_risks
|
||
WHERE region_id = %s
|
||
AND permit_id = %s
|
||
ORDER BY risk_id
|
||
FOR UPDATE
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
risk_ids = [str(risk_id) for (risk_id,) in cur.fetchall()]
|
||
|
||
snapshots: List[Dict[str, Any]] = []
|
||
total_snapshots = 0
|
||
summary_base = (change_summary or "").strip()
|
||
if not summary_base:
|
||
summary_base = f"删除许可 {permit_name}（地区：{region_name}）"
|
||
snapshot_batch_id = str(uuid.uuid4())
|
||
|
||
for idx, risk_id in enumerate(risk_ids, start=1):
|
||
detail_summary = summary_base
|
||
if len(risk_ids) > 1:
|
||
detail_summary = f"{summary_base} - 风险 {idx}/{len(risk_ids)}（ID：{risk_id}）"
|
||
snapshot_meta = _create_snapshot_with_connection(
|
||
conn,
|
||
region_id,
|
||
permit_id,
|
||
risk_id,
|
||
edited_by=edited_by,
|
||
change_summary=detail_summary,
|
||
batch_id=snapshot_batch_id,
|
||
)
|
||
snapshots.append(
|
||
{
|
||
"snapshot_id": snapshot_meta["snapshot_id"],
|
||
"permit_risk_key": snapshot_meta["permit_risk_key"],
|
||
"version": snapshot_meta["version"],
|
||
"risk_id": snapshot_meta["risk_id"],
|
||
"created_at": snapshot_meta["created_at"],
|
||
"snapshot_batch_id": snapshot_meta.get("snapshot_batch_id"),
|
||
"change_summary": snapshot_meta.get("change_summary", ""),
|
||
}
|
||
)
|
||
total_snapshots += 1
|
||
|
||
if total_snapshots:
|
||
logger.info(
|
||
"[PERMIT-DELETE] Captured %d snapshots before deleting permit %s (%s) in region %s (%s)",
|
||
total_snapshots,
|
||
permit_id,
|
||
permit_name,
|
||
region_id,
|
||
region_name,
|
||
)
|
||
else:
|
||
logger.info(
|
||
"[PERMIT-DELETE] No risk snapshots required for permit %s (%s) in region %s (%s)",
|
||
permit_id,
|
||
permit_name,
|
||
region_id,
|
||
region_name,
|
||
)
|
||
|
||
delete_counts: Dict[str, int] = {}
|
||
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM region_permit_risks
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
delete_counts["region_permit_risks"] = int(cur.rowcount or 0)
|
||
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM region_permit_subitems
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
delete_counts["region_permit_subitems"] = int(cur.rowcount or 0)
|
||
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM region_permit_scopes
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
delete_counts["region_permit_scopes"] = int(cur.rowcount or 0)
|
||
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM region_permit_details
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
delete_counts["region_permit_details"] = int(cur.rowcount or 0)
|
||
|
||
if _permit_sources_available(conn):
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM permit_sources
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
delete_counts["permit_sources"] = int(cur.rowcount or 0)
|
||
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM region_theme_permits
|
||
WHERE region_id = %s AND theme_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, theme_id, permit_id),
|
||
)
|
||
delete_counts["region_theme_permits"] = int(cur.rowcount or 0)
|
||
_clear_permit_bind_all_themes(conn, region_id, permit_id)
|
||
|
||
cur.execute(
|
||
"""
|
||
SELECT COUNT(*)
|
||
FROM region_theme_permits
|
||
WHERE region_id = %s AND theme_id = %s
|
||
""",
|
||
(region_id, theme_id),
|
||
)
|
||
remaining_theme_permits = int(cur.fetchone()[0] or 0)
|
||
|
||
theme_detached = False
|
||
if remaining_theme_permits == 0:
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM region_themes
|
||
WHERE region_id = %s AND theme_id = %s
|
||
""",
|
||
(region_id, theme_id),
|
||
)
|
||
theme_detached = cur.rowcount > 0
|
||
if theme_detached:
|
||
logger.info(
|
||
"[PERMIT-DELETE] Detached theme %s (%s) from region %s (%s) because no permits remain",
|
||
theme_id,
|
||
theme_name,
|
||
region_id,
|
||
region_name,
|
||
)
|
||
else:
|
||
logger.info(
|
||
"[PERMIT-DELETE] Theme %s (%s) retains %d permit(s) in region %s (%s); theme linkage preserved",
|
||
theme_id,
|
||
theme_name,
|
||
remaining_theme_permits,
|
||
region_id,
|
||
region_name,
|
||
)
|
||
|
||
conn.commit()
|
||
|
||
logger.info(
|
||
"[PERMIT-DELETE] Completed deletion for permit %s (%s) in region %s (%s): snapshots=%d, deleted_rows=%s",
|
||
permit_id,
|
||
permit_name,
|
||
region_id,
|
||
region_name,
|
||
total_snapshots,
|
||
delete_counts,
|
||
)
|
||
|
||
# Log the operation
|
||
log_operation(
|
||
operator=edited_by or "admin",
|
||
operation_type="DELETE",
|
||
target_type="PERMIT",
|
||
target_id=permit_id,
|
||
target_name=permit_name,
|
||
change_summary=f"Deleted permit {permit_name} in region {region_name}",
|
||
details={
|
||
"region_id": region_id,
|
||
"region_name": region_name,
|
||
"deleted_risk_count": delete_counts.get("region_permit_risks", 0),
|
||
"snapshot_batch_id": snapshot_batch_id
|
||
}
|
||
)
|
||
|
||
return {
|
||
"region_id": str(region_id),
|
||
"region_name": region_name,
|
||
"theme_id": str(theme_id),
|
||
"theme_name": theme_name,
|
||
"permit_id": str(permit_id),
|
||
"permit_name": permit_name,
|
||
"risk_ids": risk_ids,
|
||
"snapshot_count": total_snapshots,
|
||
"snapshots": snapshots,
|
||
"snapshot_batch_id": snapshot_batch_id if total_snapshots else "",
|
||
"deleted_rows": delete_counts,
|
||
"theme_detached": theme_detached,
|
||
"remaining_theme_permits": remaining_theme_permits,
|
||
}
|
||
except Exception:
|
||
conn.rollback()
|
||
raise
|
||
|
||
|
||
def restore_permit_risk_snapshot_batch(
|
||
snapshot_batch_id: str,
|
||
*,
|
||
edited_by: Optional[str] = None,
|
||
change_summary: Optional[str] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Restore region/permit/risk relations based on a snapshot batch (or single snapshot).
|
||
"""
|
||
if not snapshot_batch_id:
|
||
raise ValueError("快照批次 ID 不能为空")
|
||
|
||
with _lic_pg_conn(autocommit=False) as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"""
|
||
SELECT snapshot_id, payload, created_at, edited_by, change_summary
|
||
FROM permit_risk_snapshots
|
||
WHERE payload ->> 'snapshot_batch_id' = %s
|
||
ORDER BY created_at ASC, snapshot_id ASC
|
||
""",
|
||
(snapshot_batch_id,),
|
||
)
|
||
rows = cur.fetchall()
|
||
resolved_batch_id = snapshot_batch_id
|
||
|
||
if not rows:
|
||
cur.execute(
|
||
"""
|
||
SELECT snapshot_id, payload, created_at, edited_by, change_summary
|
||
FROM permit_risk_snapshots
|
||
WHERE snapshot_id::text = %s
|
||
""",
|
||
(snapshot_batch_id,),
|
||
)
|
||
rows = cur.fetchall()
|
||
|
||
if not rows:
|
||
raise ValueError("未找到对应的快照记录")
|
||
|
||
snapshots: List[Dict[str, Any]] = []
|
||
for snap_id, payload_raw, created_at, snap_editor, snap_summary in rows:
|
||
if isinstance(payload_raw, dict):
|
||
payload_obj = payload_raw
|
||
elif isinstance(payload_raw, (bytes, bytearray, memoryview)):
|
||
payload_obj = json.loads(bytes(payload_raw).decode("utf-8"))
|
||
elif isinstance(payload_raw, str):
|
||
payload_obj = json.loads(payload_raw)
|
||
else:
|
||
payload_obj = json.loads(payload_raw)
|
||
|
||
batch_token = payload_obj.get("snapshot_batch_id")
|
||
if batch_token:
|
||
resolved_batch_id = batch_token
|
||
|
||
snapshots.append(
|
||
{
|
||
"snapshot_id": str(snap_id),
|
||
"payload": payload_obj,
|
||
"created_at": _convert_snapshot_value(created_at),
|
||
"edited_by": snap_editor,
|
||
"change_summary": snap_summary or "",
|
||
}
|
||
)
|
||
|
||
payload0 = snapshots[0]["payload"]
|
||
region_id = str(payload0["region_id"])
|
||
permit_id = str(payload0["permit_id"])
|
||
region_name = str(payload0.get("region_name") or "")
|
||
permit_name = str(payload0.get("permit_name") or "")
|
||
|
||
theme_ids: Set[str] = set(str(t) for t in (payload0.get("theme_ids") or []) if t)
|
||
scope_ids: Set[str] = set()
|
||
subitem_ids: Set[str] = set()
|
||
binds_all_override = bool(payload0.get("binds_all_themes"))
|
||
|
||
for snap in snapshots:
|
||
payload = snap["payload"]
|
||
for tid in payload.get("theme_ids") or []:
|
||
if tid:
|
||
theme_ids.add(str(tid))
|
||
for scope in payload.get("scope_ids") or []:
|
||
if scope:
|
||
scope_ids.add(str(scope))
|
||
for subitem in payload.get("subitem_ids") or []:
|
||
if subitem:
|
||
subitem_ids.add(str(subitem))
|
||
|
||
detail_fields = {
|
||
"permit_status": payload0.get("permit_status"),
|
||
"subitem_summary": payload0.get("subitem_summary"),
|
||
"responsible_contact": payload0.get("responsible_contact"),
|
||
"responsible_contact": payload0.get("responsible_contact"),
|
||
"jurisdiction_scope": payload0.get("jurisdiction_scope"),
|
||
"contact_info": payload0.get("contact_info"),
|
||
}
|
||
|
||
source_name = _clean_text(payload0.get("permit_source_name"))
|
||
source_type = _clean_text(payload0.get("permit_source_type")) or "snapshot"
|
||
source_detail = payload0.get("permit_source_detail")
|
||
|
||
insert_counts = {
|
||
"region_themes": 0,
|
||
"region_theme_permits": 0,
|
||
"region_permit_details": 0,
|
||
"region_permit_scopes": 0,
|
||
"region_permit_subitems": 0,
|
||
"region_permit_risks": 0,
|
||
"risks_upserted": 0,
|
||
"permit_sources_synced": 0,
|
||
}
|
||
|
||
restored_risk_ids: Set[str] = set()
|
||
|
||
try:
|
||
for theme_id in sorted(theme_ids):
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_themes (region_id, theme_id)
|
||
VALUES (%s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, theme_id),
|
||
)
|
||
insert_counts["region_themes"] += cur.rowcount or 0
|
||
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_theme_permits (region_id, theme_id, permit_id)
|
||
VALUES (%s, %s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, theme_id, permit_id),
|
||
)
|
||
insert_counts["region_theme_permits"] += cur.rowcount or 0
|
||
|
||
if binds_all_override:
|
||
_set_permit_bind_all_themes(conn, region_id, permit_id, created_by=snapshots[0].get("edited_by"))
|
||
else:
|
||
_clear_permit_bind_all_themes(conn, region_id, permit_id)
|
||
|
||
_ensure_contact_info_column(conn)
|
||
cur = conn.cursor() # Re-acquire cursor if needed or just use existing
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_details (
|
||
region_id,
|
||
permit_id,
|
||
permit_status,
|
||
subitem_summary,
|
||
responsible_contact,
|
||
jurisdiction_scope,
|
||
contact_info
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||
ON CONFLICT (region_id, permit_id)
|
||
DO UPDATE SET
|
||
permit_status = EXCLUDED.permit_status,
|
||
subitem_summary = EXCLUDED.subitem_summary,
|
||
responsible_contact = EXCLUDED.responsible_contact,
|
||
jurisdiction_scope = EXCLUDED.jurisdiction_scope,
|
||
contact_info = EXCLUDED.contact_info,
|
||
updated_at = now()
|
||
""",
|
||
(
|
||
region_id,
|
||
permit_id,
|
||
detail_fields["permit_status"],
|
||
detail_fields["subitem_summary"],
|
||
detail_fields["responsible_contact"],
|
||
detail_fields["jurisdiction_scope"],
|
||
detail_fields["contact_info"],
|
||
),
|
||
)
|
||
insert_counts["region_permit_details"] += cur.rowcount or 0
|
||
|
||
for scope_id in sorted(scope_ids):
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_scopes (region_id, permit_id, scope_id)
|
||
VALUES (%s, %s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, permit_id, scope_id),
|
||
)
|
||
insert_counts["region_permit_scopes"] += cur.rowcount or 0
|
||
|
||
for subitem_id in sorted(subitem_ids):
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_subitems (region_id, permit_id, subitem_id)
|
||
VALUES (%s, %s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, permit_id, subitem_id),
|
||
)
|
||
insert_counts["region_permit_subitems"] += cur.rowcount or 0
|
||
|
||
for snap in snapshots:
|
||
payload = snap["payload"]
|
||
risk_id = str(payload["risk_id"])
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO risks (id, risk_content, legal_basis, document_no, summary)
|
||
VALUES (%s, %s, %s, %s, %s)
|
||
ON CONFLICT (id)
|
||
DO UPDATE SET
|
||
risk_content = EXCLUDED.risk_content,
|
||
legal_basis = EXCLUDED.legal_basis,
|
||
document_no = EXCLUDED.document_no,
|
||
summary = EXCLUDED.summary
|
||
""",
|
||
(
|
||
risk_id,
|
||
payload.get("risk_content"),
|
||
payload.get("legal_basis"),
|
||
payload.get("document_no"),
|
||
payload.get("summary"),
|
||
),
|
||
)
|
||
insert_counts["risks_upserted"] += cur.rowcount or 0
|
||
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO region_permit_risks (region_id, permit_id, risk_id)
|
||
VALUES (%s, %s, %s)
|
||
ON CONFLICT DO NOTHING
|
||
""",
|
||
(region_id, permit_id, risk_id),
|
||
)
|
||
if cur.rowcount:
|
||
insert_counts["region_permit_risks"] += cur.rowcount
|
||
restored_risk_ids.add(risk_id)
|
||
|
||
if source_name:
|
||
_ensure_permit_sources_table(conn)
|
||
if isinstance(source_detail, (dict, list)):
|
||
source_detail_text = json.dumps(source_detail, ensure_ascii=False)
|
||
else:
|
||
source_detail_text = _clean_text(source_detail) if source_detail is not None else None
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO permit_sources (
|
||
region_id,
|
||
permit_id,
|
||
source_type,
|
||
source_name,
|
||
source_detail,
|
||
created_at,
|
||
updated_at
|
||
)
|
||
VALUES (%s, %s, %s, %s, %s, now(), now())
|
||
ON CONFLICT (region_id, permit_id)
|
||
DO UPDATE SET
|
||
source_type = EXCLUDED.source_type,
|
||
source_name = EXCLUDED.source_name,
|
||
source_detail = EXCLUDED.source_detail,
|
||
updated_at = now()
|
||
""",
|
||
(
|
||
region_id,
|
||
permit_id,
|
||
source_type or "snapshot",
|
||
source_name,
|
||
source_detail_text,
|
||
),
|
||
)
|
||
insert_counts["permit_sources_synced"] += 1
|
||
elif _permit_sources_available(conn):
|
||
cur.execute(
|
||
"""
|
||
DELETE FROM permit_sources
|
||
WHERE region_id = %s AND permit_id = %s
|
||
""",
|
||
(region_id, permit_id),
|
||
)
|
||
if cur.rowcount:
|
||
insert_counts["permit_sources_synced"] += 1
|
||
|
||
conn.commit()
|
||
except Exception:
|
||
conn.rollback()
|
||
raise
|
||
|
||
logger.info(
|
||
"[PERMIT-RESTORE] Restored permit %s (%s) in region %s (%s) from snapshot batch %s: %d risk mappings",
|
||
permit_id,
|
||
permit_name,
|
||
region_id,
|
||
region_name,
|
||
resolved_batch_id,
|
||
len(restored_risk_ids),
|
||
)
|
||
|
||
return {
|
||
"snapshot_batch_id": resolved_batch_id,
|
||
"snapshot_ids": [snap["snapshot_id"] for snap in snapshots],
|
||
"restored_risk_count": len(restored_risk_ids),
|
||
"restored_risks": sorted(restored_risk_ids),
|
||
"region_id": region_id,
|
||
"region_name": region_name,
|
||
"permit_id": permit_id,
|
||
"permit_name": permit_name,
|
||
"applied_theme_ids": sorted(theme_ids),
|
||
"applied_scope_ids": sorted(scope_ids),
|
||
"applied_subitem_ids": sorted(subitem_ids),
|
||
"detail_fields": detail_fields,
|
||
"insert_counts": insert_counts,
|
||
"edited_by": edited_by,
|
||
"change_summary": change_summary or "",
|
||
}
|
||
|
||
|
||
def _expand_department_family(department_ids: List[str]) -> List[str]:
|
||
"""
|
||
Expand a list of department IDs to include their entire family (parents and children).
|
||
This enables 'same department' visibility across city (parent) and district (child) levels.
|
||
"""
|
||
if not department_ids:
|
||
return []
|
||
|
||
# Use a set to avoid duplicates
|
||
expanded_ids = set()
|
||
roots = set()
|
||
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
|
||
# 1. Find roots for the input departments
|
||
# We assume a 2-level hierarchy for now (City -> District) based on current seeds.
|
||
# If deeply nested, Recursive CTE would be better, but this suffices for current requirement.
|
||
placeholders = ','.join(['%s'] * len(department_ids))
|
||
sql_find_roots = f"SELECT id, parent_id FROM service_departments WHERE id IN ({placeholders})"
|
||
cur.execute(sql_find_roots, department_ids)
|
||
|
||
for dept_id, parent_id in cur.fetchall():
|
||
# If it has a parent, the parent is the root (or closer to it)
|
||
if parent_id:
|
||
roots.add(str(parent_id))
|
||
else:
|
||
# If no parent, it IS the root
|
||
roots.add(str(dept_id))
|
||
|
||
if not roots:
|
||
return department_ids
|
||
|
||
# 2. Find all departments that share these roots (the roots themselves and their children)
|
||
root_list = list(roots)
|
||
root_placeholders = ','.join(['%s'] * len(root_list))
|
||
|
||
# Select where ID is a root OR Parent ID is a root
|
||
sql_expand = f"""
|
||
SELECT id
|
||
FROM service_departments
|
||
WHERE id IN ({root_placeholders})
|
||
OR parent_id IN ({root_placeholders})
|
||
"""
|
||
# We pass root_list twice because we use the placeholders twice
|
||
cur.execute(sql_expand, root_list + root_list)
|
||
|
||
for row in cur.fetchall():
|
||
expanded_ids.add(str(row[0]))
|
||
|
||
return list(expanded_ids)
|
||
|
||
|
||
def filter_permits_advanced(
|
||
regions: Optional[List[str]] = None,
|
||
themes: Optional[List[str]] = None,
|
||
departments: Optional[List[str]] = None,
|
||
search_text: Optional[str] = None,
|
||
visibility: Optional[str] = None, # 'visible', 'hidden' or None/all
|
||
limit: int = 100,
|
||
offset: int = 0,
|
||
) -> Dict[str, Any]:
|
||
"""Filter permits using multiple dimensions (region, theme, department, search text, visibility).
|
||
|
||
Args:
|
||
regions: List of region IDs to filter by (supports multi-select)
|
||
themes: List of theme IDs to filter by (supports multi-select)
|
||
departments: List of department IDs to filter by (supports multi-select)
|
||
search_text: Search in permit name
|
||
visibility: Filter by v2 visibility ('visible', 'hidden')
|
||
limit: Maximum number of results to return
|
||
offset: Offset for pagination
|
||
|
||
Returns:
|
||
Dictionary containing filtered permits and metadata
|
||
"""
|
||
print(f"[DEBUG] filter_permits_advanced called with limit={limit}, offset={offset}, visibility={visibility}")
|
||
# Use subquery to avoid DISTINCT with window functions issue
|
||
# Subquery to get unique permits matching filters with pagination
|
||
# We use a CTE to ensure limit/offset apply to unique permits, not to rows (which can duplicate per theme)
|
||
base_where = " WHERE 1=1 "
|
||
base_params = []
|
||
|
||
if regions:
|
||
placeholders = ', '.join(['%s'] * len(regions))
|
||
base_where += f" AND rpd.region_id IN ({placeholders})"
|
||
base_params.extend(regions)
|
||
|
||
if themes:
|
||
placeholders = ', '.join(['%s'] * len(themes))
|
||
base_where += f" AND rtp.theme_id IN ({placeholders})"
|
||
base_params.extend(themes)
|
||
|
||
if departments:
|
||
# Expand departments to include family (parent + children)
|
||
# This allows cross-level visibility for the "same" department
|
||
expanded_departments = _expand_department_family(departments)
|
||
|
||
placeholders = ', '.join(['%s'] * len(expanded_departments))
|
||
base_where += f" AND (ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))"
|
||
base_params.extend(expanded_departments * 2)
|
||
|
||
if search_text:
|
||
base_where += f" AND LOWER(p.name) LIKE LOWER(%s)"
|
||
base_params.append(f"%{search_text}%")
|
||
|
||
if visibility == 'visible':
|
||
base_where += " AND (rpd.is_v2_visible IS TRUE OR rpd.is_v2_visible IS NULL)"
|
||
elif visibility == 'hidden':
|
||
base_where += " AND rpd.is_v2_visible IS FALSE"
|
||
|
||
sql = f"""
|
||
WITH filtered_p AS (
|
||
SELECT rpd.permit_id, rpd.region_id
|
||
FROM region_permit_details rpd
|
||
JOIN permits p ON p.id = rpd.permit_id
|
||
LEFT JOIN region_theme_permits rtp
|
||
ON rtp.permit_id = rpd.permit_id
|
||
AND rtp.region_id = rpd.region_id
|
||
LEFT JOIN permit_sources ps
|
||
ON ps.permit_id = rpd.permit_id
|
||
AND ps.region_id = rpd.region_id
|
||
{base_where}
|
||
GROUP BY rpd.permit_id, rpd.region_id, p.name
|
||
ORDER BY LOWER(p.name)
|
||
LIMIT %s OFFSET %s
|
||
)
|
||
SELECT
|
||
p.id AS permit_id,
|
||
p.name AS permit_name,
|
||
rpd.region_id,
|
||
r.name AS region_name,
|
||
rtp.theme_id,
|
||
t.name AS theme_name,
|
||
COALESCE(risk_counts.risk_count, 0) AS risk_count,
|
||
COALESCE(theme_counts.theme_count, 0) AS theme_count,
|
||
COALESCE(rpd.is_v2_visible, true) AS is_v2_visible
|
||
FROM filtered_p fp
|
||
JOIN region_permit_details rpd ON rpd.permit_id = fp.permit_id AND rpd.region_id = fp.region_id
|
||
JOIN permits p ON p.id = rpd.permit_id
|
||
JOIN regions r ON r.id = rpd.region_id
|
||
LEFT JOIN region_theme_permits rtp
|
||
ON rtp.permit_id = rpd.permit_id
|
||
AND rtp.region_id = rpd.region_id
|
||
LEFT JOIN themes t ON t.id = rtp.theme_id
|
||
LEFT JOIN (
|
||
SELECT
|
||
permit_id,
|
||
region_id,
|
||
COUNT(risk_id) AS risk_count
|
||
FROM region_permit_risks
|
||
GROUP BY permit_id, region_id
|
||
) risk_counts ON risk_counts.permit_id = rpd.permit_id
|
||
AND risk_counts.region_id = rpd.region_id
|
||
LEFT JOIN (
|
||
SELECT
|
||
permit_id,
|
||
region_id,
|
||
COUNT(DISTINCT theme_id) AS theme_count
|
||
FROM region_theme_permits
|
||
GROUP BY permit_id, region_id
|
||
) theme_counts ON theme_counts.permit_id = rpd.permit_id
|
||
AND theme_counts.region_id = rpd.region_id
|
||
ORDER BY LOWER(p.name), LOWER(r.name), LOWER(COALESCE(t.name, ''))
|
||
"""
|
||
params = base_params + [limit, offset]
|
||
|
||
permits_map = {}
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, params)
|
||
for (
|
||
permit_id,
|
||
permit_name,
|
||
rid,
|
||
region_name,
|
||
tid,
|
||
theme_name,
|
||
risk_count,
|
||
theme_count,
|
||
v2_visible,
|
||
) in cur.fetchall():
|
||
pid = str(permit_id)
|
||
key = f"{pid}_{rid}"
|
||
if key not in permits_map:
|
||
permits_map[key] = {
|
||
"id": pid,
|
||
"name": str(permit_name),
|
||
"region": {
|
||
"id": str(rid),
|
||
"name": str(region_name),
|
||
},
|
||
"themes": [],
|
||
"risk_count": int(risk_count or 0),
|
||
"theme_count": int(theme_count or 0),
|
||
"is_v2_visible": bool(v2_visible),
|
||
}
|
||
|
||
if tid or theme_name:
|
||
theme_payload = {
|
||
"id": str(tid) if tid else "",
|
||
"name": str(theme_name) if theme_name else "",
|
||
}
|
||
existing_themes = permits_map[key]["themes"]
|
||
if not any(
|
||
candidate.get("id") == theme_payload["id"]
|
||
and candidate.get("name") == theme_payload["name"]
|
||
for candidate in existing_themes
|
||
):
|
||
existing_themes.append(theme_payload)
|
||
|
||
# Use OrderedDict or sorted permits_list to maintain name order after dict values collection
|
||
permits_list = list(permits_map.values())
|
||
# Sort again by name to ensure order because dict.values() might not be stable depending on Python version/access
|
||
permits_list.sort(key=lambda x: x["name"].lower())
|
||
|
||
# Get total count for pagination
|
||
count_sql = f"""
|
||
SELECT COUNT(DISTINCT rpd.permit_id || '_' || rpd.region_id)
|
||
FROM region_permit_details rpd
|
||
JOIN permits p ON p.id = rpd.permit_id
|
||
LEFT JOIN region_theme_permits rtp ON rtp.permit_id = rpd.permit_id AND rtp.region_id = rpd.region_id
|
||
LEFT JOIN permit_sources ps ON ps.permit_id = rpd.permit_id AND ps.region_id = rpd.region_id
|
||
{base_where}
|
||
"""
|
||
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(count_sql, base_params)
|
||
total = cur.fetchone()[0]
|
||
|
||
return {
|
||
"permits": permits_list,
|
||
"pagination": {
|
||
"total": total,
|
||
"limit": limit,
|
||
"offset": offset,
|
||
"count": len(permits_list),
|
||
},
|
||
}
|
||
|
||
|
||
def _ensure_v2_visibility_column(conn: Optional[pg.Connection] = None) -> None:
|
||
"""Ensure that the is_v2_visible column exists in region_permit_details."""
|
||
global _V2_VISIBILITY_SCHEMA_READY
|
||
if _V2_VISIBILITY_SCHEMA_READY:
|
||
return
|
||
|
||
with _V2_VISIBILITY_SCHEMA_LOCK:
|
||
if _V2_VISIBILITY_SCHEMA_READY:
|
||
return
|
||
|
||
sql = "ALTER TABLE region_permit_details ADD COLUMN IF NOT EXISTS is_v2_visible BOOLEAN DEFAULT TRUE"
|
||
|
||
if conn is not None:
|
||
original_autocommit = conn.autocommit
|
||
try:
|
||
conn.autocommit = True
|
||
cur = conn.cursor()
|
||
cur.execute(sql)
|
||
finally:
|
||
conn.autocommit = original_autocommit
|
||
else:
|
||
with _lic_pg_conn(autocommit=True) as ensure_conn:
|
||
cur = ensure_conn.cursor()
|
||
cur.execute(sql)
|
||
|
||
_V2_VISIBILITY_SCHEMA_READY = True
|
||
|
||
|
||
def _ensure_contact_info_column(conn: pg.Connection) -> None:
|
||
"Ensure that the contact_info column exists in region_permit_details."
|
||
# This check is now redundant since schema fix script was run, but kept for safety
|
||
pass
|
||
|
||
|
||
def update_permit_v2_visibility(
|
||
region_id: str,
|
||
permit_id: str,
|
||
is_visible: bool,
|
||
operator: str = "admin"
|
||
) -> bool:
|
||
"""Update the is_v2_visible flag for a specific region-permit pair."""
|
||
_ensure_v2_visibility_column()
|
||
|
||
sql = """
|
||
UPDATE region_permit_details
|
||
SET is_v2_visible = %s,
|
||
updated_at = now()
|
||
WHERE region_id = %s AND permit_id = %s
|
||
"""
|
||
|
||
with _lic_pg_conn() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(sql, (is_visible, region_id, permit_id))
|
||
count = cur.rowcount
|
||
conn.commit()
|
||
|
||
if count > 0:
|
||
log_operation(
|
||
operator=operator,
|
||
operation_type="UPDATE",
|
||
target_type="PERMIT_VISIBILITY",
|
||
target_id=f"{region_id}:{permit_id}",
|
||
target_name=f"Permit {permit_id} in region {region_id}",
|
||
change_summary=f"Set is_v2_visible to {is_visible}",
|
||
details={"is_visible": is_visible, "region_id": region_id, "permit_id": permit_id}
|
||
)
|
||
return True
|
||
return False
|
||
|