docs: create root README and update guides with checkpoint system info; fix: JSON serialization bug in checkpoint creation for binary data

.gitignore

@@ -32,4 +32,4 @@ temp_repo.py
 excel_info.txt
 parsing_output.txt
 unmatched_permits_report.txt
-analysis/
+analysis/data/checkpoints/

README.md

@@ -0,0 +1,43 @@
+# 市监局法律风险提示系统 (LawRisk Backend)
+
+智能法律风险检索与管理系统后端服务，基于 Flask 开发，提供行政许可事项的风险提示检索、数据管理及自动化备份功能。
+
+## 🌟 核心功能
+
+- **智能检索 (V2 API)**: 支持自然语言查询，结合向量嵌入和 LLM 技术，精准匹配许可事项与风险点。
+- **行政许可管理**: 提供完善的 Excel 导入机制，支持“以新盖旧”模式，并在覆盖前自动创建风险点数据快照。
+- **自动化备份 (Checkpoint)**: 内置数据库检查点系统，定期或手动记录数据库全量状态，支持一键恢复。
+- **权限管理**: 灵活的角色与层级控制，支持市级、区级及具体单位的细粒度数据隔离。
+- **组织架构**: 动态组织架构管理，支持拖拽调整层级关系。
+
+## 🚀 快速开始
+
+### 1. 环境准备
+```bash
+python -m venv .venv
+source .venv/bin/activate  # Windows: .venv\Scripts\activate
+pip install -r requirements.txt
+```
+
+### 2. 配置环境变量
+修改 `.env` 文件，配置数据库连接、DashScope API 密钥等。详细说明请参考 [配置指南](docs/guides/README.md#2-配置环境变量)。
+
+### 3. 运行服务
+```bash
+python app.py
+```
+
+## 📖 详细文档
+
+- **[项目架构与指南](docs/guides/README.md)**: 了解技术栈、项目结构及开发流程。
+- **[API 文档 (V2)](docs/V2_API文档.md)**: 详细的 API 接口说明与示例。
+- **[文档索引](docs/README.md)**: 包含所有功能开发、测试报告及维护文档的详细列表。
+
+## 🛠️ 最近更新
+
+- **Checkpoint 系统增强**: 修复了二进制数据（Excel 原始文件）备份时的 JSON 序列化问题，增强了备份稳定性。
+- **导入机制确认**: 支持许可事项的智能覆盖模式，系统在覆盖前会自动执行风险点快照备份。
+- **数据清理**: 优化了检查点清理逻辑，支持手动触发全量备份并清空冗余历史记录。
+
+---
+© 2025 市监局项目开发组

docs/guides/README.md

@@ -109,10 +109,15 @@ curl -X POST "http://localhost:8000/fs-ai-asistant/api/workflow/lawrisk/v2" \
 
 ## 身份认证
 
-- 设置 `FLASK_SECRET_KEY` 保护会话 Cookie，同时通过 `LAWRISK_ADMIN_USERNAME` 和 `LAWRISK_ADMIN_PASSWORD` 注入首个管理员账号（可选 `LAWRISK_ADMIN_ROLE`, `LAWRISK_ADMIN_GRADE` 和 `LAWRISK_ADMIN_DISPLAY_NAME`）。
+- 设置 `FLASK_SECRET_KEY` 保护会话 Cookie，同时通过 `LAWRISK_ADMIN_USERNAME` 和 `LAWRISK_ADMIN_PASSWORD` 注入首个管理员账号。
 - 首次启动会自动创建 `auth_users` 表，用于存储用户名、哈希密码、角色（role）和级别（grade）。
-- 登录页位于 `http://localhost:8000/fs-ai-asistant/lawrisk/login`，表单提交至 `/auth/login`；成功后会在会话中写入当前用户信息。
+- 登录页位于 `/fs-ai-asistant/lawrisk/login`，API 客户端可以调用 `/auth/me` 获取当前登录信息。
-- API 客户端可以调用 `/auth/me` 获取当前登录信息，调用 `/auth/logout` 注销。
+
+## 数据备份与恢复 (Checkpoint)
+
+- 系统提供 Checkpoint 功能，可对数据库全表进行 JSON 序列化备份。
+- 支持二进制数据（如 `bytes`, `memoryview`）的 Base64 自动转换，确保 Excel 原始文件等资产的完整备份。
+- 管理员可通过管理后台 `/db_admin` 手动创建、列出或恢复检查点。
 
 ## API文档
 

lawrisk/services/licensing_repo.py

@@ -93,10 +93,14 @@ _IMPORT_HEADER_ALIASES: Dict[str, Set[str]] = {
     "summary": {
         "风险说明",
         "摘要",
-        "备注",
         "风险摘要",
         "补充说明",
     },
+    "remark": {
+        "备注",
+        "其他",
+        "备注项",
+    },
     "permit_status": {
         "许可状态",
         "事项状态",
@@ -159,7 +163,8 @@ _IMPORT_HEADER_KEYWORDS: List[Tuple[str, Tuple[str, ...]]] = [
     ("risk_content", ("风险",)),
     ("legal_basis", ("依据",)),
     ("document_no", ("文号", "编号")),
-    ("summary", ("备注", "摘要")),
+    ("summary", ("摘要", "说明")),
+    ("remark", ("备注",)),
     ("responsible_contact", ("责任", "主管")),
     ("jurisdiction_scope", ("范围", "区域")),
 ]
@@ -328,6 +333,9 @@ def _score_import_header(canonical: str, cell_text: str, col_idx: int) -> float:
     elif canonical == "summary":
         if "摘要" in text:
             score += 3
+    elif canonical == "remark":
+        if "备注" in text:
+            score += 3
 
     elif canonical == "serial_number":
         if "序号" in text:
@@ -402,6 +410,7 @@ def _normalize_import_row(
     legal_basis = _clean_empty(raw_row.get("legal_basis"))
     document_no = _clean_empty(raw_row.get("document_no"))
     summary = _clean_empty(raw_row.get("summary"))
+    remark = _clean_empty(raw_row.get("remark"))
     permit_status = _clean_empty(raw_row.get("permit_status") or sheet_defaults.get("permit_status"))
     filler_name = _clean_empty(raw_row.get("filler_name") or sheet_defaults.get("filler_name"))
     unit_name = _clean_empty(raw_row.get("unit_name") or sheet_defaults.get("unit_name"))
@@ -436,6 +445,7 @@ def _normalize_import_row(
         "legal_basis": legal_basis,
         "document_no": document_no,
         "summary": summary,
+        "remark": remark,
         "permit_status": permit_status,
         "responsible_contact": responsible_contact,
         "jurisdiction_scope": jurisdiction_scope,
@@ -1025,17 +1035,18 @@ def _ensure_risk(
     legal_basis: Optional[str],
     document_no: Optional[str],
     summary: Optional[str],
+    remark: Optional[str] = None,
 ) -> str:
     cur = conn.cursor()
     cur.execute(
         """
-        INSERT INTO risks (risk_content, legal_basis, document_no, summary)
+        INSERT INTO risks (risk_content, legal_basis, document_no, summary, remark)
-        VALUES (%s, %s, %s, %s)
+        VALUES (%s, %s, %s, %s, %s)
-        ON CONFLICT (risk_content, legal_basis, document_no, summary)
+        ON CONFLICT (risk_content, legal_basis, document_no, summary, remark)
         DO UPDATE SET risk_content = EXCLUDED.risk_content
         RETURNING id
         """,
-        (risk_content, legal_basis, document_no, summary),
+        (risk_content, legal_basis, document_no, summary, remark),
     )
     risk_id = cur.fetchone()[0]
     return str(risk_id)
@@ -1570,6 +1581,7 @@ def commit_permit_import_session(
                             legal_basis=row.get("legal_basis"),
                             document_no=row.get("document_no"),
                             summary=row.get("summary"),
+                            remark=row.get("remark"),
                         )
                         cur.execute(
                             """
@@ -2444,6 +2456,45 @@ def list_all_themes() -> List[Dict[str, Any]]:
     return items
 
 
+def list_unbound_permits() -> List[Dict[str, Any]]:
+    """Return all permits that are in a region but not bound to any theme in that region."""
+    sql = """
+        SELECT 
+            r.id AS region_id,
+            r.name AS region_name,
+            p.id AS permit_id,
+            p.name AS permit_name,
+            rpd.unit_name,
+            rpd.updated_at
+        FROM region_permit_details rpd
+        JOIN regions r ON r.id = rpd.region_id
+        JOIN permits p ON p.id = rpd.permit_id
+        LEFT JOIN region_theme_permits rtp 
+            ON rtp.region_id = rpd.region_id 
+            AND rtp.permit_id = rpd.permit_id
+        WHERE rtp.theme_id IS NULL
+        ORDER BY r.name, p.name
+    """
+    items: List[Dict[str, Any]] = []
+    try:
+        with _lic_pg_conn() as conn:
+            cur = conn.cursor()
+            cur.execute(sql)
+            rows = cur.fetchall()
+            columns = tuple(col[0] for col in cur.description)
+            for row in rows:
+                record = {columns[idx]: row[idx] for idx in range(len(columns))}
+                # Serialize UUIDs and timestamps
+                record["region_id"] = str(record["region_id"])
+                record["permit_id"] = str(record["permit_id"])
+                if record["updated_at"]:
+                    record["updated_at"] = record["updated_at"].isoformat()
+                items.append(record)
+    except Exception as e:
+        logger.error(f"Error listing unbound permits: {e}")
+    return items
+
+
 def create_theme(name: str) -> Dict[str, Any]:
     normalized = _clean_text(name)
     if not normalized:
@@ -3329,6 +3380,7 @@ def load_permits_and_risks(
             rk.legal_basis,
             rk.document_no,
             rk.summary,
+            rk.remark,
             rpr.serial_number,
             rpd.permit_status,
             rpd.subitem_summary,
@@ -3336,21 +3388,19 @@ def load_permits_and_risks(
             rpd.jurisdiction_scope,
             rpd.filler_name,
             rpd.unit_name,
-            rpd.filler_name,
-            rpd.unit_name,
             rpd.source_update_date,
             rpd.contact_info
-        FROM region_theme_permits rtp
+        FROM region_permit_details rpd
-        JOIN permits p ON p.id = rtp.permit_id
+        JOIN permits p ON p.id = rpd.permit_id
+        LEFT JOIN region_theme_permits rtp
+               ON rtp.region_id = rpd.region_id
+              AND rtp.permit_id = rpd.permit_id
         LEFT JOIN themes t ON t.id = rtp.theme_id
         LEFT JOIN region_permit_risks rpr
-               ON rpr.region_id = rtp.region_id
+               ON rpr.region_id = rpd.region_id
-              AND rpr.permit_id = rtp.permit_id
+              AND rpr.permit_id = rpd.permit_id
         LEFT JOIN risks rk ON rk.id = rpr.risk_id
-        LEFT JOIN region_permit_details rpd
+        WHERE rpd.region_id = %s
-               ON rpd.region_id = rtp.region_id
-              AND rpd.permit_id = rtp.permit_id
-        WHERE rtp.region_id = %s
     """
     params: List[Any] = [region_id]
     theme_filter = theme_id if (theme_id and not _is_all_theme_marker(theme_id)) else None
@@ -3358,13 +3408,14 @@ def load_permits_and_risks(
         sql += " AND (rtp.theme_id = %s OR t.name = '所有主题事项')"
         params.append(theme_filter)
     if permit_id is not None:
-        sql += " AND rtp.permit_id = %s"
+        sql += " AND rpd.permit_id = %s"
         params.append(permit_id)
 
     sql += """
         ORDER BY p.name, rk.risk_content
     """
     permits: Dict[str, Dict[str, object]] = {}
+    risk_seen_map: Dict[str, Set[str]] = {}  # pid -> set of risk_ids
     with _lic_pg_conn() as conn:
         _ensure_contact_info_column(conn)
         cur = conn.cursor()
@@ -3380,6 +3431,7 @@ def load_permits_and_risks(
                 legal_basis,
                 document_no,
                 summary,
+                remark,
                 serial_number,
                 permit_status,
                 subitem_summary,
@@ -3387,8 +3439,6 @@ def load_permits_and_risks(
                 jurisdiction_scope,
                 filler_name,
                 unit_name,
-                filler_name,
-                unit_name,
                 source_update_date,
                 contact_info,
             ) = row
@@ -3408,7 +3458,6 @@ def load_permits_and_risks(
                     "jurisdiction_scope": None,
                     "filler_name": None,
                     "unit_name": None,
-                    "unit_name": None,
                     "source_update_date": None,
                     "contact_info": None,
                     "theme": {
@@ -3463,17 +3512,24 @@ def load_permits_and_risks(
             if entry["contact_info"] is None and contact_info:
                 entry["contact_info"] = contact_info.strip() or None
             if risk_id is not None:
-                summary_markdown = _format_summary_markdown(summary or "")
+                risk_id_str = str(risk_id)
-                entry["risks"].append(
+                # Avoid duplicates when a permit has multiple themes
-                    {
+                seen_risk_ids = risk_seen_map.setdefault(pid, set())
-                        "id": str(risk_id),
+                if risk_id_str not in seen_risk_ids:
-                        "risk_content": risk_content or "",
+                    seen_risk_ids.add(risk_id_str)
-                        "legal_basis": legal_basis or "",
+                    summary_markdown = _format_summary_markdown(summary or "")
-                        "document_no": document_no or "",
+                    remark_markdown = _format_summary_markdown(remark or "")
-                        "summary": summary_markdown,
+                    entry["risks"].append(
-                        "serial_number": serial_number,
+                        {
-                    }
+                            "id": risk_id_str,
-                )
+                            "risk_content": risk_content or "",
+                            "legal_basis": legal_basis or "",
+                            "document_no": document_no or "",
+                            "summary": summary_markdown,
+                            "remark": remark_markdown,
+                            "serial_number": serial_number,
+                        }
+                    )
 
         permit_ids = list(permits.keys())
         scope_map = _load_permit_scopes_for_region(conn, region_id, permit_ids)
@@ -4133,6 +4189,11 @@ def create_checkpoint(description: str = "") -> Dict[str, Any]:
         except ImportError:
             pass
 
+        if isinstance(obj, (bytes, bytearray, memoryview)):
+            import base64
+            # For data integrity in JSON, base64 is standard
+            return base64.b64encode(obj).decode('ascii')
+
         if hasattr(obj, 'isoformat'):
             return str(obj)
         raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
@@ -5600,19 +5661,62 @@ def filter_permits_advanced(
     """
     print(f"[DEBUG] filter_permits_advanced called with limit={limit}, offset={offset}")
     # Use subquery to avoid DISTINCT with window functions issue
-    sql = """
+    # Subquery to get unique permits matching filters with pagination
+    # We use a CTE to ensure limit/offset apply to unique permits, not to rows (which can duplicate per theme)
+    base_where = " WHERE 1=1 "
+    base_params = []
+    
+    if regions:
+        placeholders = ', '.join(['%s'] * len(regions))
+        base_where += f" AND rpd.region_id IN ({placeholders})"
+        base_params.extend(regions)
+
+    if themes:
+        placeholders = ', '.join(['%s'] * len(themes))
+        base_where += f" AND rtp.theme_id IN ({placeholders})"
+        base_params.extend(themes)
+
+    if departments:
+        placeholders = ', '.join(['%s'] * len(departments))
+        base_where += f" AND (ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))"
+        base_params.extend(departments * 2)
+
+    if search_text:
+        base_where += f" AND LOWER(p.name) LIKE LOWER(%s)"
+        base_params.append(f"%{search_text}%")
+
+    sql = f"""
+        WITH filtered_p AS (
+            SELECT rpd.permit_id, rpd.region_id
+            FROM region_permit_details rpd
+            JOIN permits p ON p.id = rpd.permit_id
+            LEFT JOIN region_theme_permits rtp 
+                   ON rtp.permit_id = rpd.permit_id 
+                  AND rtp.region_id = rpd.region_id
+            LEFT JOIN permit_sources ps
+                   ON ps.permit_id = rpd.permit_id
+                  AND ps.region_id = rpd.region_id
+            {base_where}
+            GROUP BY rpd.permit_id, rpd.region_id, p.name
+            ORDER BY LOWER(p.name)
+            LIMIT %s OFFSET %s
+        )
         SELECT
             p.id AS permit_id,
             p.name AS permit_name,
-            rtp.region_id,
+            rpd.region_id,
             r.name AS region_name,
             rtp.theme_id,
             t.name AS theme_name,
             COALESCE(risk_counts.risk_count, 0) AS risk_count,
             COALESCE(theme_counts.theme_count, 0) AS theme_count
-        FROM region_theme_permits rtp
+        FROM filtered_p fp
-        JOIN permits p ON p.id = rtp.permit_id
+        JOIN region_permit_details rpd ON rpd.permit_id = fp.permit_id AND rpd.region_id = fp.region_id
-        JOIN regions r ON r.id = rtp.region_id
+        JOIN permits p ON p.id = rpd.permit_id
+        JOIN regions r ON r.id = rpd.region_id
+        LEFT JOIN region_theme_permits rtp 
+               ON rtp.permit_id = rpd.permit_id 
+              AND rtp.region_id = rpd.region_id
         LEFT JOIN themes t ON t.id = rtp.theme_id
         LEFT JOIN (
             SELECT
@@ -5621,8 +5725,8 @@ def filter_permits_advanced(
                 COUNT(risk_id) AS risk_count
             FROM region_permit_risks
             GROUP BY permit_id, region_id
-        ) risk_counts ON risk_counts.permit_id = rtp.permit_id
+        ) risk_counts ON risk_counts.permit_id = rpd.permit_id
-                      AND risk_counts.region_id = rtp.region_id
+                      AND risk_counts.region_id = rpd.region_id
         LEFT JOIN (
             SELECT
                 permit_id,
@@ -5630,51 +5734,11 @@ def filter_permits_advanced(
                 COUNT(DISTINCT theme_id) AS theme_count
             FROM region_theme_permits
             GROUP BY permit_id, region_id
-        ) theme_counts ON theme_counts.permit_id = rtp.permit_id
+        ) theme_counts ON theme_counts.permit_id = rpd.permit_id
-                       AND theme_counts.region_id = rtp.region_id
+                       AND theme_counts.region_id = rpd.region_id
-        LEFT JOIN permit_sources ps
-               ON ps.permit_id = rtp.permit_id
-              AND ps.region_id = rtp.region_id
-        WHERE 1=1
-    """
-    params = []
-    param_count = 0
-
-    if regions:
-        placeholders = ', '.join(['%s'] * len(regions))
-        param_count += len(regions)
-        sql += f" AND rtp.region_id IN ({placeholders})"
-        params.extend(regions)
-
-    if themes:
-        placeholders = ', '.join(['%s'] * len(themes))
-        param_count += len(themes)
-        sql += f" AND rtp.theme_id IN ({placeholders})"
-        params.extend(themes)
-
-    if departments:
-        placeholders = ', '.join(['%s'] * len(departments))
-        param_count += len(departments)
-        sql += f" AND (ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))"
-        params.extend(departments * 2)
-
-    if search_text:
-        param_count += 1
-        sql += f" AND LOWER(p.name) LIKE LOWER(%s)"
-        params.append(f"%{search_text}%")
-
-    sql += """
         ORDER BY LOWER(p.name), LOWER(r.name), LOWER(COALESCE(t.name, ''))
     """
-
+    params = base_params + [limit, offset]
-    # Add pagination
-    param_count += 1
-    sql += f" LIMIT %s"
-    params.append(limit)
-
-    param_count += 1
-    sql += f" OFFSET %s"
-    params.append(offset)
 
     permits_map = {}
     with _lic_pg_conn() as conn:
@@ -5718,38 +5782,24 @@ def filter_permits_advanced(
                 ):
                     existing_themes.append(theme_payload)
 
+    # Use OrderedDict or sorted permits_list to maintain name order after dict values collection
     permits_list = list(permits_map.values())
+    # Sort again by name to ensure order because dict.values() might not be stable depending on Python version/access
+    permits_list.sort(key=lambda x: x["name"].lower())
 
     # Get total count for pagination
-    count_sql = """
+    count_sql = f"""
-        SELECT COUNT(DISTINCT rtp.permit_id || '_' || rtp.region_id)
+        SELECT COUNT(DISTINCT rpd.permit_id || '_' || rpd.region_id)
-        FROM region_theme_permits rtp
+        FROM region_permit_details rpd
-        JOIN permits p ON p.id = rtp.permit_id
+        JOIN permits p ON p.id = rpd.permit_id
-        LEFT JOIN permit_sources ps
+        LEFT JOIN region_theme_permits rtp ON rtp.permit_id = rpd.permit_id AND rtp.region_id = rpd.region_id
-               ON ps.permit_id = rtp.permit_id
+        LEFT JOIN permit_sources ps ON ps.permit_id = rpd.permit_id AND ps.region_id = rpd.region_id
-              AND ps.region_id = rtp.region_id
+        {base_where}
-        WHERE 1=1
     """
-    count_params = []
+    
-    if regions:
-        placeholders = ', '.join(['%s'] * len(regions))
-        count_sql += f" AND rtp.region_id IN ({placeholders})"
-        count_params.extend(regions)
-    if themes:
-        placeholders = ', '.join(['%s'] * len(themes))
-        count_sql += f" AND rtp.theme_id IN ({placeholders})"
-        count_params.extend(themes)
-    if departments:
-        placeholders = ', '.join(['%s'] * len(departments))
-        count_sql += f" AND (ps.uploader_department_id IN ({placeholders}) OR ps.bound_department_id IN ({placeholders}))"
-        count_params.extend(departments * 2)
-    if search_text:
-        count_sql += " AND LOWER(p.name) LIKE LOWER(%s)"
-        count_params.append(f"%{search_text}%")
-
     with _lic_pg_conn() as conn:
         cur = conn.cursor()
-        cur.execute(count_sql, count_params)
+        cur.execute(count_sql, base_params)
         total = cur.fetchone()[0]
 
     return {
@@ -5762,6 +5812,7 @@ def filter_permits_advanced(
         },
     }
 
+
 def _ensure_contact_info_column(conn: pg.Connection) -> None:
     "Ensure that the contact_info column exists in region_permit_details."
     # This check is now redundant since schema fix script was run, but kept for safety