feat: Obsidian 知识库批量导入 + 语义搜索体验升级

- 新增 import_obsidian_kb.py 批量导入脚本（164篇入库，知识库186条）
- parse_md_file 补强：来源fallback、相关装备/应用领域实体提取、
  Obsidian双链去括号、原始类别/源文件路径存metadata、TYPE_MAP扩充
- search_similar 改进：智能摘要（中文2-gram拆词+加权段落匹配）、
  min_similarity=0.3过滤、top_k 5→10、返回完整content_md
- 前端搜索卡片升级：展开全文、关键词加粗渲染、相关度分档样式
- CLAUDE.md 状态更新

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

backend/app/api/knowledge.py

@@ -109,7 +109,8 @@ def get_grouped_knowledge_items(
 
 class SearchRequest(BaseModel):
     query: str
-    top_k: int = 5
+    top_k: int = 10
+    min_similarity: float = 0.3
 
 
 @router.post("/search")
@@ -121,10 +122,15 @@ def search_knowledge(
     """
     语义检索：输入一段文字，返回最相关的知识库条目及相似度。
     查询向量用 type="query"（区分于存入时的 type="db"）。
+    top_k 默认 10，min_similarity 默认 0.3 过滤低相关条目。
     三角色均可读。
     """
     svc = KnowledgeService()
-    results = svc.search_similar(query_text=body.query, top_k=body.top_k)
+    results = svc.search_similar(
+        query_text=body.query,
+        top_k=body.top_k,
+        min_similarity=body.min_similarity,
+    )
     return {
         "results": results,
         "query": body.query,

backend/app/services/knowledge_service.py

@@ -23,8 +23,15 @@ class KnowledgeService:
     SOURCE_TYPE_MAP = {
         "杂志文章": "military_report",
         "军报": "military_report",
+        "军报文章": "military_report",
         "节目文稿": "manuscript",
         "报题单": "baoti",
+        "装备": "manual",
+        "技术": "manual",
+        "动态": "manual",
+        "术语": "manual",
+        "厂商": "manual",
+        "索引": "manual",
     }
 
     # 来源大类固定显示顺序（制片人 Obsidian 习惯）
@@ -89,6 +96,12 @@ class KnowledgeService:
         else:
             source_detail = None
 
+        # fallback：如果 source_detail 仍为空，取"来源"字段
+        if not source_detail:
+            raw_source = str(fm.get("来源", "") or "").strip()
+            if raw_source:
+                source_detail = raw_source
+
         # —— 播出日期：容错 "待补充" 等非日期文本——
         raw_date = str(fm.get("播出日期", "") or "").strip()
         publish_date = None
@@ -107,17 +120,22 @@ class KnowledgeService:
         # —— 权重（不展示，存 JSONB 备 Phase 4）——
         weight = str(fm.get("权重", "") or "").strip() or None
 
-        # —— 相关实体（涉及装备/涉及技术/涉及厂商/主题）——
+        # —— 相关实体（涉及装备/涉及技术/涉及厂商/主题/相关装备/应用领域）——
+        import re as _re
         related_entities = []
-        for key in ("涉及装备", "涉及技术", "涉及厂商", "主题"):
+        for key in ("涉及装备", "涉及技术", "涉及厂商", "主题", "相关装备", "应用领域"):
             val = fm.get(key)
             if val:
                 if isinstance(val, list):
-                    related_entities.extend(val)
+                    for item in val:
+                        # 去掉 Obsidian 双链格式 [[xxx]]
+                        cleaned = _re.sub(r"\[\[|\]\]", "", str(item)).strip()
+                        if cleaned:
+                            related_entities.append(cleaned)
                 elif isinstance(val, str):
                     # 可能是 "山东舰, 福建舰" 这样的逗号分隔字符串
                     for item in val.replace("，", ",").split(","):
-                        item = item.strip()
+                        item = _re.sub(r"\[\[|\]\]", "", item).strip()
                         if item:
                             related_entities.append(item)
 
@@ -130,6 +148,26 @@ class KnowledgeService:
         # related_concepts 字段预留给双链解析（Phase 4），本 Task 原样存入
         metadata["double_bracket_links"] = self._extract_double_brackets(parsed.content)
 
+        # 保留原始 Obsidian 类别（映射到 manual 的类型，如装备/技术/动态/术语/厂商等）
+        if raw_type and source_type == "manual":
+            metadata["obsidian_category"] = raw_type
+
+        # 源文件路径保留（预埋 PDF 原文链接）
+        raw_source_files = fm.get("源文件")
+        if raw_source_files:
+            cleaned_files = []
+            if isinstance(raw_source_files, list):
+                for sf in raw_source_files:
+                    cleaned = _re.sub(r'[\[\]"]', "", str(sf)).strip()
+                    if cleaned:
+                        cleaned_files.append(cleaned)
+            elif isinstance(raw_source_files, str):
+                cleaned = _re.sub(r'[\[\]"]', "", raw_source_files).strip()
+                if cleaned:
+                    cleaned_files.append(cleaned)
+            if cleaned_files:
+                metadata["source_files"] = cleaned_files
+
         # —— 正文（去掉 frontmatter 的纯内容）——
         content_md = parsed.content
 
@@ -229,13 +267,11 @@ class KnowledgeService:
                     sources.add(tags["source_detail"])
             return sorted(list(sources))
 
-    def search_similar(self, query_text: str, top_k: int = 5) -> list[dict]:
+    def search_similar(self, query_text: str, top_k: int = 10, min_similarity: float = 0.3) -> list[dict]:
         """
         语义检索：查询句转为向量，用 SQL 余弦距离（<=>）在数据库层检索
-        返回 top_k 条相似笔记，含相似度分数 + 原文片段（SQL 端截断前 200 字）。
-
-        注意：当前取前 200 字是已知妥协（整篇向量检索无法定位中段命中点），
-        Phase 4a 做切块检索（chunk）时可优化为取最相关片段。
+        返回 top_k 条相似笔记，含相似度分数 + 智能摘要片段。
+        过滤掉 similarity < min_similarity 的条目。
         """
         query_vector = self.embedder.embed_single(query_text, embed_type="query")
         vec_str = "[" + ",".join(str(v) for v in query_vector) + "]"
@@ -248,7 +284,7 @@ class KnowledgeService:
                     ki.source_type,
                     ki.author,
                     ki.tags,
-                    SUBSTRING(ki.content_md, 1, 200) AS snippet,
+                    ki.content_md,
                     1 - (ke.embedding <=> '{vec_str}'::vector) AS similarity
                 FROM knowledge_embeddings ke
                 JOIN knowledge_items ki ON ke.knowledge_id = ki.id
@@ -260,19 +296,88 @@ class KnowledgeService:
             rows = session.execute(stmt).all()
             results = []
             for r in rows:
+                similarity = round(r.similarity, 4)
+                # 过滤低于阈值的条目
+                if similarity < min_similarity:
+                    continue
                 tags = r.tags or {}
                 source_detail = tags.get("source_detail") if isinstance(tags, dict) else None
+                snippet = self._extract_smart_snippet(r.content_md, query_text)
                 results.append({
                     "id": r.id,
                     "title": r.title,
                     "source_type": r.source_type,
                     "author": r.author,
                     "source_detail": source_detail,
-                    "snippet": r.snippet,
-                    "similarity": round(r.similarity, 4),
+                    "snippet": snippet,
+                    "content_md": r.content_md,
+                    "similarity": similarity,
                 })
             return results
 
+    def _extract_smart_snippet(self, content_md: str, query_text: str, max_len: int = 300) -> str:
+        """
+        智能摘要提取：根据搜索词定位最相关段落，截取摘要并加粗关键词。
+        中文连续字符串会被拆成 2-gram 子串以提高段落匹配命中率。
+        """
+        import re as _re
+
+        if not content_md:
+            return ""
+
+        # 1. 切分关键词：先按空格/标点拆，再把长中文词拆成 2-gram
+        raw_parts = _re.split(r'[\s，。！？、；：""''（）\(\)\[\]\-]+', query_text)
+        raw_parts = [p.strip() for p in raw_parts if len(p.strip()) > 1]
+
+        keywords = []
+        for part in raw_parts:
+            keywords.append(part)
+            if len(part) > 2 and _re.fullmatch(r'[一-鿿]+', part):
+                for i in range(len(part) - 1):
+                    bigram = part[i:i+2]
+                    if bigram not in keywords:
+                        keywords.append(bigram)
+
+        if not keywords:
+            return content_md[:max_len]
+
+        # 2. 按段落分割（跳过纯 markdown 标记行如 # ## --- 等）
+        paragraphs = content_md.split("\n\n")
+        if len(paragraphs) < 3:
+            paragraphs = content_md.split("\n")
+        paragraphs = [p.strip() for p in paragraphs
+                       if p.strip() and not _re.fullmatch(r'[#\-=\s>]+', p.strip())]
+
+        # 3. 计算每段关键词命中数（完整词权重 3，bigram 权重 1）
+        best_para = ""
+        best_score = 0
+        for para in paragraphs:
+            score = 0
+            for kw in keywords:
+                if kw in para:
+                    score += 3 if kw in raw_parts else 1
+            if score > best_score:
+                best_score = score
+                best_para = para
+
+        # 4. fallback：无关键词命中时用正文前 max_len 字
+        if best_score == 0:
+            snippet = content_md[:max_len]
+        else:
+            snippet = best_para[:max_len]
+
+        # 5. 加粗命中关键词（优先标记完整词，避免 bigram 重复标记）
+        marked = set()
+        for kw in sorted(keywords, key=len, reverse=True):
+            if kw in snippet and kw not in marked:
+                snippet = snippet.replace(kw, f"**{kw}**", 1)
+                marked.add(kw)
+                for sub_kw in keywords:
+                    if sub_kw != kw and sub_kw in kw:
+                        marked.add(sub_kw)
+
+        return snippet
+
     def get_item_count(self) -> int:
         with Session(engine) as session:
             return len(session.exec(select(KnowledgeItem)).all())

backend/scripts/import_obsidian_kb.py

@@ -0,0 +1,169 @@
+"""
+批量导入脚本：将 Obsidian 知识库目录下的 .md 文件
+导入 TPS 知识库（knowledge_items + knowledge_embeddings）。
+
+运行方式：
+    cd backend && python -m scripts.import_obsidian_kb --limit 10
+    cd backend && python -m scripts.import_obsidian_kb              # 全量
+    cd backend && python -m scripts.import_obsidian_kb --dry-run    # 只预览不写库
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from sqlmodel import Session, select
+
+from app.db.session import engine
+from app.models.knowledge import KnowledgeItem
+from app.services.knowledge_service import KnowledgeService
+
+# ── 默认 Obsidian 知识库根目录 ────────────────────────────────
+DEFAULT_OBSIDIAN_DIR = r"E:\AIworks\obsidian\MSTknowledge_base\知识库"
+
+# ── 跳过的目录名 ────────────────────────────────────────────────
+SKIP_DIRS = {".obsidian", "99-原始素材"}
+
+
+def collect_md_files(root: Path) -> list[Path]:
+    """递归收集 root 下所有 .md 文件，跳过 SKIP_DIRS 中的目录。"""
+    md_files = []
+    for p in root.rglob("*.md"):
+        # 检查路径中是否包含跳过目录
+        parts = p.relative_to(root).parts
+        if any(skip in parts for skip in SKIP_DIRS):
+            continue
+        md_files.append(p)
+    return sorted(md_files, key=lambda p: str(p.relative_to(root)))
+
+
+def check_exists(source_file_name: str) -> bool:
+    """查 knowledge_items 表，判断 source_file_name 是否已存在。"""
+    with Session(engine) as session:
+        stmt = select(KnowledgeItem).where(
+            KnowledgeItem.source_file_name == source_file_name
+        )
+        existing = session.exec(stmt).first()
+        return existing is not None
+
+
+def has_frontmatter(content: str) -> bool:
+    """检查文件内容是否包含 YAML frontmatter（--- 包裹）。"""
+    stripped = content.strip()
+    if not stripped.startswith("---"):
+        return False
+    # 找到第二个 ---
+    second = stripped.find("---", 3)
+    return second > 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description="批量导入 Obsidian 知识库 .md 文件到 TPS 知识库")
+    parser.add_argument("--dir", type=str, default=DEFAULT_OBSIDIAN_DIR,
+                        help="Obsidian 知识库根目录")
+    parser.add_argument("--limit", type=int, default=0,
+                        help="只处理前 N 个文件（测试用，0=全量）")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="只打印解析结果，不真正调 API 写库")
+    args = parser.parse_args()
+
+    root = Path(args.dir)
+    if not root.is_dir():
+        print(f"[ERROR] 目录不存在: {root}")
+        sys.exit(1)
+
+    # 收集文件
+    md_files = collect_md_files(root)
+    if not md_files:
+        print("[ERROR] 未找到任何 .md 文件")
+        sys.exit(1)
+
+    total = len(md_files)
+    if args.limit > 0:
+        md_files = md_files[:args.limit]
+        print(f"共发现 {total} 个 .md 文件，本次处理前 {len(md_files)} 个\n")
+    else:
+        print(f"共发现 {total} 个 .md 文件，开始全量导入...\n")
+
+    if args.dry_run:
+        print("【DRY-RUN 模式】只预览解析结果，不写入数据库\n")
+
+    service = KnowledgeService()
+    success_count = 0
+    skip_exists_count = 0
+    skip_empty_count = 0
+    skip_no_fm_count = 0
+    fail_count = 0
+    processed = len(md_files)
+
+    for idx, file_path in enumerate(md_files, 1):
+        rel_path = str(file_path.relative_to(root)).replace("\\", "/")
+
+        # 跳过空文件
+        if file_path.stat().st_size == 0:
+            print(f"[{idx}/{processed}] [SKIP] {rel_path} -- 空文件，跳过")
+            skip_empty_count += 1
+            continue
+
+        # 读取内容
+        try:
+            content_bytes = file_path.read_bytes()
+        except Exception as e:
+            print(f"[{idx}/{processed}] [FAIL] {rel_path} -- 读取失败：{e}")
+            fail_count += 1
+            continue
+
+        # 检查 frontmatter
+        content_str = content_bytes.decode("utf-8", errors="replace")
+        if not has_frontmatter(content_str):
+            print(f"[{idx}/{processed}] [SKIP] {rel_path} -- 无 frontmatter，跳过")
+            skip_no_fm_count += 1
+            continue
+
+        # 查重（用 Obsidian 相对路径作为 source_file_name）
+        source_file_name = rel_path
+        if check_exists(source_file_name):
+            print(f"[{idx}/{processed}] [SKIP] {rel_path} -- 已存在，跳过")
+            skip_exists_count += 1
+            continue
+
+        # dry-run 模式：只解析不入库
+        if args.dry_run:
+            try:
+                parsed = service.parse_md_file(content_bytes, source_file_name)
+                char_count = len(parsed["content_md"])
+                print(f"[{idx}/{processed}] [PREVIEW] {rel_path}")
+                print(f"         title={parsed['title']}")
+                print(f"         source_type={parsed['source_type']}, author={parsed['author']}")
+                print(f"         metadata={parsed['metadata']}")
+                print(f"         entities={len(parsed['related_entities'] or [])}条, 正文{char_count}字")
+                success_count += 1
+            except Exception as e:
+                print(f"[{idx}/{processed}] [FAIL] {rel_path} -- 解析失败：{e}")
+                fail_count += 1
+            continue
+
+        # 正式入库
+        try:
+            parsed = service.parse_md_file(content_bytes, source_file_name)
+            char_count = len(parsed["content_md"])
+            service.store_md_file(file_content=content_bytes, file_name=source_file_name)
+            print(f"[{idx}/{processed}] [OK] {rel_path} -- 入库成功（{char_count}字，类型={parsed['source_type']}）")
+            success_count += 1
+        except Exception as e:
+            print(f"[{idx}/{processed}] [FAIL] {rel_path} -- 入库失败：{e}")
+            fail_count += 1
+
+    # 汇总
+    print(f"\n{'='*60}")
+    mode_label = "预览" if args.dry_run else "导入"
+    print(f"{mode_label}完成：成功 {success_count} 篇"
+          f" / 跳过(已存在) {skip_exists_count} 篇"
+          f" / 跳过(空文件) {skip_empty_count} 篇"
+          f" / 跳过(无frontmatter) {skip_no_fm_count} 篇"
+          f" / 失败 {fail_count} 篇")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    main()