From 6582b05adec2d47cc44f4c13f7e5ef3995c89ac8 Mon Sep 17 00:00:00 2001 From: liubiren Date: Thu, 30 Jul 2026 20:04:21 +0800 Subject: [PATCH] 1 --- .gitignore | 2 +- agent/application/preprocessing.py | 102 +++++++++++++++++++++++++++++ agent/test.py | 31 --------- utils/feishu.py | 27 +++----- 4 files changed, 112 insertions(+), 50 deletions(-) create mode 100644 agent/application/preprocessing.py delete mode 100644 agent/test.py diff --git a/.gitignore b/.gitignore index b1a60d4..cd6638c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ __pycache__/ **/__pycache__/ .DS_Store .gitignore -agent/models/ \ No newline at end of file +agent/application/models/ \ No newline at end of file diff --git a/agent/application/preprocessing.py b/agent/application/preprocessing.py new file mode 100644 index 0000000..30f58f3 --- /dev/null +++ b/agent/application/preprocessing.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +""" +预处理： +获取飞书知识库节点列表，并就文档节点与预处理任务比照，若： +1）该文档不存在则下载该文档MD文件，然后切块、向量化、存储到数据库中 +2）该文档存在但更新时间戳不一致则先再数据库中删除该文档所有块向量记录，再重新切块、向量化、存储到数据库中 +""" + +import asyncio +from pathlib import Path +from pydantic_ai import Embedder +from pydantic_ai.embeddings.sentence_transformers import ( + SentenceTransformerEmbeddingModel, + SentenceTransformersEmbeddingSettings, +) +from typing import cast, Dict, List, Any + +from json import loads + +from lark_oapi.api.wiki.service import WikiService + +from lark_oapi import Client, JSON +from lark_oapi.api.wiki.v2 import ( + ListSpaceNodeRequest, + ListSpaceNodeResponse, + ListSpaceNodeResponseBody, + Node, +) + + +# 实例化飞书服务端 +server = ( + Client.builder() + .app_id("cli_a1587980be78500c") + .app_secret("vZXGZomwfmyaHXoG8s810d1YYGLsIqCA") + .build() +) +# 实例化知识库 +if not (wiki := server.wiki): + raise Exception("server.wiki is None") + +# 在线文档唯一标识字典 +online_document_ids: Dict[str, int] = {} +# 分页标记 +page_token = "" +while True: + # 构造获取知识库空间子节点列表请求实例 + request: ListSpaceNodeRequest = ( + ListSpaceNodeRequest.builder() + .space_id("7615153684095257820") # 默认为产品设计知识库 + .page_size(50) # 分页大小 + .page_token(page_token) + .build() + ) + # 请求 + response: ListSpaceNodeResponse = wiki.v2.space_node.list(request) + if not response.success(): + raise Exception("response not success") + # 响应数据 + data = cast(ListSpaceNodeResponseBody, response.data) + for item in cast(List[Node], data.items): + if cast(str, item.obj_type) == "docx": + # 文档唯一标识 + document_id: str = cast(str, item.obj_token) + # 文档更新时间戳 + document_updated_at: int = cast(int, item.obj_edit_time) + online_document_ids[document_id] = document_updated_at + # 更新分页标记 + page_token = cast(str, data.page_token) + # 若是否还有更多项为否则跳出循环 + if not cast(bool, data.has_more): + break + +print(online_document_ids) + +exit() + + +async def main(): + # 自动定位：脚本目录/models/模型文件夹 + base_path = Path(__file__).parent + model_path = str(base_path / "models" / "paraphrase-multilingual-MiniLM-L12-v2") + + model = SentenceTransformerEmbeddingModel( + model_path, + settings=SentenceTransformersEmbeddingSettings( + sentence_transformers_device="cpu", + sentence_transformers_normalize_embeddings=True, + ), + ) + embedder = Embedder(model) + + query_text = "刘弼仁" + counts = await embedder.count_tokens(query_text) + print(counts) + max_tokens = await embedder.max_input_tokens() + print(f"Max tokens: {max_tokens}") + res = await embedder.embed_query(query_text) + print(f"向量维度：{len(res.embeddings[0])}") + + +asyncio.run(main()) diff --git a/agent/test.py b/agent/test.py deleted file mode 100644 index 0a275fa..0000000 --- a/agent/test.py +++ /dev/null @@ -1,31 +0,0 @@ -import asyncio -from pathlib import Path -from pydantic_ai import Embedder -from pydantic_ai.embeddings.sentence_transformers import ( - SentenceTransformerEmbeddingModel, - SentenceTransformersEmbeddingSettings -) - -async def main(): - # 自动定位：脚本目录/models/模型文件夹 - base_path = Path(__file__).parent - model_path = str(base_path / "models" / "paraphrase-multilingual-MiniLM-L12-v2") - - model = SentenceTransformerEmbeddingModel( - model_path, - settings=SentenceTransformersEmbeddingSettings( - sentence_transformers_device="cpu", - sentence_transformers_normalize_embeddings=True, - ) - ) - embedder = Embedder(model) - - query_text = "刘弼仁" - counts = await embedder.count_tokens(query_text) - print(counts) - max_tokens = await embedder.max_input_tokens() - print(f'Max tokens: {max_tokens}') - res = await embedder.embed_query(query_text) - print(f"向量维度：{len(res.embeddings[0])}") - -asyncio.run(main()) \ No newline at end of file diff --git a/utils/feishu.py b/utils/feishu.py index 5db577d..6b711f6 100644 --- a/utils/feishu.py +++ b/utils/feishu.py @@ -24,10 +24,8 @@ class Feishu: def __init__(self): # 实例化认证器 self.authenticator = Authenticator() - # 实例化请求客户端 self.http_client = Request() - # 实例化 Cloudreve 客户端 self.cloudreve = Cloudreve() @@ -67,25 +65,20 @@ class Feishu: start_timestamp = time() # 上一次查询时间戳 last_timestamp = 0 - while True: # 当前时间戳 current_timestamp = time() - # 若当前时间戳大于超时时间戳则登出并返回空 if current_timestamp > start_timestamp + 120: connection.logout() return None - # 若当前时间戳和上一次查询时间戳间隔小于5秒则跳转至下一次循环 if current_timestamp - last_timestamp < 5: continue - last_timestamp = current_timestamp # 选择邮箱文件夹 connection.select(mailbox=folder) - # 查询该邮箱文件夹内所有邮件 status, indices = connection.search( "utf-8", "ALL" @@ -94,7 +87,6 @@ class Feishu: if status != "OK": connection.logout() raise RuntimeError(f"查询邮箱文件夹内所有邮件失败") - # 拼接所有邮件索引并拆分为邮件索引列表 indices = b" ".join(indices).split() # 若邮件索引列表为空则跳转至下一次循环 @@ -157,9 +149,9 @@ class Feishu: connection.logout() return matched.group(1) - def _get_headers(self) -> Dict[str, Any]: + def _build_headers(self) -> Dict[str, Any]: """ - 获取请求头 + 构建请求头 :return: 请求头 """ return { @@ -184,10 +176,11 @@ class Feishu: feishu.get_bitable_records(app_token="A17bbGqZZaVWnfsFgencdls7nNf", table_id="tblijCBxHdfWyGcu") """ # 构建多维表格查询记录的请求地址 - url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/search" # https://open.feishu.cn/document/docs/bitable-v1/app-table-record/search 默认分页大小为 20 + # https://open.feishu.cn/document/docs/bitable-v1/app-table-record/search 默认分页大小为 20 + url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/search" - headers = self._get_headers() - # 添加 Content-Type 请求头 + # 构建请求头 + headers = self._build_headers() headers.update( { "Content-Type": "application/json; charset=utf-8", @@ -232,9 +225,9 @@ class Feishu: """ # 构建下载素材的请求地址 url = f"https://open.feishu.cn/open-apis/drive/v1/medias/{material_token}/download" # https://open.feishu.cn/document/server-docs/docs/drive-v1/media/download - - headers = self._get_headers() - # 添加 Content-Type 请求头 + + # 构建请求头 + headers = self._build_headers() headers.update( { "Content-Type": "application/json; charset=utf-8", @@ -268,5 +261,3 @@ class Feishu: return self.cloudreve.get_direct_link( uri=material_uri, size=material_size, generator=generator ) - -