liubiren
/
Python
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

1

This commit is contained in:
liubiren 2026-07-30 20:04:21 +08:00
parent 5f438285a9
commit 6582b05ade
4 changed files with 112 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -2,4 +2,4 @@ __pycache__/
**/__pycache__/
.DS_Store
.gitignore
agent/models/
agent/application/models/

102
agent/application/preprocessing.py Normal file
View File

@ -0,0 +1,102 @@
# -*- coding: utf-8 -*-
"""
预处理：
获取飞书知识库节点列表，并就文档节点与预处理任务比照，若：
1）该文档不存在则下载该文档MD文件，然后切块、向量化、存储到数据库中
2）该文档存在但更新时间戳不一致则先再数据库中删除该文档所有块向量记录，再重新切块、向量化、存储到数据库中
"""
import asyncio
from pathlib import Path
from pydantic_ai import Embedder
from pydantic_ai.embeddings.sentence_transformers import (
SentenceTransformerEmbeddingModel,
SentenceTransformersEmbeddingSettings,
)
from typing import cast, Dict, List, Any
from json import loads
from lark_oapi.api.wiki.service import WikiService
from lark_oapi import Client, JSON
from lark_oapi.api.wiki.v2 import (
ListSpaceNodeRequest,
ListSpaceNodeResponse,
ListSpaceNodeResponseBody,
Node,
)
# 实例化飞书服务端
server = (
Client.builder()
.app_id("cli_a1587980be78500c")
.app_secret("vZXGZomwfmyaHXoG8s810d1YYGLsIqCA")
.build()
)
# 实例化知识库
if not (wiki := server.wiki):
raise Exception("server.wiki is None")
# 在线文档唯一标识字典
online_document_ids: Dict[str, int] = {}
# 分页标记
page_token = ""
while True:
# 构造获取知识库空间子节点列表请求实例
request: ListSpaceNodeRequest = (
ListSpaceNodeRequest.builder()
.space_id("7615153684095257820") # 默认为产品设计知识库
.page_size(50) # 分页大小
.page_token(page_token)
.build()
)
# 请求
response: ListSpaceNodeResponse = wiki.v2.space_node.list(request)
if not response.success():
raise Exception("response not success")
# 响应数据
data = cast(ListSpaceNodeResponseBody, response.data)
for item in cast(List[Node], data.items):
if cast(str, item.obj_type) == "docx":
# 文档唯一标识
document_id: str = cast(str, item.obj_token)
# 文档更新时间戳
document_updated_at: int = cast(int, item.obj_edit_time)
online_document_ids[document_id] = document_updated_at
# 更新分页标记
page_token = cast(str, data.page_token)
# 若是否还有更多项为否则跳出循环
if not cast(bool, data.has_more):
break
print(online_document_ids)
exit()
async def main():
# 自动定位：脚本目录/models/模型文件夹
base_path = Path(__file__).parent
model_path = str(base_path / "models" / "paraphrase-multilingual-MiniLM-L12-v2")
model = SentenceTransformerEmbeddingModel(
model_path,
settings=SentenceTransformersEmbeddingSettings(
sentence_transformers_device="cpu",
sentence_transformers_normalize_embeddings=True,
),
)
embedder = Embedder(model)
query_text = "刘弼仁"
counts = await embedder.count_tokens(query_text)
print(counts)
max_tokens = await embedder.max_input_tokens()
print(f"Max tokens: {max_tokens}")
res = await embedder.embed_query(query_text)
print(f"向量维度：{len(res.embeddings[0])}")
asyncio.run(main())

31
agent/test.py
View File

@ -1,31 +0,0 @@
import asyncio
from pathlib import Path
from pydantic_ai import Embedder
from pydantic_ai.embeddings.sentence_transformers import (
SentenceTransformerEmbeddingModel,
SentenceTransformersEmbeddingSettings
)
async def main():
# 自动定位：脚本目录/models/模型文件夹
base_path = Path(__file__).parent
model_path = str(base_path / "models" / "paraphrase-multilingual-MiniLM-L12-v2")
model = SentenceTransformerEmbeddingModel(
model_path,
settings=SentenceTransformersEmbeddingSettings(
sentence_transformers_device="cpu",
sentence_transformers_normalize_embeddings=True,
)
)
embedder = Embedder(model)
query_text = "刘弼仁"
counts = await embedder.count_tokens(query_text)
print(counts)
max_tokens = await embedder.max_input_tokens()
print(f'Max tokens: {max_tokens}')
res = await embedder.embed_query(query_text)
print(f"向量维度：{len(res.embeddings[0])}")
asyncio.run(main())

27
utils/feishu.py
View File

@ -24,10 +24,8 @@ class Feishu:
def __init__(self):
# 实例化认证器
self.authenticator = Authenticator()
# 实例化请求客户端
self.http_client = Request()
# 实例化 Cloudreve 客户端
self.cloudreve = Cloudreve()
@ -67,25 +65,20 @@ class Feishu:
start_timestamp = time()
# 上一次查询时间戳
last_timestamp = 0
while True:
# 当前时间戳
current_timestamp = time()
# 若当前时间戳大于超时时间戳则登出并返回空
if current_timestamp > start_timestamp + 120:
connection.logout()
return None
# 若当前时间戳和上一次查询时间戳间隔小于5秒则跳转至下一次循环
if current_timestamp - last_timestamp < 5:
continue
last_timestamp = current_timestamp
# 选择邮箱文件夹
connection.select(mailbox=folder)
# 查询该邮箱文件夹内所有邮件
status, indices = connection.search(
"utf-8", "ALL"
@ -94,7 +87,6 @@ class Feishu:
if status != "OK":
connection.logout()
raise RuntimeError(f"查询邮箱文件夹内所有邮件失败")
# 拼接所有邮件索引并拆分为邮件索引列表
indices = b" ".join(indices).split()
# 若邮件索引列表为空则跳转至下一次循环
@ -157,9 +149,9 @@ class Feishu:
connection.logout()
return matched.group(1)
def _get_headers(self) -> Dict[str, Any]:
def _build_headers(self) -> Dict[str, Any]:
"""
获取请求头
构建请求头
:return: 请求头
"""
return {
@ -184,10 +176,11 @@ class Feishu:
feishu.get_bitable_records(app_token="A17bbGqZZaVWnfsFgencdls7nNf", table_id="tblijCBxHdfWyGcu")
"""
# 构建多维表格查询记录的请求地址
url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/search" # https://open.feishu.cn/document/docs/bitable-v1/app-table-record/search 默认分页大小为 20
# https://open.feishu.cn/document/docs/bitable-v1/app-table-record/search 默认分页大小为 20
url = f"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/search"
headers = self._get_headers()
# 添加 Content-Type 请求头
# 构建请求头
headers = self._build_headers()
headers.update(
{
"Content-Type": "application/json; charset=utf-8",
@ -232,9 +225,9 @@ class Feishu:
"""
# 构建下载素材的请求地址
url = f"https://open.feishu.cn/open-apis/drive/v1/medias/{material_token}/download" # https://open.feishu.cn/document/server-docs/docs/drive-v1/media/download
headers = self._get_headers()
# 添加 Content-Type 请求头
# 构建请求头
headers = self._build_headers()
headers.update(
{
"Content-Type": "application/json; charset=utf-8",
@ -268,5 +261,3 @@ class Feishu:
return self.cloudreve.get_direct_link(
uri=material_uri, size=material_size, generator=generator
)