diff --git a/agent/document_preprocess/main.py b/agent/document_preprocess/main.py index ff4f85e..1de4327 100644 --- a/agent/document_preprocess/main.py +++ b/agent/document_preprocess/main.py @@ -20,6 +20,27 @@ from pydantic import BaseModel, Field from requests import Response, exceptions, post from transformers import AutoTokenizer +""" +文档分片表建表SQL +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE TABLE document_chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id TEXT NOT NULL, + index INT NOT NULL, + text TEXT NOT NULL, + text_vector vector(1024) NOT NULL, + text_fingerprint TEXT NOT NULL, + text_tokens INT NOT NULL, + return_text TEXT +); + +CREATE INDEX idx_document_chunks_text ON document_chunks USING bm25 (text); +CREATE INDEX idx_document_chunks_text_vector ON document_chunks USING hnsw (text_vector vector_cosine_ops); +CREATE INDEX idx_document_chunks_text_fingerprint ON document_chunks(text_fingerprint); +""" + class Chunk(BaseModel): """