1

agent/document_preprocess/main.py

@@ -20,6 +20,27 @@ from pydantic import BaseModel, Field
 from requests import Response, exceptions, post
 from transformers import AutoTokenizer
 
+"""
+文档分片表建表SQL
+CREATE EXTENSION IF NOT EXISTS vector;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+CREATE TABLE document_chunks (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    document_id TEXT NOT NULL,
+    index INT NOT NULL,
+    text TEXT NOT NULL,
+    text_vector vector(1024) NOT NULL,
+    text_fingerprint TEXT NOT NULL,
+    text_tokens INT NOT NULL,
+    return_text TEXT
+);
+
+CREATE INDEX idx_document_chunks_text ON document_chunks USING bm25 (text);
+CREATE INDEX idx_document_chunks_text_vector ON document_chunks USING hnsw (text_vector vector_cosine_ops);
+CREATE INDEX idx_document_chunks_text_fingerprint ON document_chunks(text_fingerprint);
+"""
+
 
 class Chunk(BaseModel):
     """