text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=200,
    chunk_overlap=20
)
chunks = text_splitter.split_text(text)

results = COLLECTION.query(
    query_embeddings=query_embedding,
    n_results=5,
    include=['documents', 'metadatas']
)

# 檢測(cè)矛盾
conflict_detected = detect_conflicts(sources_for_conflict_detection)


# 獲取可信源
if conflict_detected:
    credible_sources = [s for s in sources_for_conflict_detection 
                      if s['type'] == 'web' and evaluate_source_credibility(s) > 0.7]

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=800,        # 增大到800-1000字符
    chunk_overlap=150,     # 增加重疊以保持連貫性
    separators=["\n\n", "\n", "。", "，", " "] # 優(yōu)先在自然段落分割
)

# 引入BM25關(guān)鍵詞檢索
from rank_bm25 import BM25Okapi
tokenized_corpus = [doc.split() for doc in corpus]
bm25 = BM25Okapi(tokenized_corpus)


# 混合檢索結(jié)果
semantic_results = COLLECTION.query(query_embeddings, n_results=7)
bm25_results = bm25.get_top_n(query.split(), corpus, n=7)
combined_results = hybrid_merge(semantic_results, bm25_results)

# 方案A: 使用交叉編碼器重排序
from sentence_transformers import CrossEncoder
reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
candidate_pairs = [[query, doc] for doc in retrieved_docs]
scores = reranker.predict(candidate_pairs)
reranked_results = [doc for _, doc in sorted(zip(scores, retrieved_docs), reverse=True)]


# 方案B: 使用LLM進(jìn)行相關(guān)性評(píng)分
def llm_rerank(query, docs, llm_client):
    results = []
    for doc in docs:
        prompt = f"問題: {query}\n文檔: {doc}\n這個(gè)文檔與問題的相關(guān)性是否高? 0-10分打分:"
        score = float(llm_client.generate(prompt).strip())
        results.append((score, doc))
    return [doc for _, doc in sorted(results, reverse=True)]

def recursive_retrieval(initial_query, max_iteratinotallow=3):
    query = initial_query
    all_contexts = []
    
    for i in range(max_iterations):
        # 當(dāng)前查詢的檢索結(jié)果
        current_results = retrieve_documents(query)
        all_contexts.extend(current_results)
        
        # 使用LLM分析還需要查詢什么
        next_query_prompt = f"""
        基于原始問題: {initial_query}
        以及已檢索信息: {summarize(current_results)}
        是否需要進(jìn)一步查詢? 如果需要,請(qǐng)?zhí)峁┬碌牟樵儐栴}:
        """
        
        next_query = llm_client.generate(next_query_prompt)
        if "不需要" in next_query or i == max_iterations-1:
            break
            
        query = next_query
        
    return all_contexts

def compress_context(query, documents, max_tokens=2000):
    compressed_docs = []
    
    for doc in documents:
        # 方案A: 使用map-reduce模式總結(jié)
        summary_prompt = f"原文: {doc}\n請(qǐng)?zhí)崛∨c問題'{query}'最相關(guān)的信息,總結(jié)在100詞以內(nèi):"
        compressed = llm_client.generate(summary_prompt)
        compressed_docs.append(compressed)
        
        # 方案B: 使用抽取式摘要選擇關(guān)鍵句子
        # sentences = split_into_sentences(doc)
        # scores = sentence_similarity(query, sentences)
        # top_sentences = [s for s, _ in sorted(zip(sentences, scores), key=lambda x: x[1], reverse=True)[:5]]
        # compressed_docs.append(" ".join(top_sentences))
    
    return compressed_docs

def create_advanced_prompt(query, context, question_type):
    if question_type == "factual":
        template = """
        你是一個(gè)精確的事實(shí)回答助手。以下是與問題相關(guān)的信息:
        {context}
        
        問題: {query}
        請(qǐng)基于以上信息提供精確的事實(shí)回答。如信息不足,請(qǐng)明確指出。請(qǐng)?jiān)诨卮鹉┪矘?biāo)明信息來源。
        """
    elif question_type == "analytical":
        template = """
        你是一個(gè)分析型助手。以下是相關(guān)參考信息:
        {context}
        
        問題: {query}
        請(qǐng)分析以上信息,提供深入見解。注意分析信息的一致性與可靠性,標(biāo)明不同來源間的差異。最后給出綜合結(jié)論并注明信息來源。
        """
    # 更多問題類型...
    
    # 添加思維鏈指導(dǎo)
    template += """
    思考步驟:
    1. 提取問題關(guān)鍵點(diǎn)
    2. 識(shí)別相關(guān)文檔中最有價(jià)值的信息
    3. 對(duì)比不同來源信息
    4. 形成清晰、全面的回答
    """
    
    return template.format(cnotallow=context, query=query)

# 豐富元數(shù)據(jù)
metadatas = [{
    "source": file_name,
    "doc_id": doc_id,
    "date_processed": datetime.now().isoformat(),
    "chunk_index": i,
    "total_chunks": len(chunks),
    "document_type": detect_document_type(text),
    "language": detect_language(chunk),
    "entities": extract_entities(chunk)
} for i, chunk in enumerate(chunks)]


# 檢索時(shí)使用元數(shù)據(jù)過濾
results = COLLECTION.query(
    query_embeddings=query_embedding,
    n_results=10,
    where={"document_type": {"$in": ["report", "research_paper"]}},
    include=['documents', 'metadatas']
)

# 升級(jí)為更強(qiáng)大的雙語模型
from sentence_transformers import SentenceTransformer
EMBED_MODEL = SentenceTransformer('BAAI/bge-large-zh-v1.5') # 1024維,中文效果更好


# 或考慮OpenAI嵌入模型
import openai
embeddings = openai.Embedding.create(
    input=chunks, 
    model="text-embedding-ada-002"
)

def evaluate_response(query, response, retrieved_docs):
    # LLM自評(píng)估
    evaluation_prompt = f"""
    問題: {query}
    回答: {response}
    請(qǐng)?jiān)u估這個(gè)回答的質(zhì)量(1-10分),考慮以下標(biāo)準(zhǔn):
    1. 準(zhǔn)確性: 回答是否符合檢索文檔的事實(shí)?
    2. 完整性: 回答是否全面覆蓋問題的各個(gè)方面?
    3. 相關(guān)性: 回答是否切中問題要點(diǎn)?
    4. 一致性: 回答內(nèi)部是否存在矛盾?
    請(qǐng)?jiān)敿?xì)說明評(píng)分理由。
    """
    
    feedback = llm_client.generate(evaluation_prompt)
    
    # 存儲(chǔ)評(píng)估結(jié)果用于系統(tǒng)優(yōu)化
    save_evaluation_result(query, response, retrieved_docs, feedback)
    
    return feedback

# 問題-檢索結(jié)果緩存
import hashlib
import pickle
from functools import lru_cache


@lru_cache(maxsize=100)
def cached_retrieval(query):
    # 計(jì)算查詢哈希
    query_hash = hashlib.md5(query.encode()).hexdigest()
    cache_file = f"cache/{query_hash}.pkl"
    
    # 檢查緩存
    try:
        with open(cache_file, 'rb') as f:
            return pickle.load(f)
    except FileNotFoundError:
        # 執(zhí)行檢索
        results = perform_retrieval(query)
        # 保存到緩存
        os.makedirs("cache", exist_ok=True)
        with open(cache_file, 'wb') as f:
            pickle.dump(results, f)
        return results