python3 -m venv ov20251 

source ov20251/bin/activate

pip install openvino

from openvino import get_version
print(get_version())

pip install openvino-genai

conda install ocl-icd-system

import argparse
import openvino_genai
from llm_config import SUPPORTED_LLM_MODELS, DEFAULT_SYSTEM_PROMPT, DEFAULT_SYSTEM_PROMPT_CHINESE

def streamer(subword):
    print(subword, end='', flush=True)
    return False

def apply_chat_template(prompt, system_prompt, language="English"):  # 確保參數(shù)名與變量名一致
    """
    使用自定義的聊天模板格式化提示詞
    Args:
        prompt: 用戶(hù)輸入的提示
        system_prompt: 系統(tǒng)提示
        language: 語(yǔ)言選擇
    Returns:
        formatted_prompt: 格式化后的提示
    """
    messages = [
        {"role": "system", "content": system_prompt},
        {"role": "user", "content": prompt}
    ]
    
    if language == "Chinese":
        template = SUPPORTED_LLM_MODELS["Chinese"]["DeepSeek-R1-Distill-Qwen-7B"]["genai_chat_template"]
    else:
        template = SUPPORTED_LLM_MODELS["English"]["DeepSeek-R1-Distill-Qwen-7B"]["genai_chat_template"]

    formatted_prompt = ""
    for i, message in enumerate(messages):
        if i == 0:
            formatted_prompt += "<｜begin of sentence｜>"
        if message["role"] == "system" and message["content"]:
            formatted_prompt += message["content"]
        elif message["role"] == "user":
            formatted_prompt += f"<｜User｜>{message['content']}"
        elif message["role"] == "assistant":
            formatted_prompt += f"<｜Assistant｜>{message['content']}<｜end of sentence｜>"
    
    return formatted_prompt

# 為 lang.value 添加默認(rèn)值
lang_value = "English"  # 默認(rèn)語(yǔ)言設(shè)置為 English

if __name__ == "__main__":
    parser = argparse.ArgumentParser(add_help=False)
    parser.add_argument('-h',
                        '--help',
                        action='help',
                        help='Show this help message and exit.')
    parser.add_argument('-m',
                        '--model_path',
                        required=True,
                        type=str,
                        help='Required. model path')
    parser.add_argument('-l',
                        '--max_sequence_length',
                        default=256,
                        required=False,
                        type=int,
                        help='Required. maximun length of output')
    parser.add_argument('-d',
                        '--device',
                        default='CPU',
                        required=False,
                        type=str,
                        help='Required. device for inference')
    args = parser.parse_args()

    try:
        print(f"正在加載模型: {args.model_path}")
        pipe = openvino_genai.LLMPipeline(args.model_path, args.device)
        
        
        # 設(shè)置生成配置
        config = openvino_genai.GenerationConfig()
        config.max_new_tokens = args.max_sequence_length
        config.stop_strings = set()  # 修正為空集合（原代碼會(huì)匹配空字符串）
        
        print("模型加載完成，開(kāi)始聊天...\n")
        
        pipe.start_chat()
        while True:
            try:
                prompt = input('問(wèn)題:\n')
                if not prompt.strip():
                    break
                    
                system_prompt = DEFAULT_SYSTEM_PROMPT_CHINESE if lang_value == "Chinese" else DEFAULT_SYSTEM_PROMPT
                formatted_prompt = apply_chat_template(prompt, system_prompt, language=lang_value)  # 顯式指定參數(shù)名
                pipe.get_tokenizer().set_chat_template("""
{% for message in messages %}
{% if loop.first %}{{ '<｜begin▁of▁sentence｜>' }}{% endif %}
{% if message['role'] == 'system' and message['content'] %}{{ message['content'] }}{% endif %}
{% if message['role'] == 'user' %}{{ '<｜User｜>' + message['content'] }}{% endif %}
{% if message['role'] == 'assistant' %}{{ '<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>' }}{% endif %}
{% if loop.last and add_generation_prompt and message['role'] != 'assistant' %}{{ '<｜Assistant｜>' }}{% endif %}
{% endfor %}
""")
                
                print("\n生成回答中...\n")
                pipe.generate(formatted_prompt, config, streamer)
                print('\n----------')
            except KeyboardInterrupt:
                print("\n對(duì)話(huà)被用戶(hù)中斷")
                break
            except Exception as e:
                print(f"\n錯(cuò)誤: {e}")
                continue
                
        pipe.finish_chat()
        
    except Exception as e:
        print(f"初始化失敗: {e}")

python3 chat_genai.py --model_path /DeepSeek-R1-Distill-Qwen-7B-sym-int4-ov --max_sequence_length 4096