pip install transformers evaluate

import os
  os.environ['HTTP_PROXY'] = 'http://127.0.0.1:7890'
  os.environ['HTTPS_PROXY'] = 'http://127.0.0.1:7890'

import evaluate
google_bleu = evaluate.load("google_bleu")

sentence1 = "the cat sat on the mat"
sentence2 = "the cat ate the mat"
result1 = google_bleu.compute(predictinotallow=[sentence1], references=[[sentence2]])
print(result1)
# result1 {'google_bleu': 0.3333333333333333}


result2 = google_bleu.compute(predictinotallow=[sentence1], references=[[sentence1]])
print(result2)
# result2 {'google_bleu': 1.0}

predictions = ["The cat is on the mat."]
references = [["The cat is on the mat.", "There is a cat on the mat."]]
print(google_bleu.compute(predictinotallow=predictions, references=references))
>>> {'google_bleu': 1.0}

google_bleu.compute(
    predictinotallow=["我愛(ài)你"], 
    references=[["我愛(ài)我的祖國(guó)"]]
)
# >>> {'google_bleu': 0.0}

google_bleu.compute(
    predictinotallow=["我 愛(ài) 你"], 
    references=[["我 愛(ài) 我 的 祖 國(guó)"]]
)
# >>> {'google_bleu': 0.16666666666666666}

google_bleu.compute(
    predictinotallow=["我 愛(ài) 你"], 
    references=[["我 愛(ài) 我 的 祖國(guó)"]]
)
# >>> {'google_bleu': 0.21428571428571427}

!pip install modelscope
from modelscope import AutoModel
from numpy.linalg import norm

cos_sim = lambda a,b: (a @ b.T) / (norm(a)*norm(b))
# trust_remote_code is needed to use the encode method
model = AutoModel.from_pretrained('jinaai/jina-embeddings-v2-base-zh', trust_remote_code=True) 
embeddings = model.encode(['How is the weather today?', '今天天氣怎么樣?'])
print(cos_sim(embeddings[0], embeddings[1]))

import numpy as np
from numpy.linalg import norm
from modelscope import AutoModel

# 定義余弦相似度計(jì)算函數(shù)
cos_sim = lambda a, b: (a @ b.T) / (norm(a) * norm(b))


# 加載模型
model = AutoModel.from_pretrained('jinaai/jina-embeddings-v2-base-zh', trust_remote_code=True)

# 輸入字符串和候選字符串
input_string = 'How is the weather today?'
candidates = ['今天天氣怎么樣?', '我今天很高興', '天氣預(yù)報(bào)說(shuō)今天會(huì)下雨', '你最喜歡的顏色是什么?']

# 計(jì)算輸入字符串的嵌入向量
input_embedding = model.encode([input_string])[0]

# 計(jì)算候選字符串的嵌入向量
candidate_embeddings = model.encode(candidates)

# 計(jì)算相似度并排序
similarities = [cos_sim(input_embedding, candidate_embedding) for candidate_embedding in candidate_embeddings]
sorted_candidates = sorted(zip(candidates, similarities), key=lambda x: x[1], reverse=True)

# 輸出排序結(jié)果
for candidate, similarity in sorted_candidates:
    print(f"({input_string} - {candidate}), Similarity: {similarity:.4f}")

Downloading Model to directory: C:\Users\user_name\.cache\modelscope\hub\jinaai/jina-embeddings-v2-base-zh
(How is the weather today? - 今天天氣怎么樣?), Similarity: 0.7861
(How is the weather today? - 天氣預(yù)報(bào)說(shuō)今天會(huì)下雨), Similarity: 0.5470
(How is the weather today? - 我今天很高興), Similarity: 0.4202
(How is the weather today? - 你最喜歡的顏色是什么?), Similarity: 0.1032

JUDGE_PROMPT = """
You will be given a user_question and system_answer couple.
Your task is to provide a 'total rating' scoring how well the system_answer answers the user concerns expressed in the user_question.
Give your answer as a float on a scale of 0 to 10, where 0 means that the system_answer is not helpful at all, and 10 means that the answer completely and helpfully addresses the question.

Provide your feedback as follows:

Feedback:::
Total rating: (your rating, as a float between 0 and 10)

Now here are the question and answer.

Question: {question}
Answer: {answer}

Feedback:::
Total rating: """