计算文本相似度，输出相似度最高的n个

TF-IDF

import jieba
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# 示例句子列表
sentences = [
    "今天天气真好，阳光明媚。",
    "关键字匹配是一种常见的文本处理任务。",
    "计算机不认识人类语言，要转成词向量。",
    "富强、民主、文明、和谐、自由、平等、公正、法治、爱国、敬业、诚信、友善。",
    "中文分词工具对文本处理很有帮助。",
]

# 输入的查询句子
query_sentence = "关键字匹配和文本处理任务"

# 创建DataFrame
data = {"Sentence": sentences}
df = pd.DataFrame(data)

# 分词并建立TF-IDF特征向量
def preprocess(text):
    words = jieba.lcut(text)
    return " ".join(words)

df["Preprocessed_Sentence"] = df["Sentence"].apply(preprocess)
query_sentence = preprocess(query_sentence)

# 计算相似度
vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(list(df["Preprocessed_Sentence"]) + [query_sentence])
similarities = cosine_similarity(tfidf_matrix)
query_similarity = similarities[-1, :-1]  # 最后一行是查询句子

# 获取相似度最高的n个句子
n = 10  # 想要获取的相似句子数量
top_indices = query_similarity.argsort()[-n:][::-1]

# 创建包含相似句子和相似度的DataFrame
similar_sentences = df.loc[top_indices, "Sentence"].tolist()
similarity_scores = [query_similarity[i] for i in top_indices]

result_data = {"Similar_Sentence": similar_sentences, "Similarity_Score": similarity_scores}
result_df = pd.DataFrame(result_data)

print("查询句子:", query_sentence)
print("\n相似度最高的句子:")
result_df

Spacy

import warnings
warnings.filterwarnings("ignore")
import spacy
import pandas as pd

# 加载Spacy中文语言模型
nlp = spacy.load("zh_core_web_sm")

# 示例中文句子数据框
data = {
    "Sentences": [
    "今天天气真好，阳光明媚。",
    "关键字匹配是一种常见的文本处理任务。",
    "计算机不认识人类语言，要转成词向量。",
    "富强、民主、文明、和谐、自由、平等、公正、法治、爱国、敬业、诚信、友善。",
    "中文分词工具对文本处理很有帮助。",
    ]
}
df = pd.DataFrame(data)

# 选择要比较的目标句子
target_sentence = "关键字匹配和文本处理任务"

# 计算所有句子的相似度
similarity_scores = []
for sentence in df["Sentences"]:
    doc1 = nlp(target_sentence)
    doc2 = nlp(sentence)
    similarity = doc1.similarity(doc2)
    similarity_scores.append(similarity)

# 将相似度得分添加到数据框中
df["Similarity Score"] = similarity_scores

# 根据相似度得分降序排序，选择相似度最高的n个句子
n = 10  # 选择前n个最相似的句子
top_n_similar_sentences = df.sort_values(by="Similarity Score", ascending=False).head(n)

# 打印DataFrame
top_n_similar_sentences

Bert

import warnings
warnings.filterwarnings("ignore")
from transformers import AutoTokenizer, AutoModel
import pandas as pd
import torch
from sklearn.metrics.pairwise import cosine_similarity

# 加载BERT模型和分词器
model_name = "model/bert-base-chinese"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

# 示例中文句子数据框
data = {
    "Sentences": [
    "今天天气真好，阳光明媚。",
    "关键字匹配是一种常见的文本处理任务。",
    "计算机不认识人类语言，要转成词向量。",
    "富强、民主、文明、和谐、自由、平等、公正、法治、爱国、敬业、诚信、友善。",
    "中文分词工具对文本处理很有帮助。",
    ]
}
df = pd.DataFrame(data)

# 选择要比较的目标句子
target_sentence = "关键字匹配和文本处理任务"

# 计算所有句子的相似度
similarity_scores = []
target_embedding = model(**tokenizer(target_sentence, return_tensors="pt", padding=True, truncation=True)).last_hidden_state.mean(dim=1)

for sentence in df["Sentences"]:
    # 使用分词器编码句子
    sentence_embedding = model(**tokenizer(sentence, return_tensors="pt", padding=True, truncation=True)).last_hidden_state.mean(dim=1)
    
    # 计算余弦相似度
    similarity = cosine_similarity(target_embedding.detach().numpy(), sentence_embedding.detach().numpy())[0][0]
    similarity_scores.append(similarity)

# 将相似度得分添加到数据框中
df["Similarity Score"] = similarity_scores

# 根据相似度得分降序排序，选择相似度最高的n个句子
n = 10  # 选择前两个最相似的句子
top_n_similar_sentences = df.sort_values(by="Similarity Score", ascending=False).head(n)

# 打印DataFrame
top_n_similar_sentences