|
@@ -0,0 +1,341 @@
|
|
|
+{
|
|
|
+ "cells": [
|
|
|
+ {
|
|
|
+ "cell_type": "markdown",
|
|
|
+ "id": "33d34c29",
|
|
|
+ "metadata": {},
|
|
|
+ "source": [
|
|
|
+ "### 评价情感判断"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "id": "ca680f71",
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "# 导入所需库\n",
|
|
|
+ "from openai import OpenAI\n",
|
|
|
+ "from dotenv import load_dotenv \n",
|
|
|
+ "from IPython.display import display, HTML\n",
|
|
|
+ "import pandas as pd\n",
|
|
|
+ "import json\n",
|
|
|
+ "import os\n",
|
|
|
+ "load_dotenv()\n",
|
|
|
+ "# 用openAI client 调用模型\n",
|
|
|
+ "\n",
|
|
|
+ "client = OpenAI(\n",
|
|
|
+ " base_url=os.getenv(\"BAILIAN_API_BASE_URL\"),\n",
|
|
|
+ " api_key=os.getenv(\"BAILIAN_API_KEY\")\n",
|
|
|
+ ")\n"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "markdown",
|
|
|
+ "id": "f9c40826",
|
|
|
+ "metadata": {},
|
|
|
+ "source": [
|
|
|
+ "**用于大模型的函数调用**"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "id": "fe3620f8",
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "# 用于function calling的实践\n",
|
|
|
+ "def handle_positive_sentiment(reason, result):\n",
|
|
|
+ " \"\"\"处理积极情感的方法\n",
|
|
|
+ " Args:\n",
|
|
|
+ " reason: 大模型给出的积极判断原因\n",
|
|
|
+ " result: 大模型返回的完整结果\n",
|
|
|
+ " \"\"\"\n",
|
|
|
+ " print(\"这是一个积极的评价!\")\n",
|
|
|
+ " print(f\"判断原因: {reason}\")\n",
|
|
|
+ " print(f\"完整结果: {result}\")\n",
|
|
|
+ " \n",
|
|
|
+ "def handle_negative_sentiment(reason, result):\n",
|
|
|
+ " \"\"\"处理消极情感的方法\n",
|
|
|
+ " Args:\n",
|
|
|
+ " reason: 大模型给出的消极判断原因\n",
|
|
|
+ " result: 大模型返回的完整结果\n",
|
|
|
+ " \"\"\" \n",
|
|
|
+ " print(\"这是一个消极的评价!\")\n",
|
|
|
+ " print(f\"判断原因: {reason}\")\n",
|
|
|
+ " print(f\"完整结果: {result}\")\n",
|
|
|
+ " "
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "markdown",
|
|
|
+ "id": "f1fb34ca",
|
|
|
+ "metadata": {},
|
|
|
+ "source": [
|
|
|
+ "**定义预测方法**"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "id": "e874b885",
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "#预测方法\n",
|
|
|
+ "def predict_sentiment(text,model=\"qwen3-30b-a3b\"):\n",
|
|
|
+ " # 定义提示词\n",
|
|
|
+ " system_message = \"\"\"You are a sentiment analysis expert. Please analyze the sentiment tendency of the provided review text.\"\"\"\n",
|
|
|
+ " user_message = f\"\"\"\n",
|
|
|
+ " Please analyze the sentiment tendency (positive or negative) of the following review text (provided within <>), and return the result in JSON format.\n",
|
|
|
+ " Review text: <{text}>\n",
|
|
|
+ "\n",
|
|
|
+ " Please only return a JSON containing the following fields:\n",
|
|
|
+ " - sentiment: The sentiment tendency (positive or negative)\n",
|
|
|
+ " - reason: The reason for the sentiment tendency\n",
|
|
|
+ " \"\"\"\n",
|
|
|
+ "\n",
|
|
|
+ " # 定义工具用以大模型调用\n",
|
|
|
+ " tools=[]\n",
|
|
|
+ " \n",
|
|
|
+ " # 调用大模型进行情感分析\n",
|
|
|
+ " response = client.chat.completions.create(\n",
|
|
|
+ " model=model , # 如果model参数为空则使用默认值\n",
|
|
|
+ " messages=[\n",
|
|
|
+ " # 添加系统提示词\n",
|
|
|
+ " {\"role\": \"system\", \"content\": system_message},\n",
|
|
|
+ " # TODO few-shot/one-shot 来增强表现的实践\n",
|
|
|
+ " # {\"role\":\"user\", \"content\": \"\"\"Once again Mr. Costner has dragged out a movie for far longer than necessary. \n",
|
|
|
+ " # Aside from the terrific sea rescue sequences, of which there are very few I just did not care about any of the characters. \n",
|
|
|
+ " # Most of us have ghosts in the closet, and Costner's character are realized early on, and then forgotten until much later, \n",
|
|
|
+ " # by which time I did not care. The character we should really care about is a very cocky, overconfident Ashton Kutcher. \n",
|
|
|
+ " # The problem is he comes off as kid who thinks he's better than anyone else around him and shows no signs of a cluttered closet.\n",
|
|
|
+ " # His only obstacle appears to be winning over Costner. \n",
|
|
|
+ " # Finally when we are well past the half way point of this stinker, Costner tells us all about Kutcher's ghosts. \n",
|
|
|
+ " # We are told why Kutcher is driven to be the best with no prior inkling or foreshadowing. \n",
|
|
|
+ " # No magic here, it was all I could do to keep from turning it off an hour in.\"\"\"},\n",
|
|
|
+ " # {\"role\":\"assistant\", \"content\": \"\"},\n",
|
|
|
+ " {\"role\": \"user\", \"content\": user_message}\n",
|
|
|
+ " ],\n",
|
|
|
+ " # Qwen3模型通过enable_thinking参数控制思考过程(开源版默认True,商业版默认False)\n",
|
|
|
+ " # 使用Qwen3开源版模型时,若未启用流式输出,请将下行取消注释,否则会报错\n",
|
|
|
+ " extra_body={\"enable_thinking\": False},\n",
|
|
|
+ "\n",
|
|
|
+ " #tools=tools,\n",
|
|
|
+ " temperature=0.3,\n",
|
|
|
+ " response_format={\"type\": \"json_object\"} # 指定返回JSON格式\n",
|
|
|
+ " )\n",
|
|
|
+ " \n",
|
|
|
+ " # 获取返回结果\n",
|
|
|
+ " result = response.choices[0].message.content\n",
|
|
|
+ " return result\n",
|
|
|
+ "\n",
|
|
|
+ "#测试\n",
|
|
|
+ "print(predict_sentiment(\"你好!我很感谢你\"))"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "id": "df14f73d",
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "#文件读取方法\n",
|
|
|
+ "def load_data():\n",
|
|
|
+ " # 定义数据文件夹路径\n",
|
|
|
+ " data_dir = \"../data/acllmdb_sentiment_small\"\n",
|
|
|
+ "\n",
|
|
|
+ " # 读取正面评价数据\n",
|
|
|
+ " positive_dir = os.path.join(data_dir, \"positive\")\n",
|
|
|
+ " positive_files = os.listdir(positive_dir)\n",
|
|
|
+ " positive_texts = []\n",
|
|
|
+ " for file in positive_files:\n",
|
|
|
+ " with open(os.path.join(positive_dir, file), 'r', encoding='utf-8') as f:\n",
|
|
|
+ " text = f.read()\n",
|
|
|
+ " positive_texts.append({'text': text, 'sentiment': 'positive'})\n",
|
|
|
+ "\n",
|
|
|
+ " # 读取负面评价数据 \n",
|
|
|
+ " negative_dir = os.path.join(data_dir, \"negative\") \n",
|
|
|
+ " negative_files = os.listdir(negative_dir)\n",
|
|
|
+ " negative_texts = []\n",
|
|
|
+ " for file in negative_files:\n",
|
|
|
+ " with open(os.path.join(negative_dir, file), 'r', encoding='utf-8') as f:\n",
|
|
|
+ " text = f.read()\n",
|
|
|
+ " negative_texts.append({'text': text, 'sentiment': 'negative'})\n",
|
|
|
+ "\n",
|
|
|
+ " # 合并数据并创建DataFrame\n",
|
|
|
+ " df = pd.DataFrame(positive_texts + negative_texts)\n",
|
|
|
+ " print(f\"总共读取了 {len(df)} 条评价数据\")\n",
|
|
|
+ " print(f\"其中正面评价 {len(positive_texts)} 条,负面评价 {len(negative_texts)} 条\")\n",
|
|
|
+ " return df"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "id": "b6c31df9",
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "# 批量预测情感倾向\n",
|
|
|
+ "def predict_sentiment_batch(model):\n",
|
|
|
+ " \n",
|
|
|
+ " try:\n",
|
|
|
+ " # 数据加载\n",
|
|
|
+ " data_to_predict = load_data()\n",
|
|
|
+ " if data_to_predict.empty:\n",
|
|
|
+ " #加载失败\n",
|
|
|
+ " print(\"数据加载为空,请检查数据文件路径是否正确\")\n",
|
|
|
+ " else:\n",
|
|
|
+ " #加载成功\n",
|
|
|
+ " predictions = []\n",
|
|
|
+ "\n",
|
|
|
+ " print(f\"开始进行情感预测...(model={model})\")\n",
|
|
|
+ " # 遍历每条评论数据\n",
|
|
|
+ " for i in range(len(data_to_predict)):\n",
|
|
|
+ " row = data_to_predict.iloc[i]\n",
|
|
|
+ " text = row['text']\n",
|
|
|
+ " true_sentiment=row['sentiment']\n",
|
|
|
+ " try:\n",
|
|
|
+ " # 调用大模型进行预测当前行\n",
|
|
|
+ " result = predict_sentiment(text,model)\n",
|
|
|
+ " #得到json格式结果\n",
|
|
|
+ " if result is not None:\n",
|
|
|
+ " result_dict = json.loads(result)\n",
|
|
|
+ " predicted_sentiment = result_dict.get('sentiment', 'unknown')\n",
|
|
|
+ " reason = result_dict.get('reason', 'unknown')\n",
|
|
|
+ " \n",
|
|
|
+ " # 保存预测结果\n",
|
|
|
+ " predictions.append({\n",
|
|
|
+ " 'text': text,\n",
|
|
|
+ " 'true_sentiment': true_sentiment,\n",
|
|
|
+ " 'predicted_sentiment': predicted_sentiment,\n",
|
|
|
+ " 'reason':reason\n",
|
|
|
+ " })\n",
|
|
|
+ " \n",
|
|
|
+ " # 打印进度 每完成三十条打印一次\n",
|
|
|
+ " if i%30 == 0 or i == len(data_to_predict)-1 :\n",
|
|
|
+ " print(f\"已完成 {len(predictions)}/{len(data_to_predict)} 条预测\")\n",
|
|
|
+ "\n",
|
|
|
+ " except Exception as e:\n",
|
|
|
+ " print(f\"第 {i + 1} 条数据预测失败: {str(e)}\")\n",
|
|
|
+ " continue\n",
|
|
|
+ " \n",
|
|
|
+ " # 将预测结果转换为DataFrame\n",
|
|
|
+ " predictions_df = pd.DataFrame(predictions)\n",
|
|
|
+ " print(f\"成功预测 {len(predictions_df)} 条数据\")\n",
|
|
|
+ " return predictions_df\n",
|
|
|
+ " except Exception as e:\n",
|
|
|
+ " print(f\"数据加载出错: {str(e)}\")\n"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "markdown",
|
|
|
+ "id": "e56e5bd3",
|
|
|
+ "metadata": {},
|
|
|
+ "source": [
|
|
|
+ "**比较以下大模型在这个任务上的 accuracy 差异**"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "id": "f99ac855",
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "# 计算预测准确率\n",
|
|
|
+ "# 解决思路:\n",
|
|
|
+ "# 1. 遍历不同的模型名称列表 models\n",
|
|
|
+ "# 2. 对每个模型调用 predict_sentiment_batch() 获取预测结果\n",
|
|
|
+ "# 3. 计算每个模型的预测准确率:\n",
|
|
|
+ "# - 使用 predictions_df 中的 true_sentiment 和 predicted_sentiment 列进行比较\n",
|
|
|
+ "# - 使用 == 运算符比较两列的值是否相等\n",
|
|
|
+ "# - 使用 mean() 计算相等的比例得到准确率\n",
|
|
|
+ "# 4. 将每个模型的准确率结果以百分比格式打印输出\n",
|
|
|
+ "# 5. 对比不同模型的准确率,分析性能差异\n",
|
|
|
+ "\n",
|
|
|
+ "# 定义要测试的模型列表\n",
|
|
|
+ "models = [\"qwen3-32b\", \"qwen3-30b-a3b\", \"qwen3-0.6b\"]\n",
|
|
|
+ "\n",
|
|
|
+ "# 存储每个模型的准确率结果\n",
|
|
|
+ "model_accuracies = {}\n",
|
|
|
+ "\n",
|
|
|
+ "# 遍历每个模型进行预测和评估\n",
|
|
|
+ "for model_using in models:\n",
|
|
|
+ " predictions_df=predict_sentiment_batch(model_using)\n",
|
|
|
+ " if predictions_df is not None and len(predictions_df) > 0:\n",
|
|
|
+ " # 计算准确率\n",
|
|
|
+ " accuracy = (predictions_df['true_sentiment'] == predictions_df['predicted_sentiment']).mean()\n",
|
|
|
+ " model_accuracies[model_using] = accuracy\n",
|
|
|
+ " \n",
|
|
|
+ " # 打印当前模型的准确率\n",
|
|
|
+ " print(f\"模型 {model_using} 的预测准确率: {accuracy:.2%}\")\n",
|
|
|
+ " \n",
|
|
|
+ " # 统计错误预测的样本\n",
|
|
|
+ " wrong_predictions = predictions_df[predictions_df['true_sentiment'] != predictions_df['predicted_sentiment']]\n",
|
|
|
+ " print(f\"错误预测数量: {len(wrong_predictions)}/{len(predictions_df)}\")\n",
|
|
|
+ " \n",
|
|
|
+ " # # 显示一些错误预测的例子\n",
|
|
|
+ " # if len(wrong_predictions) > 0:\n",
|
|
|
+ " # print(\"\\n错误预测示例:\")\n",
|
|
|
+ " # for _, row in wrong_predictions.head(3).iterrows():\n",
|
|
|
+ " # print(f\"文本: {row['text']}\")\n",
|
|
|
+ " # print(f\"真实情感: {row['true_sentiment']}\")\n",
|
|
|
+ " # print(f\"预测情感: {row['predicted_sentiment']}\")\n",
|
|
|
+ " # print(f\"预测理由: {row['reason']}\\n\")\n",
|
|
|
+ " else:\n",
|
|
|
+ " print(f\"模型 {model_using} 预测结果为空\")\n",
|
|
|
+ "\n",
|
|
|
+ "# 比较不同模型的性能\n",
|
|
|
+ "if model_accuracies:\n",
|
|
|
+ " print(\"\\n模型性能对比:\")\n",
|
|
|
+ " best_model = max(model_accuracies.items(), key=lambda x: x[1])\n",
|
|
|
+ " print(f\"最佳模型: {best_model[0]}, 准确率: {best_model[1]:.2%}\")\n",
|
|
|
+ " \n",
|
|
|
+ "# import matplotlib.pyplot as plt\n",
|
|
|
+ "# # 绘制准确率对比图\n",
|
|
|
+ "# plt.figure(figsize=(10, 6))\n",
|
|
|
+ "# plt.bar(model_accuracies.keys(), [acc * 100 for acc in model_accuracies.values()])\n",
|
|
|
+ "# plt.title('不同模型的准确率对比')\n",
|
|
|
+ "# plt.xlabel('模型')\n",
|
|
|
+ "# plt.ylabel('准确率 (%)')\n",
|
|
|
+ "# plt.xticks(rotation=45)\n",
|
|
|
+ "# plt.tight_layout()\n",
|
|
|
+ "# plt.show()\n"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": null,
|
|
|
+ "id": "47872e36",
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": []
|
|
|
+ }
|
|
|
+ ],
|
|
|
+ "metadata": {
|
|
|
+ "kernelspec": {
|
|
|
+ "display_name": ".venv",
|
|
|
+ "language": "python",
|
|
|
+ "name": "python3"
|
|
|
+ },
|
|
|
+ "language_info": {
|
|
|
+ "codemirror_mode": {
|
|
|
+ "name": "ipython",
|
|
|
+ "version": 3
|
|
|
+ },
|
|
|
+ "file_extension": ".py",
|
|
|
+ "mimetype": "text/x-python",
|
|
|
+ "name": "python",
|
|
|
+ "nbconvert_exporter": "python",
|
|
|
+ "pygments_lexer": "ipython3",
|
|
|
+ "version": "3.11.13"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "nbformat": 4,
|
|
|
+ "nbformat_minor": 5
|
|
|
+}
|