{ "cells": [ { "cell_type": "markdown", "id": "33d34c29", "metadata": {}, "source": [ "### 评价情感判断" ] }, { "cell_type": "code", "execution_count": 23, "id": "ca680f71", "metadata": {}, "outputs": [], "source": [ "# 导入所需库\n", "from openai import OpenAI\n", "from dotenv import load_dotenv \n", "from IPython.display import display, HTML\n", "import pandas as pd\n", "import json\n", "import os\n", "load_dotenv()\n", "# 用openAI client 调用模型\n", "\n", "client = OpenAI(\n", " base_url=os.getenv(\"BAILIAN_API_BASE_URL\"),\n", " api_key=os.getenv(\"BAILIAN_API_KEY\")\n", ")\n" ] }, { "cell_type": "markdown", "id": "f9c40826", "metadata": {}, "source": [ "**用于大模型的函数调用**" ] }, { "cell_type": "code", "execution_count": 24, "id": "fe3620f8", "metadata": {}, "outputs": [], "source": [ "# 用于function calling的实践\n", "def handle_positive_sentiment(reason, result):\n", " \n", " print(\"这是一个积极的评价!\")\n", " print(f\"判断原因: {reason}\")\n", " print(f\"完整结果: {result}\")\n", " \n", "def handle_negative_sentiment(reason, result):\n", " \n", " print(\"这是一个消极的评价!\")\n", " print(f\"判断原因: {reason}\")\n", " print(f\"完整结果: {result}\")\n", " " ] }, { "cell_type": "markdown", "id": "f1fb34ca", "metadata": {}, "source": [ "**定义预测方法**" ] }, { "cell_type": "code", "execution_count": null, "id": "e874b885", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"sentiment\": \"positive\",\n", " \"reason\": \"The text expresses gratitude and a positive emotion.\"\n", "}\n" ] } ], "source": [ "#预测方法\n", "def predict_sentiment(text,model=\"qwen3-30b-a3b\"):\n", " # 定义提示词\n", " system_message = \"\"\"You are a sentiment analysis expert. Please analyze the sentiment tendency of the provided review text.\"\"\"\n", " user_message = f\"\"\"\n", " Please analyze the sentiment tendency (positive or negative) of the following review text (provided within <>), and return the result in JSON format.\n", " Review text: <{text}>\n", "\n", " Please only return a JSON containing the following fields:\n", " - sentiment: The sentiment tendency (positive or negative)\n", " - reason: The reason for the sentiment tendency\n", " \"\"\"\n", "\n", " # TODO 定义工具用以大模型调用\n", " tools=[{\n", " \"type\": \"function\",\n", " \"function\": {\n", " \"name\": \"handle_positive_sentiment\",\n", " \"description\": \"当你判断一段文本的情感倾向为积极positive的时候,请使用这个方法\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"reason\": {\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为积极的原因\"\n", " },\n", " \"result\":{\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为积极的结果\",\n", " \"enum\": [\"positive\", \"negative\"]\n", " }\n", " },\n", " \"required\": [\"reason\", \"result\"],\n", " }\n", " }\n", " },{\n", " \"type\": \"function\",\n", " \"function\": {\n", " \"name\": \"handle_positive_sentiment\",\n", " \"description\": \"当你判断一段文本的情感倾向为消极negative的时候,请使用这个方法\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"reason\": {\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为积极的原因\"\n", " },\n", " \"result\":{\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为积极的结果\",\n", " \"enum\": [\"positive\", \"negative\"]\n", " }\n", " },\n", " \"required\": [\"reason\", \"result\"],\n", " }\n", " }\n", " }]\n", " \n", " # 调用大模型进行情感分析\n", " response = client.chat.completions.create(\n", " model=model , # 如果model参数为空则使用默认值\n", " messages=[\n", " # 添加系统提示词\n", " {\"role\": \"system\", \"content\": system_message},\n", " # TODO few-shot/one-shot 来增强表现的实践\n", " # {\"role\":\"user\", \"content\": \"\"\"Once again Mr. Costner has dragged out a movie for far longer than necessary. \n", " # Aside from the terrific sea rescue sequences, of which there are very few I just did not care about any of the characters. \n", " # Most of us have ghosts in the closet, and Costner's character are realized early on, and then forgotten until much later, \n", " # by which time I did not care. The character we should really care about is a very cocky, overconfident Ashton Kutcher. \n", " # The problem is he comes off as kid who thinks he's better than anyone else around him and shows no signs of a cluttered closet.\n", " # His only obstacle appears to be winning over Costner. \n", " # Finally when we are well past the half way point of this stinker, Costner tells us all about Kutcher's ghosts. \n", " # We are told why Kutcher is driven to be the best with no prior inkling or foreshadowing. \n", " # No magic here, it was all I could do to keep from turning it off an hour in.\"\"\"},\n", " # {\"role\":\"assistant\", \"content\": \"\"},\n", " {\"role\": \"user\", \"content\": user_message}\n", " ],\n", " # Qwen3模型通过enable_thinking参数控制思考过程(开源版默认True,商业版默认False)\n", " # 使用Qwen3开源版模型时,若未启用流式输出,请将下行取消注释,否则会报错\n", " extra_body={\"enable_thinking\": False},\n", "\n", " tools=tools,\n", " temperature=0.3,\n", " response_format={\"type\": \"json_object\"} # 指定返回JSON格式\n", " )\n", " # 获取返回结果\n", " result = response.choices[0].message.content\n", " return result\n", "\n", "#测试\n", "print(predict_sentiment(\"你好!我很感谢你\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "df14f73d", "metadata": {}, "outputs": [], "source": [ "#文件读取方法\n", "def load_data():\n", " # 定义数据文件夹路径\n", " data_dir = \"../data/acllmdb_sentiment_small\"\n", "\n", " # 读取正面评价数据\n", " positive_dir = os.path.join(data_dir, \"positive\")\n", " positive_files = os.listdir(positive_dir)\n", " positive_texts = []\n", " for file in positive_files:\n", " with open(os.path.join(positive_dir, file), 'r', encoding='utf-8') as f:\n", " text = f.read()\n", " positive_texts.append({'text': text, 'sentiment': 'positive'})\n", "\n", " # 读取负面评价数据 \n", " negative_dir = os.path.join(data_dir, \"negative\") \n", " negative_files = os.listdir(negative_dir)\n", " negative_texts = []\n", " for file in negative_files:\n", " with open(os.path.join(negative_dir, file), 'r', encoding='utf-8') as f:\n", " text = f.read()\n", " negative_texts.append({'text': text, 'sentiment': 'negative'})\n", "\n", " # 合并数据并创建DataFrame\n", " df = pd.DataFrame(positive_texts + negative_texts)\n", " print(f\"总共读取了 {len(df)} 条评价数据\")\n", " print(f\"其中正面评价 {len(positive_texts)} 条,负面评价 {len(negative_texts)} 条\")\n", " return df" ] }, { "cell_type": "code", "execution_count": null, "id": "b6c31df9", "metadata": {}, "outputs": [], "source": [ "# 批量预测情感倾向\n", "def predict_sentiment_batch(model):\n", " \n", " try:\n", " # 数据加载\n", " data_to_predict = load_data()\n", " if data_to_predict.empty:\n", " #加载失败\n", " print(\"数据加载为空,请检查数据文件路径是否正确\")\n", " else:\n", " #加载成功\n", " predictions = []\n", "\n", " print(f\"开始进行情感预测...(model={model})\")\n", " # 遍历每条评论数据\n", " for i in range(len(data_to_predict)):\n", " row = data_to_predict.iloc[i]\n", " text = row['text']\n", " true_sentiment=row['sentiment']\n", " try:\n", " # 调用大模型进行预测当前行\n", " result = predict_sentiment(text,model)\n", " #得到json格式结果\n", " if result is not None:\n", " result_dict = json.loads(result)\n", " predicted_sentiment = result_dict.get('sentiment', 'unknown')\n", " reason = result_dict.get('reason', 'unknown')\n", " \n", " # 保存预测结果\n", " predictions.append({\n", " 'text': text,\n", " 'true_sentiment': true_sentiment,\n", " 'predicted_sentiment': predicted_sentiment,\n", " 'reason':reason\n", " })\n", " \n", " # 打印进度 每完成三十条打印一次\n", " if i%30 == 0 or i == len(data_to_predict)-1 :\n", " print(f\"已完成 {len(predictions)}/{len(data_to_predict)} 条预测\")\n", "\n", " except Exception as e:\n", " print(f\"第 {i + 1} 条数据预测失败: {str(e)}\")\n", " continue\n", " \n", " # 将预测结果转换为DataFrame\n", " predictions_df = pd.DataFrame(predictions)\n", " print(f\"成功预测 {len(predictions_df)} 条数据\")\n", " return predictions_df\n", " except Exception as e:\n", " print(f\"数据加载出错: {str(e)}\")\n" ] }, { "cell_type": "markdown", "id": "e56e5bd3", "metadata": {}, "source": [ "**比较以下大模型在这个任务上的 accuracy 差异**" ] }, { "cell_type": "code", "execution_count": null, "id": "f99ac855", "metadata": {}, "outputs": [], "source": [ "# 计算预测准确率\n", "# 解决思路:\n", "# 1. 遍历不同的模型名称列表 models\n", "# 2. 对每个模型调用 predict_sentiment_batch() 获取预测结果\n", "# 3. 计算每个模型的预测准确率:\n", "# - 使用 predictions_df 中的 true_sentiment 和 predicted_sentiment 列进行比较\n", "# - 使用 == 运算符比较两列的值是否相等\n", "# - 使用 mean() 计算相等的比例得到准确率\n", "# 4. 将每个模型的准确率结果以百分比格式打印输出\n", "# 5. 对比不同模型的准确率,分析性能差异\n", "\n", "# 定义要测试的模型列表\n", "models = [\"qwen3-32b\", \"qwen3-30b-a3b\", \"qwen3-0.6b\"]\n", "\n", "# 存储每个模型的准确率结果\n", "model_accuracies = {}\n", "\n", "# 遍历每个模型进行预测和评估\n", "for model_using in models:\n", " predictions_df=predict_sentiment_batch(model_using)\n", " if predictions_df is not None and len(predictions_df) > 0:\n", " # 计算准确率\n", " accuracy = (predictions_df['true_sentiment'] == predictions_df['predicted_sentiment']).mean()\n", " model_accuracies[model_using] = accuracy\n", " \n", " # 打印当前模型的准确率\n", " print(f\"模型 {model_using} 的预测准确率: {accuracy:.2%}\")\n", " \n", " # 统计错误预测的样本\n", " wrong_predictions = predictions_df[predictions_df['true_sentiment'] != predictions_df['predicted_sentiment']]\n", " print(f\"错误预测数量: {len(wrong_predictions)}/{len(predictions_df)}\")\n", " \n", " # # 显示一些错误预测的例子\n", " # if len(wrong_predictions) > 0:\n", " # print(\"\\n错误预测示例:\")\n", " # for _, row in wrong_predictions.head(3).iterrows():\n", " # print(f\"文本: {row['text']}\")\n", " # print(f\"真实情感: {row['true_sentiment']}\")\n", " # print(f\"预测情感: {row['predicted_sentiment']}\")\n", " # print(f\"预测理由: {row['reason']}\\n\")\n", " else:\n", " print(f\"模型 {model_using} 预测结果为空\")\n", "\n", "# 比较不同模型的性能\n", "if model_accuracies:\n", " print(\"\\n模型性能对比:\")\n", " best_model = max(model_accuracies.items(), key=lambda x: x[1])\n", " print(f\"最佳模型: {best_model[0]}, 准确率: {best_model[1]:.2%}\")\n", " \n", "# import matplotlib.pyplot as plt\n", "# # 绘制准确率对比图\n", "# plt.figure(figsize=(10, 6))\n", "# plt.bar(model_accuracies.keys(), [acc * 100 for acc in model_accuracies.values()])\n", "# plt.title('不同模型的准确率对比')\n", "# plt.xlabel('模型')\n", "# plt.ylabel('准确率 (%)')\n", "# plt.xticks(rotation=45)\n", "# plt.tight_layout()\n", "# plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "47872e36", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.13" } }, "nbformat": 4, "nbformat_minor": 5 }