{ "cells": [ { "cell_type": "markdown", "id": "33d34c29", "metadata": {}, "source": [ "### 评价情感判断" ] }, { "cell_type": "code", "execution_count": 23, "id": "ca680f71", "metadata": {}, "outputs": [], "source": [ "# 导入所需库\n", "from openai import OpenAI\n", "from dotenv import load_dotenv \n", "from IPython.display import display, HTML\n", "import pandas as pd\n", "import json\n", "import os\n", "load_dotenv()\n", "# 用openAI client 调用模型\n", "\n", "client = OpenAI(\n", " base_url=os.getenv(\"BAILIAN_API_BASE_URL\"),\n", " api_key=os.getenv(\"BAILIAN_API_KEY\")\n", ")\n" ] }, { "cell_type": "markdown", "id": "f9c40826", "metadata": {}, "source": [ "**用于大模型的函数调用**" ] }, { "cell_type": "markdown", "id": "f1fb34ca", "metadata": {}, "source": [ "**定义预测方法**" ] }, { "cell_type": "code", "execution_count": null, "id": "e874b885", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"sentiment\": \"positive\",\n", " \"reason\": \"The review text contains words like '感谢' (thankful), which indicate a positive sentiment.\"\n", "}\n" ] } ], "source": [ "#预测方法\n", "from ast import List\n", "\n", "\n", "def predict_sentiment(text,model=\"qwen3-30b-a3b\"):\n", " # 定义提示词\n", " system_message = \"\"\"You are a sentiment analysis expert. Please analyze the sentiment tendency of the provided review text.\"\"\"\n", " user_message = f\"\"\"\n", " Please analyze the sentiment tendency (positive or negative) of the following review text (provided within <>), and return the result in JSON format.\n", " Review text: <{text}>\n", "\n", " Please only return a JSON containing the following fields:\n", " - sentiment: The sentiment tendency (positive or negative)\n", " - reason: The reason for the sentiment tendency\n", " \"\"\"\n", "\n", " # 调用大模型进行情感分析\n", " response = client.chat.completions.create(\n", " model=model , # 如果model参数为空则使用默认值\n", " messages=[\n", " # 添加系统提示词\n", " {\"role\": \"system\", \"content\": system_message},\n", " # TODO few-shot/one-shot 来增强表现的实践\n", " {\"role\": \"user\", \"content\": user_message}\n", " ],\n", " # Qwen3模型通过enable_thinking参数控制思考过程(开源版默认True,商业版默认False)\n", " # 使用Qwen3开源版模型时,若未启用流式输出,请将下行取消注释,否则会报错\n", " # TODO 定义工具用以大模型调用\n", " tools=[{\n", " \"type\": \"function\",\n", " \"function\": {\n", " \"name\": \"handle_positive_sentiment\",\n", " \"description\": \"When you determine that the sentiment of a piece of text is positive, please use this method.\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"reason\": {\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为积极的原因\"\n", " },\n", " \"result\":{\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为积极的结果\",\n", " \"enum\": [\"positive\", \"negative\"]\n", " }\n", " },\n", " \"required\": [\"reason\", \"result\"],\n", " }\n", " }\n", " },{\n", " \"type\": \"function\",\n", " \"function\": {\n", " \"name\": \"handle_negative_sentiment\",\n", " \"description\": \"When you determine that the sentiment of a piece of text is negative, please use this method.\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"reason\": {\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为消极的原因\"\n", " },\n", " \"result\":{\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为消极的结果\",\n", " \"enum\": [\"positive\", \"negative\"]\n", " }\n", " },\n", " \"required\": [\"reason\", \"result\"],\n", " }\n", " }\n", " }],\n", " extra_body={\"enable_thinking\": False},\n", " temperature=0.3,\n", " response_format={\"type\": \"json_object\"} # 指定返回JSON格式\n", " )\n", " # 获取返回结果\n", " result = response.choices[0].message.content\n", " return result\n", "\n", "#测试\n", "print(predict_sentiment(\"你好!我很感谢你\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "df14f73d", "metadata": {}, "outputs": [], "source": [ "#文件读取方法\n", "def load_data():\n", " # 定义数据文件夹路径\n", " data_dir = \"../data/acllmdb_sentiment_small\"\n", "\n", " # 读取正面评价数据\n", " positive_dir = os.path.join(data_dir, \"positive\")\n", " positive_files = os.listdir(positive_dir)\n", " positive_texts = []\n", " for file in positive_files:\n", " with open(os.path.join(positive_dir, file), 'r', encoding='utf-8') as f:\n", " text = f.read()\n", " positive_texts.append({'text': text, 'sentiment': 'positive'})\n", "\n", " # 读取负面评价数据 \n", " negative_dir = os.path.join(data_dir, \"negative\") \n", " negative_files = os.listdir(negative_dir)\n", " negative_texts = []\n", " for file in negative_files:\n", " with open(os.path.join(negative_dir, file), 'r', encoding='utf-8') as f:\n", " text = f.read()\n", " negative_texts.append({'text': text, 'sentiment': 'negative'})\n", "\n", " # 合并数据并创建DataFrame\n", " df = pd.DataFrame(positive_texts + negative_texts)\n", " print(f\"总共读取了 {len(df)} 条评价数据\")\n", " print(f\"其中正面评价 {len(positive_texts)} 条,负面评价 {len(negative_texts)} 条\")\n", " return df" ] }, { "cell_type": "code", "execution_count": null, "id": "b6c31df9", "metadata": {}, "outputs": [], "source": [ "# 批量预测情感倾向\n", "def predict_sentiment_batch(model):\n", " \n", " try:\n", " # 数据加载\n", " data_to_predict = load_data()\n", " if data_to_predict.empty:\n", " #加载失败\n", " print(\"数据加载为空,请检查数据文件路径是否正确\")\n", " else:\n", " #加载成功\n", " predictions = []\n", "\n", " print(f\"开始进行情感预测...(model={model})\")\n", " # 遍历每条评论数据\n", " for i in range(len(data_to_predict)):\n", " row = data_to_predict.iloc[i]\n", " text = row['text']\n", " true_sentiment=row['sentiment']\n", " try:\n", " # 调用大模型进行预测当前行\n", " result = predict_sentiment(text,model)\n", " #得到json格式结果\n", " if result is not None:\n", " result_dict = json.loads(result)\n", " predicted_sentiment = result_dict.get('sentiment', 'unknown')\n", " reason = result_dict.get('reason', 'unknown')\n", " \n", " # 保存预测结果\n", " predictions.append({\n", " 'text': text,\n", " 'true_sentiment': true_sentiment,\n", " 'predicted_sentiment': predicted_sentiment,\n", " 'reason':reason\n", " })\n", " \n", " # 打印进度 每完成三十条打印一次\n", " if i%30 == 0 or i == len(data_to_predict)-1 :\n", " print(f\"已完成 {len(predictions)}/{len(data_to_predict)} 条预测\")\n", "\n", " except Exception as e:\n", " print(f\"第 {i + 1} 条数据预测失败: {str(e)}\")\n", " continue\n", " \n", " # 将预测结果转换为DataFrame\n", " predictions_df = pd.DataFrame(predictions)\n", " print(f\"成功预测 {len(predictions_df)} 条数据\")\n", " return predictions_df\n", " except Exception as e:\n", " print(f\"数据加载出错: {str(e)}\")\n" ] }, { "cell_type": "markdown", "id": "e56e5bd3", "metadata": {}, "source": [ "**比较以下大模型在这个任务上的 accuracy 差异**" ] }, { "cell_type": "code", "execution_count": null, "id": "f99ac855", "metadata": {}, "outputs": [], "source": [ "# 计算预测准确率\n", "# 解决思路:\n", "# 1. 遍历不同的模型名称列表 models\n", "# 2. 对每个模型调用 predict_sentiment_batch() 获取预测结果\n", "# 3. 计算每个模型的预测准确率:\n", "# - 使用 predictions_df 中的 true_sentiment 和 predicted_sentiment 列进行比较\n", "# - 使用 == 运算符比较两列的值是否相等\n", "# - 使用 mean() 计算相等的比例得到准确率\n", "# 4. 将每个模型的准确率结果以百分比格式打印输出\n", "# 5. 对比不同模型的准确率,分析性能差异\n", "\n", "# 定义要测试的模型列表\n", "models = [\"qwen3-32b\", \"qwen3-30b-a3b\", \"qwen3-0.6b\"]\n", "\n", "# 存储每个模型的准确率结果\n", "model_accuracies = {}\n", "\n", "# 遍历每个模型进行预测和评估\n", "for model_using in models:\n", " predictions_df=predict_sentiment_batch(model_using)\n", " if predictions_df is not None and len(predictions_df) > 0:\n", " # 计算准确率\n", " accuracy = (predictions_df['true_sentiment'] == predictions_df['predicted_sentiment']).mean()\n", " model_accuracies[model_using] = accuracy\n", " \n", " # 打印当前模型的准确率\n", " print(f\"模型 {model_using} 的预测准确率: {accuracy:.2%}\")\n", " \n", " # 统计错误预测的样本\n", " wrong_predictions = predictions_df[predictions_df['true_sentiment'] != predictions_df['predicted_sentiment']]\n", " print(f\"错误预测数量: {len(wrong_predictions)}/{len(predictions_df)}\")\n", " \n", " # # 显示一些错误预测的例子\n", " # if len(wrong_predictions) > 0:\n", " # print(\"\\n错误预测示例:\")\n", " # for _, row in wrong_predictions.head(3).iterrows():\n", " # print(f\"文本: {row['text']}\")\n", " # print(f\"真实情感: {row['true_sentiment']}\")\n", " # print(f\"预测情感: {row['predicted_sentiment']}\")\n", " # print(f\"预测理由: {row['reason']}\\n\")\n", " else:\n", " print(f\"模型 {model_using} 预测结果为空\")\n", "\n", "# 比较不同模型的性能\n", "if model_accuracies:\n", " print(\"\\n模型性能对比:\")\n", " best_model = max(model_accuracies.items(), key=lambda x: x[1])\n", " print(f\"最佳模型: {best_model[0]}, 准确率: {best_model[1]:.2%}\")\n", " \n", "# import matplotlib.pyplot as plt\n", "# # 绘制准确率对比图\n", "# plt.figure(figsize=(10, 6))\n", "# plt.bar(model_accuracies.keys(), [acc * 100 for acc in model_accuracies.values()])\n", "# plt.title('不同模型的准确率对比')\n", "# plt.xlabel('模型')\n", "# plt.ylabel('准确率 (%)')\n", "# plt.xticks(rotation=45)\n", "# plt.tight_layout()\n", "# plt.show()\n" ] }, { "cell_type": "markdown", "id": "837a4320", "metadata": {}, "source": [ "**Function Calling的尝试**" ] }, { "cell_type": "code", "execution_count": null, "id": "292cee13", "metadata": {}, "outputs": [], "source": [ "# 用于function calling的实践\n", "def handle_positive_sentiment(reason, result):\n", " print(\"这是一个积极的评价!\")\n", " print(f\"判断原因: {reason}\")\n", " print(f\"完整结果: {result}\")\n", " \n", "def handle_negative_sentiment(reason, result):\n", " print(\"这是一个消极的评价!\")\n", " print(f\"判断原因: {reason}\")\n", " print(f\"完整结果: {result}\")\n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "47872e36", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ChatCompletion(id='chatcmpl-eda3227c-0060-9319-84f9-ff3701626208', choices=[Choice(finish_reason='tool_calls', index=0, logprobs=None, message=ChatCompletionMessage(content='', refusal=None, role='assistant', annotations=None, audio=None, function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_2fde3180722a4c0cb6d352', function=Function(arguments='{\"reason\": \"The review text \\'你好!我很感谢你\\' contains positive expressions such as \\'你好\\' (hello) and \\'很感谢你\\' (I am very grateful to you), which indicate a positive sentiment.\", \"result\": \"positive\"}', name='handle_positive_sentiment'), type='function', index=0)], reasoning_content=''))], created=1752227871, model='qwen3-4b', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=66, prompt_tokens=468, total_tokens=534, completion_tokens_details=None, prompt_tokens_details=None))\n" ] } ], "source": [ "def predict_sentiment_functioncalling(text,model=\"qwen3-4b\"):\n", " # 定义提示词\n", " system_message = \"\"\"You are a sentiment analysis expert. Please analyze the sentiment tendency of the provided review text.\"\"\"\n", " user_message = f\"\"\"\n", " Please analyze the sentiment tendency (positive or negative) of the following review text (provided within <>).\n", " You can use the following tools to help you analyze the sentiment tendency:\n", " - handle_positive_sentiment: When you determine that the sentiment of a piece of text is positive, please use this method.\n", " - handle_negative_sentiment: When you determine that the sentiment of a piece of text is negative, please use this method.\n", "\n", " Review text: <{text}>\n", " \"\"\"\n", "\n", " # 调用大模型进行情感分析\n", " response = client.chat.completions.create(\n", " model=model , # 如果model参数为空则使用默认值\n", " # TODO 定义工具用以大模型调用\n", " tools=[{\n", " \"type\": \"function\",\n", " \"function\": {\n", " \"name\": \"handle_positive_sentiment\",\n", " \"description\": \"When you determine that the sentiment of a piece of text is positive, please use this method.\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"reason\": {\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为积极的原因\"\n", " },\n", " \"result\":{\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为积极的结果\",\n", " \"enum\": [\"positive\", \"negative\"]\n", " }\n", " },\n", " \"required\": [\"reason\", \"result\"],\n", " }\n", " }\n", " },{\n", " \"type\": \"function\",\n", " \"function\": {\n", " \"name\": \"handle_negative_sentiment\",\n", " \"description\": \"When you determine that the sentiment of a piece of text is negative, please use this method.\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"reason\": {\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为消极的原因\"\n", " },\n", " \"result\":{\n", " \"type\": \"string\",\n", " \"description\": \"你判断情感倾向为消极的结果\",\n", " \"enum\": [\"positive\", \"negative\"]\n", " }\n", " },\n", " \"required\": [\"reason\", \"result\"],\n", " }\n", " }\n", " }],\n", " messages=[\n", " # 添加系统提示词\n", " {\"role\": \"system\", \"content\": system_message},\n", " # TODO few-shot/one-shot 来增强表现的实践\n", " {\"role\": \"user\", \"content\": user_message}\n", " ],\n", " # Qwen3模型通过enable_thinking参数控制思考过程(开源版默认True,商业版默认False)\n", " # 使用Qwen3开源版模型时,若未启用流式输出,请将下行取消注释,否则会报错\n", " extra_body={\"enable_thinking\": False},\n", " tool_choice=\"auto\",\n", " temperature=0.3,\n", " #response_format={\"type\": \"json_object\"} # 指定返回JSON格式\n", " )\n", " # 获取返回结果\n", " # result = response.choices[0].message\n", " \n", " return response\n", "\n", "#测试\n", "completion=predict_sentiment_functioncalling(\"你好!我很感谢你\")\n", "print(completion)" ] }, { "cell_type": "code", "execution_count": null, "id": "865f16a8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "这是一个积极的评价!\n", "判断原因: The review text '你好!我很感谢你' contains positive expressions such as '你好' (hello) and '很感谢你' (I am very grateful to you), which indicate a positive sentiment.\n", "完整结果: positive\n", "工具函数输出:None\n", "\n" ] } ], "source": [ "#执行工具函数\n", "# 从返回的结果中获取函数名称和入参\n", "if completion.choices[0].message.tool_calls:\n", " function_name = completion.choices[0].message.tool_calls[0].function.name\n", " arguments_string = completion.choices[0].message.tool_calls[0].function.arguments\n", "\n", " # 使用json模块解析参数字符串\n", " arguments = json.loads(arguments_string)\n", " # 创建一个函数映射表\n", " function_mapper = {\n", " \"handle_positive_sentiment\": handle_positive_sentiment,\n", " \"handle_negative_sentiment\": handle_negative_sentiment\n", " }\n", " # 获取函数实体\n", " function = function_mapper[function_name]\n", " # 如果入参为空,则直接调用函数\n", " if arguments == {}:\n", " arguments = None\n", " # 否则,传入参数后调用函数\n", " else:\n", " function_output = function(**arguments)\n", " \n", "else:\n", " function_name = None\n", " arguments_string = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "85dc78a7", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.13" } }, "nbformat": 4, "nbformat_minor": 5 }