{ "cells": [ { "cell_type": "markdown", "id": "9e428868", "metadata": {}, "source": [ "## 练习1:情感分类\n", "\n", "### 数据集: data/acllmdb_sentiment_small\n", "包含样本:\n", "- negative: 包含负面评价120条\n", "- positive: 包含正面评价120条\n", "\n", "### 目标\n", "取出下面每一条文本, 并使用大模型预测每个评价的文本是 positive(正面)/negative(负面),并给出大模型预测的准确率(accuracy).\n", "\n", "Accuracy = 预测正确的样本数量 / 总样本(240)\n", "\n", "### 要求\n", "1. 分别使用大模型(qwen3_4b)三种输出方式来进行这个测试\n", " 1. text: 大模型纯文本生成\n", " 2. json mode / json schema: 大模型结构化输出能力\n", " 3. ⭐️⭐️⭐️(80%)tool choices / function call: 使用大模型工具调用能力\n", "2. 能够正确输出每种方法的 metrics: \n", " 1. 必须:Accuracy(准确率)\n", " 2. 【非必须】:Precision、Recall、F1 score\n" ] }, { "cell_type": "code", "execution_count": 1, "id": "2fb513d9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "已设置 OpenAI 客户端,base_url: http://103.154.31.78:20001/compatible-mode/v1/\n" ] } ], "source": [ "import os\n", "from dotenv import load_dotenv\n", "from openai import OpenAI\n", "import glob\n", "import json\n", "from tqdm import tqdm\n", "from openai import BadRequestError\n", "\n", "def create_client():\n", " load_dotenv()\n", " client = OpenAI(\n", " base_url=os.getenv(\"BAILIAN_API_BASE_URL\"),\n", " api_key=os.getenv(\"BAILIAN_API_KEY\")\n", " )\n", " return client\n", "\n", "client = create_client()\n", "print(f\"已设置 OpenAI 客户端,base_url: {client.base_url}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "5850031f", "metadata": {}, "outputs": [], "source": [ "# 初始化,读取文件\n", "def read_files(directory):\n", " texts = []\n", " for filepath in glob.glob(os.path.join(directory, '*')):\n", " with open(filepath, 'r', encoding='utf-8') as f:\n", " texts.append(f.read())\n", " return texts\n", "\n", "positive_texts = read_files('../../data/acllmdb_sentiment_small/positive')\n", "negative_texts = read_files('../../data/acllmdb_sentiment_small/negative')\n", "\n", "# 输出方式\n", "output_modes = {\n", " \"text\": {\n", " \"system_message\": \"You are a sentiment analyzer. Reply with only one word: either 'positive' or 'negative'.\",\n", " \"user_message\": \"Analyze the sentiment of this text: {text}\"\n", " },\n", " \"json\": {\n", " \"system_message\": \"You are a sentiment analyzer. Reply in JSON format with a 'sentiment' field that is either 'positive' or 'negative'.\",\n", " \"user_message\": \"Analyze the sentiment of this text and return JSON: {text}\"\n", " },\n", " \"function\": {\n", " \"system_message\": \"You are a sentiment analyzer. Use the provided function to analyze sentiment.\",\n", " \"user_message\": \"Analyze the sentiment of this text: {text}\",\n", " \"functions\": [\n", " {\n", " \"type\": \"function\",\n", " \"function\":{\n", " \"name\": \"analyze_sentiment\",\n", " \"description\": \"Analyze the sentiment of a text\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"sentiment\": {\n", " \"type\": \"string\",\n", " \"enum\": [\"positive\", \"negative\"],\n", " \"description\": \"The sentiment of the text\"\n", " }\n", " },\n", " \"required\": [\"sentiment\"]\n", " },\n", " \"strict\": True\n", " }\n", " }\n", " ]\n", " },\n", " \"function_reason\": {\n", " \"system_message\": \"You are a sentiment analyzer. Use the provided function to analyze sentiment and explain your reasoning. You must use the provided function and return the right json format containing two properties, which is sentiment and reasoning, or you will be fired. Specially, you should return reasoning in Chinese.\",\n", " \"user_message\": \"Analyze the sentiment of this text: {text}\",\n", " \"functions\": [{\n", " \"type\": \"function\",\n", " \"function\": {\n", " \"name\": \"analyze_sentiment_with_reasoning\",\n", " \"description\": \"Analyze the sentiment of a text and provide reasoning\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"sentiment\": {\n", " \"type\": \"string\",\n", " \"enum\": [\"positive\", \"negative\"],\n", " \"description\": \"The sentiment of the text\"\n", " },\n", " \"reasoning\": {\n", " \"type\": \"string\",\n", " \"description\": \"Explanation for the sentiment classification in Chinese\"\n", " }\n", " },\n", " \"required\": [\"sentiment\", \"reasoning\"]\n", " },\n", " \"strict\": True\n", " }\n", " }]\n", " }\n", "}\n", "\n", "def get_sentiment(client, model, text, mode='text'):\n", " \"\"\"获取文本情感\n", " \n", " Args:\n", " client: OpenAI客户端\n", " model: 使用的模型名称\n", " text: 输入文本\n", " mode: 输出模式,可选 'text', 'json', 'function'\n", " \n", " Returns:\n", " str: 'positive' 或 'negative'\n", " \"\"\"\n", " messages = [\n", " {\"role\": \"system\", \"content\": output_modes[mode][\"system_message\"]},\n", " {\"role\": \"user\", \"content\": output_modes[mode][\"user_message\"].format(text=text)}\n", " ]\n", " \n", " if mode == 'function' or mode == 'function_reason':\n", " response = client.chat.completions.create(\n", " model=model,\n", " messages=messages,\n", " tools=output_modes[mode][\"functions\"],\n", " extra_body={\"enable_thinking\": False},\n", " )\n", " \n", " # 检查是否有工具调用\n", " if response.choices[0].message.tool_calls:\n", " result = json.loads(response.choices[0].message.tool_calls[0].function.arguments)\n", " if not isinstance(result, dict):\n", " raise Exception(\"Invalid result format: expected dictionary\")\n", " \n", " if \"sentiment\" in result:\n", " if mode == 'function_reason':\n", " return result[\"sentiment\"], result[\"reasoning\"]\n", " elif mode == 'function':\n", " return result[\"sentiment\"]\n", " \n", " # 如果没有工具调用,尝试从内容中提取情感\n", " content = response.choices[0].message.content.lower()\n", " temp_sentiment = ''\n", " if 'positive' in content:\n", " temp_sentiment = 'positive'\n", " elif 'negative' in content:\n", " temp_sentiment = 'negative'\n", " elif 'neutral' in content:\n", " temp_sentiment = 'neutral'\n", " elif 'mixed' in content:\n", " temp_sentiment = 'mixed'\n", " \n", " if temp_sentiment:\n", " if mode == 'function_reason':\n", " return temp_sentiment, content\n", " elif mode == 'function':\n", " return temp_sentiment\n", " else:\n", " raise Exception(f\"无法从响应中提取情感, content: {content}\")\n", " \n", " response = client.chat.completions.create(\n", " model=model,\n", " messages=messages,\n", " extra_body={\"enable_thinking\": False},\n", " )\n", " \n", " if mode == 'json':\n", " response = client.chat.completions.create(\n", " model=model,\n", " messages=messages,\n", " response_format={\"type\": \"json_object\"},\n", " extra_body={\"enable_thinking\": False},\n", " )\n", " result = json.loads(response.choices[0].message.content)\n", " return result[\"sentiment\"]\n", " \n", " return response.choices[0].message.content.strip().lower()\n", "\n", "# 计算混淆矩阵\n", "def calculate_metrics(tp, fp, tn, fn, total_samples):\n", " accuracy = (tp + tn) / total_samples\n", " precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n", " recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n", " f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n", " output = []\n", "\n", " # 主要指标\n", " metrics = {\n", " 'accuracy': '准确率',\n", " 'precision': '精确率',\n", " 'recall': '召回率',\n", " 'f1': 'F1分数'\n", " }\n", " \n", " for key, name in metrics.items():\n", " value = locals()[key]\n", " output.append(f\"{name}: {value:.2%}\")\n", " \n", " # 混淆矩阵\n", " output.append(\"\\n混淆矩阵:\")\n", " output.append(f\"真正例 (TP): {tp:4d}\\t假正例 (FP): {fp:4d}\")\n", " output.append(f\"假负例 (FN): {fn:4d}\\t真负例 (TN): {tn:4d}\")\n", " \n", " return output\n", "\n", "def evaluate_sentiment(client, model, positive_texts, negative_texts, mode='text'):\n", " \"\"\"评估情感分析性能\n", " \n", " Args:\n", " client: OpenAI客户端\n", " model: 使用的模型名称\n", " positive_texts: 正面评价文本列表\n", " negative_texts: 负面评价文本列表\n", " mode: 输出模式,可选 'text', 'json', 'function'\n", " \"\"\"\n", " \n", " tp = fp = tn = fn = 0\n", " total_samples = len(positive_texts) + len(negative_texts)\n", " \n", " # 处理正面样本\n", " for text in tqdm(positive_texts, desc=f\"Processing positive samples ({mode} mode)\"):\n", " try:\n", " prediction = get_sentiment(client, model, text, mode)\n", " if prediction == 'positive':\n", " tp += 1\n", " else:\n", " fn += 1\n", " except BadRequestError as e:\n", " fn += 1\n", " \n", " # 处理负面样本\n", " for text in tqdm(negative_texts, desc=f\"Processing negative samples ({mode} mode)\"):\n", " try:\n", " prediction = get_sentiment(client, model, text, mode)\n", " if prediction == 'negative':\n", " tn += 1\n", " else:\n", " fp += 1\n", " except BadRequestError as e:\n", " fn += 1\n", " \n", " # 计算并返回指标\n", " return calculate_metrics(tp, fp, tn, fn, total_samples)" ] }, { "cell_type": "code", "execution_count": 12, "id": "735ac5fa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "开始评估情感分析性能...\n", "使用模型: qwen3-4b, 输出模式: text\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing positive samples (text mode): 100%|██████████| 121/121 [00:56<00:00, 2.13it/s]\n", "Processing negative samples (text mode): 100%|██████████| 121/121 [00:57<00:00, 2.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "评估结果: {'accuracy': 0.9049586776859504, 'precision': 0.9380530973451328, 'recall': 0.8688524590163934, 'f1': 0.902127659574468, 'confusion_matrix': {'tp': 106, 'fp': 7, 'tn': 113, 'fn': 16}}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "model = \"qwen3-4b\"\n", "mode = 'text'\n", "print(f\"开始评估情感分析性能...\\n使用模型: {model}, 输出模式: {mode}\")\n", "results = evaluate_sentiment(client, model, positive_texts, negative_texts, mode)\n", "print(f\"评估结果: {results}\")" ] }, { "cell_type": "code", "execution_count": 14, "id": "9a3ca72e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "开始评估情感分析性能...\n", "使用模型: qwen3-4b, 输出模式: json\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing positive samples (json mode): 100%|██████████| 121/121 [03:32<00:00, 1.75s/it]\n", "Processing negative samples (json mode): 100%|██████████| 121/121 [03:20<00:00, 1.66s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "评估结果: \n", "['准确率: 91.74%', '精确率: 93.97%', '召回率: 89.34%', 'F1分数: 91.60%', '\\n混淆矩阵:', '真正例 (TP): 109\\t假正例 (FP): 7', '假负例 (FN): 13\\t真负例 (TN): 113']\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "model = \"qwen3-4b\"\n", "mode = 'json'\n", "print(f\"开始评估情感分析性能...\\n使用模型: {model}, 输出模式: {mode}\")\n", "results = evaluate_sentiment(client, model, positive_texts, negative_texts, mode)\n", "print(f\"评估结果: \\n{results}\")" ] }, { "cell_type": "code", "execution_count": 31, "id": "a074541e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "开始评估情感分析性能...\n", "使用模型: qwen3-4b, 输出模式: function\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing positive samples (function mode): 100%|██████████| 121/121 [01:43<00:00, 1.17it/s]\n", "Processing negative samples (function mode): 100%|██████████| 121/121 [01:38<00:00, 1.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "评估结果: \n", "准确率: 91.74%\n", "精确率: 96.40%\n", "召回率: 86.99%\n", "F1分数: 91.45%\n", "\n", "混淆矩阵:\n", "真正例 (TP): 107\t假正例 (FP): 4\n", "假负例 (FN): 16\t真负例 (TN): 115\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "model = \"qwen3-4b\"\n", "mode = 'function'\n", "print(f\"开始评估情感分析性能...\\n使用模型: {model}, 输出模式: {mode}\")\n", "results = evaluate_sentiment(client, model, positive_texts, negative_texts, mode)\n", "print(\"\\n评估结果: \")\n", "print(\"\\n\".join(results))" ] }, { "cell_type": "markdown", "id": "1c4d4ecc", "metadata": {}, "source": [ "3. 比较以下大模型在这个任务上的 accuracy 差异\n", " 1. qwen3-32b\n", " 2. qwen3-30b-a3b\n", " 3. qwen3-0.6b\n" ] }, { "cell_type": "code", "execution_count": 41, "id": "660d3af7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "开始比较不同模型的性能...\n", "\n", "评估模型: qwen3-0.6b\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing positive samples (function mode): 100%|██████████| 121/121 [00:59<00:00, 2.05it/s]\n", "Processing negative samples (function mode): 100%|██████████| 121/121 [00:56<00:00, 2.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "模型准确率比较:\n", "qwen3-0.6b: 79.34%\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# models = ['qwen3-32b', 'qwen3-30b-a3b', 'qwen3-0.6b']\n", "models = ['qwen3-0.6b']\n", "mode = 'function'\n", "\n", "print(\"开始比较不同模型的性能...\")\n", "model_results = {}\n", "\n", "for model in models:\n", " print(f\"\\n评估模型: {model}\")\n", " try:\n", " results = evaluate_sentiment(client, model, positive_texts, negative_texts, mode)\n", " accuracy = results[0].split(': ')[1].rstrip('%')\n", " model_results[model] = float(accuracy)\n", " except Exception as e:\n", " print(f\"评估 {model} 时出错: {str(e)}\")\n", " model_results[model] = None\n", "\n", "print(\"\\n模型准确率比较:\")\n", "for model, accuracy in model_results.items():\n", " if accuracy is not None:\n", " print(f\"{model}: {accuracy:.2f}%\")\n", " else:\n", " print(f\"{model}: 评估失败\")" ] }, { "cell_type": "code", "execution_count": 42, "id": "5c2257a4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "开始比较不同模型的性能...\n", "\n", "评估模型: qwen3-32b\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing positive samples (function mode): 100%|██████████| 121/121 [02:17<00:00, 1.13s/it]\n", "Processing negative samples (function mode): 100%|██████████| 121/121 [02:19<00:00, 1.15s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "评估模型: qwen3-30b-a3b\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing positive samples (function mode): 100%|██████████| 121/121 [01:41<00:00, 1.19it/s]\n", "Processing negative samples (function mode): 100%|██████████| 121/121 [01:39<00:00, 1.21it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "模型准确率比较:\n", "qwen3-32b: 92.98%\n", "qwen3-30b-a3b: 93.80%\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "models = ['qwen3-32b', 'qwen3-30b-a3b']\n", "mode = 'function'\n", "\n", "print(\"开始比较不同模型的性能...\")\n", "model_results = {}\n", "\n", "for model in models:\n", " print(f\"\\n评估模型: {model}\")\n", " try:\n", " results = evaluate_sentiment(client, model, positive_texts, negative_texts, mode)\n", " accuracy = results[0].split(': ')[1].rstrip('%')\n", " model_results[model] = float(accuracy)\n", " except Exception as e:\n", " print(f\"评估 {model} 时出错: {str(e)}\")\n", " model_results[model] = None\n", "\n", "print(\"\\n模型准确率比较:\")\n", "for model, accuracy in model_results.items():\n", " if accuracy is not None:\n", " print(f\"{model}: {accuracy:.2f}%\")\n", " else:\n", " print(f\"{model}: 评估失败\")" ] }, { "cell_type": "markdown", "id": "cf01d5d7", "metadata": {}, "source": [ "4. 按照以下步骤分析做 bad cases 分析:\n", " 1. 使用: qwen3-0.6b 模型, 使用tools进行预测\n", " 2. 进行情感分析的时候, 让能够同时给出情感分析结果和原因(大模型输出)\n", " 3. 挑选大模型预测错误的 cases\n", " 4. 人工查看错误 cases大模型分类错误的原因, 并进行归类. 得出每一类错误造成的错误数量.\n", " 5. 根据错误分析结果, 优化prompt, 提升大模型对分类任务的准确率.\n", " 6. 注意temperature参数对结果的影响.\n", " 7. 产出物: csv文件(index,对应文件路径,positive/negative,arguments)" ] }, { "cell_type": "code", "execution_count": null, "id": "1bad4d36", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from datetime import datetime\n", "\n", "def analyze_bad_cases(client, model, positive_texts, negative_texts):\n", " \"\"\"分析错误案例\n", " \n", " Args:\n", " client: OpenAI客户端\n", " model: 使用的模型名称\n", " positive_texts: 正面评价文本列表\n", " negative_texts: 负面评价文本列表\n", " \"\"\"\n", " bad_cases = []\n", " bad_cases_count=0\n", " \n", " # 分析正面样本\n", " for idx, text in enumerate(tqdm(positive_texts, desc=\"分析正面样本\")):\n", " try:\n", " sentiment, reasoning = get_sentiment(client, model, text, mode='function_reason')\n", " if sentiment != 'positive':\n", " bad_cases.append({\n", " 'text_id': f'positive_{idx}',\n", " 'text': text[:200], # 只保存前200个字符\n", " 'expected': 'positive',\n", " 'predicted': sentiment,\n", " 'reasoning': reasoning\n", " })\n", " except BadRequestError as e:\n", " bad_cases.append({\n", " 'text_id': f'positive_{idx}',\n", " 'text': text[:200],\n", " 'expected': 'positive',\n", " 'predicted': 'error',\n", " 'reasoning': str(e)\n", " })\n", " \n", " # 保存正面样本的错误案例\n", " df_positive = pd.DataFrame(bad_cases)\n", " timestamp = datetime.now().strftime('%Y%m%d')\n", " filename = f'bad_cases_{model}_{timestamp}.csv'\n", " df_positive.to_csv(filename, index=False, encoding='utf-8-sig')\n", " \n", " # 清空bad_cases列表,准备分析负面样本\n", " bad_cases_count = bad_cases_count + len(bad_cases)\n", " bad_cases.clear()\n", " \n", " # 分析负面样本\n", " for idx, text in enumerate(tqdm(negative_texts, desc=\"分析负面样本\")):\n", " try:\n", " sentiment, reasoning = get_sentiment(client, model, text, mode='function_reason')\n", " if sentiment != 'negative':\n", " bad_cases.append({\n", " 'text_id': f'negative_{idx}',\n", " 'text': text[:200],\n", " 'expected': 'negative',\n", " 'predicted': sentiment,\n", " 'reasoning': reasoning\n", " })\n", " except BadRequestError as e:\n", " bad_cases.append({\n", " 'text_id': f'negative_{idx}',\n", " 'text': text[:200],\n", " 'expected': 'negative',\n", " 'predicted': 'error',\n", " 'reasoning': str(e)\n", " })\n", " \n", " # 保存结果到CSV\n", " df = pd.DataFrame(bad_cases)\n", " filename = f'bad_cases_{model}_{timestamp}.csv'\n", " df.to_csv(filename, mode='a', header=False, index=False, encoding='utf-8-sig')\n", " bad_cases_count = bad_cases_count + len(bad_cases)\n", " \n", " print(f\"\\n分析完成!共发现 {len(bad_cases_count)} 个错误案例\")\n", " print(f\"结果已保存至: {filename}\")\n", " \n", " return bad_cases" ] }, { "cell_type": "code", "execution_count": 31, "id": "324ec3d8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "开始进行情感分析...\n", "使用模型: qwen3-0.6b, 输出模式: function_reason\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "分析正面样本: 100%|██████████| 36/36 [00:24<00:00, 1.49it/s]\n", "分析负面样本: 100%|██████████| 36/36 [00:25<00:00, 1.42it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "分析完成!共发现 1 个错误案例\n", "结果已保存至: bad_cases_qwen3-0.6b_20250722.csv\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "model = \"qwen3-0.6b\"\n", "mode = 'function_reason'\n", "print(f\"开始进行情感分析...\\n使用模型: {model}, 输出模式: {mode}\")\n", "bad_cases = analyze_bad_cases(client, model, positive_texts, negative_texts)" ] } ], "metadata": { "kernelspec": { "display_name": "ai-learning", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.13" } }, "nbformat": 4, "nbformat_minor": 5 }