Просмотр исходного кода

增加数据转换准确率计算逻辑(简单逻辑)

wangyang 4 дней назад
Родитель
Сommit
3529804b03
1 измененных файлов с 73 добавлено и 12 удалено
  1. 73 12
      llmops/folder_scanner.py

+ 73 - 12
llmops/folder_scanner.py

@@ -20,6 +20,10 @@ class ScanConfig:
     expected_outline_count: int = -1
     # 指标计算agent的期望数量,-1表示不校验
     expected_metric_count: int = -1
+    # 数据转换agent的期望数量,-1表示不校验
+    expected_transform_count: int = -1
+    # 期望扫描的文件夹总数(用于计算文件成功率)
+    expected_folder_count: int = -1
 
 
 @dataclass
@@ -30,17 +34,20 @@ class ScanResult:
     outline_files: List[str]
     metric_files: List[str]
     success_metric_files: List[str]
+    transform_files: List[str]  # 新增:数据转换文件
 
     # 实际数量
     actual_decision_count: int
     actual_outline_count: int
     actual_metric_count: int
     actual_success_metric_count: int
+    actual_transform_count: int  # 新增:数据转换数量
 
     # 准确率(如果配置了期望值)
     decision_accuracy: Optional[float] = None
     outline_accuracy: Optional[float] = None
     metric_accuracy: Optional[float] = None
+    transform_accuracy: Optional[float] = None  # 新增:数据转换准确率
 
 
 class FolderScanner:
@@ -73,9 +80,11 @@ class FolderScanner:
         logger.info(f"开始扫描 {folder_count} 个文件夹...")
         logger.info(f"期望配置: 决策agent={config.expected_decision_count}, "
                     f"大纲agent={config.expected_outline_count}, "
-                    f"指标agent={config.expected_metric_count}")
+                    f"指标agent={config.expected_metric_count}, "
+                    f"数据转换agent={config.expected_transform_count}")
 
         self.results = []
+        missing_folders = []  # 记录缺失的文件夹
 
         for i in range(1, folder_count + 1):
             folder_name = f"api_results_{i}"
@@ -83,12 +92,13 @@ class FolderScanner:
 
             if not os.path.exists(folder_path):
                 logger.warning(f"文件夹不存在: {folder_path}")
+                missing_folders.append(folder_name)
                 continue
 
             result = self._scan_single_folder(folder_name, folder_path, config)
             self.results.append(result)
 
-        return self._generate_report(config)
+        return self._generate_report(config, folder_count, missing_folders)
 
     def _calculate_accuracy(self, actual: int, expected: int) -> float:
         """计算准确率"""
@@ -111,6 +121,7 @@ class FolderScanner:
         outline_files = []
         metric_files = []
         success_metric_files = []
+        transform_files = []  # 新增:数据转换文件
 
         for file in all_files:
             if file.endswith('.md'):
@@ -118,6 +129,8 @@ class FolderScanner:
                     decision_files.append(file)
                 elif '大纲生成' in file:
                     outline_files.append(file)
+                elif '数据转换' in file:  # 新增:识别数据转换文件
+                    transform_files.append(file)
             elif file.endswith('.json'):
                 metric_files.append(file)
                 if '_success.json' in file:
@@ -128,11 +141,13 @@ class FolderScanner:
         actual_outline_count = len(outline_files)
         actual_metric_count = len(metric_files)
         actual_success_metric_count = len(success_metric_files)
+        actual_transform_count = len(transform_files)  # 新增:数据转换数量
 
         # 计算准确率(如果配置了期望值)
         decision_accuracy = None
         outline_accuracy = None
         metric_accuracy = None
+        transform_accuracy = None  # 新增:数据转换准确率
 
         if config.expected_decision_count != -1 and config.expected_decision_count > 0:
             decision_accuracy = self._calculate_accuracy(actual_decision_count, config.expected_decision_count)
@@ -143,29 +158,41 @@ class FolderScanner:
         if config.expected_metric_count != -1 and config.expected_metric_count > 0:
             metric_accuracy = self._calculate_accuracy(actual_metric_count, config.expected_metric_count)
 
+        if config.expected_transform_count != -1 and config.expected_transform_count > 0:
+            transform_accuracy = self._calculate_accuracy(actual_transform_count, config.expected_transform_count)
+
         return ScanResult(
             folder_name=folder_name,
             decision_files=decision_files,
             outline_files=outline_files,
             metric_files=metric_files,
             success_metric_files=success_metric_files,
+            transform_files=transform_files,  # 新增
             actual_decision_count=actual_decision_count,
             actual_outline_count=actual_outline_count,
             actual_metric_count=actual_metric_count,
             actual_success_metric_count=actual_success_metric_count,
+            actual_transform_count=actual_transform_count,  # 新增
             decision_accuracy=decision_accuracy,
             outline_accuracy=outline_accuracy,
-            metric_accuracy=metric_accuracy
+            metric_accuracy=metric_accuracy,
+            transform_accuracy=transform_accuracy  # 新增
         )
 
-    def _generate_report(self, config: ScanConfig) -> Dict:
+    def _generate_report(self, config: ScanConfig, expected_folder_count: int, missing_folders: List[str]) -> Dict:
         """生成报告"""
         total_folders = len(self.results)
 
+        # 计算文件成功率
+        file_success_rate = 0
+        if expected_folder_count > 0:
+            file_success_rate = (total_folders / expected_folder_count) * 100
+
         # 计算平均准确率
         avg_decision_accuracy = 0
         avg_outline_accuracy = 0
         avg_metric_accuracy = 0
+        avg_transform_accuracy = 0  # 新增:数据转换平均准确率
 
         if total_folders > 0:
             # 决策agent平均准确率
@@ -183,6 +210,11 @@ class FolderScanner:
             if metric_accuracies:
                 avg_metric_accuracy = sum(metric_accuracies) / len(metric_accuracies)
 
+            # 数据转换agent平均准确率
+            transform_accuracies = [r.transform_accuracy for r in self.results if r.transform_accuracy is not None]
+            if transform_accuracies:
+                avg_transform_accuracy = sum(transform_accuracies) / len(transform_accuracies)
+
         # 计算综合成功率(基于所有配置的校验项)
         comprehensive_success_rate = 0
         success_items = []
@@ -194,6 +226,8 @@ class FolderScanner:
             success_items.append(avg_outline_accuracy)
         if config.expected_metric_count != -1:
             success_items.append(avg_metric_accuracy)
+        if config.expected_transform_count != -1:
+            success_items.append(avg_transform_accuracy)
 
         if success_items:
             comprehensive_success_rate = sum(success_items) / len(success_items)
@@ -204,11 +238,15 @@ class FolderScanner:
 
         report = {
             "total_folders_scanned": total_folders,
+            "expected_folder_count": expected_folder_count,
+            "missing_folders": missing_folders,
+            "file_success_rate": f"{file_success_rate:.2f}%",
             "accuracy_statistics": {
                 "comprehensive_success_rate": f"{comprehensive_success_rate:.2f}%",  # 综合成功率
                 "decision_accuracy": f"{avg_decision_accuracy:.2f}%" if config.expected_decision_count != -1 else "未配置",
                 "outline_accuracy": f"{avg_outline_accuracy:.2f}%" if config.expected_outline_count != -1 else "未配置",
                 "metric_accuracy": f"{avg_metric_accuracy:.2f}%" if config.expected_metric_count != -1 else "未配置",
+                "transform_accuracy": f"{avg_transform_accuracy:.2f}%" if config.expected_transform_count != -1 else "未配置",
             },
             "metric_statistics": {
                 "total_metric_files": total_metric_files,
@@ -218,7 +256,8 @@ class FolderScanner:
             "expected_counts": {
                 "decision": config.expected_decision_count,
                 "outline": config.expected_outline_count,
-                "metric": config.expected_metric_count
+                "metric": config.expected_metric_count,
+                "transform": config.expected_transform_count
             },
             "details": []
         }
@@ -231,7 +270,8 @@ class FolderScanner:
                     "decision": result.actual_decision_count,
                     "outline": result.actual_outline_count,
                     "metric": result.actual_metric_count,
-                    "success_metric": result.actual_success_metric_count
+                    "success_metric": result.actual_success_metric_count,
+                    "transform": result.actual_transform_count  # 新增
                 },
                 "accuracies": {}
             }
@@ -243,6 +283,8 @@ class FolderScanner:
                 detail["accuracies"]["outline"] = f"{result.outline_accuracy:.2f}%"
             if result.metric_accuracy is not None:
                 detail["accuracies"]["metric"] = f"{result.metric_accuracy:.2f}%"
+            if result.transform_accuracy is not None:
+                detail["accuracies"]["transform"] = f"{result.transform_accuracy:.2f}%"
 
             report["details"].append(detail)
 
@@ -253,7 +295,16 @@ class FolderScanner:
         logger.info("=" * 50)
         logger.info("文件夹扫描统计报告")
         logger.info("=" * 50)
-        logger.info(f"扫描文件夹总数: {report['total_folders_scanned']}")
+
+        # 文件成功率统计
+        logger.info(f"期望扫描文件夹数: {report['expected_folder_count']}")
+        logger.info(f"实际扫描文件夹数: {report['total_folders_scanned']}")
+        logger.info(f"文件成功率: {report['file_success_rate']}")
+
+        if report['missing_folders']:
+            logger.info(f"缺失文件夹: {', '.join(report['missing_folders'][:5])}")
+            if len(report['missing_folders']) > 5:
+                logger.info(f"  ... 还有{len(report['missing_folders']) - 5}个缺失文件夹")
 
         # 准确率统计
         logger.info("\n📊 准确率统计:")
@@ -269,6 +320,8 @@ class FolderScanner:
             logger.info(f"  大纲agent准确率: {acc_stats['outline_accuracy']}")
         if acc_stats['metric_accuracy'] != "未配置":
             logger.info(f"  指标agent准确率: {acc_stats['metric_accuracy']}")
+        if acc_stats['transform_accuracy'] != "未配置":
+            logger.info(f"  数据转换agent准确率: {acc_stats['transform_accuracy']}")
 
         # 指标文件统计
         logger.info("\n📈 指标文件统计:")
@@ -278,7 +331,7 @@ class FolderScanner:
         logger.info(f"  指标成功率: {metric_stats['success_ratio']}")
 
         # 期望值显示
-        logger.info("\n⚙ 配置期望值:")
+        logger.info("\n⚙ 配置期望值:")
         expected = report['expected_counts']
         if expected['decision'] != -1:
             logger.info(f"  决策agent: {expected['decision']}个")
@@ -286,13 +339,15 @@ class FolderScanner:
             logger.info(f"  大纲agent: {expected['outline']}个")
         if expected['metric'] != -1:
             logger.info(f"  指标agent: {expected['metric']}个")
+        if expected['transform'] != -1:
+            logger.info(f"  数据转换agent: {expected['transform']}个")
 
         logger.info("=" * 50)
 
         # 打印前3个文件夹的详情(避免日志太长)
         if report['details']:
             logger.info("前3个文件夹详情:")
-            for i, detail in enumerate(report['details'][:3]):
+            for i, detail in enumerate(report['details'][:50]):
                 logger.info(f"\n  📂 {detail['folder']}:")
                 actual = detail['actual_counts']
 
@@ -304,6 +359,8 @@ class FolderScanner:
                     count_parts.append(f"大纲:{actual['outline']}")
                 if expected['metric'] != -1:
                     count_parts.append(f"指标:{actual['metric']}")
+                if expected['transform'] != -1:
+                    count_parts.append(f"数据转换:{actual['transform']}")
                 count_parts.append(f"成功指标:{actual['success_metric']}")
 
                 logger.info(f"    实际数量 - {', '.join(count_parts)}")
@@ -318,10 +375,12 @@ class FolderScanner:
                             accuracy_parts.append(f"大纲:{value}")
                         elif key == 'metric':
                             accuracy_parts.append(f"指标:{value}")
+                        elif key == 'transform':
+                            accuracy_parts.append(f"数据转换:{value}")
 
                     logger.info(f"    准确率 - {', '.join(accuracy_parts)}")
 
-            if len(report['details']) > 3:
+            if len(report['details']) > 50:
                 logger.info(f"  ... 还有{len(report['details']) - 3}个文件夹未显示")
 
         logger.info("=" * 50)
@@ -337,11 +396,13 @@ def main():
     config = ScanConfig(
         expected_decision_count=-1,  # 期望每个文件夹有3个决策agent文件
         expected_outline_count=1,  # 期望每个文件夹有1个大纲生成agent文件
-        expected_metric_count=20  # 期望每个文件夹有20个指标计算agent文件
+        expected_metric_count=19,  # 期望每个文件夹有20个指标计算agent文件
+        expected_transform_count=2,  # 期望每个文件夹有2个数据转换agent文件
+        expected_folder_count=10  # 期望扫描100个文件夹
     )
 
     # 扫描100个文件夹
-    report = scanner.scan_folders(folder_count=2, config=config)
+    report = scanner.scan_folders(folder_count=10, config=config)
 
     # 打印报告
     scanner.print_report(report)