|
|
@@ -20,6 +20,10 @@ class ScanConfig:
|
|
|
expected_outline_count: int = -1
|
|
|
# 指标计算agent的期望数量,-1表示不校验
|
|
|
expected_metric_count: int = -1
|
|
|
+ # 数据转换agent的期望数量,-1表示不校验
|
|
|
+ expected_transform_count: int = -1
|
|
|
+ # 期望扫描的文件夹总数(用于计算文件成功率)
|
|
|
+ expected_folder_count: int = -1
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
@@ -30,17 +34,20 @@ class ScanResult:
|
|
|
outline_files: List[str]
|
|
|
metric_files: List[str]
|
|
|
success_metric_files: List[str]
|
|
|
+ transform_files: List[str] # 新增:数据转换文件
|
|
|
|
|
|
# 实际数量
|
|
|
actual_decision_count: int
|
|
|
actual_outline_count: int
|
|
|
actual_metric_count: int
|
|
|
actual_success_metric_count: int
|
|
|
+ actual_transform_count: int # 新增:数据转换数量
|
|
|
|
|
|
# 准确率(如果配置了期望值)
|
|
|
decision_accuracy: Optional[float] = None
|
|
|
outline_accuracy: Optional[float] = None
|
|
|
metric_accuracy: Optional[float] = None
|
|
|
+ transform_accuracy: Optional[float] = None # 新增:数据转换准确率
|
|
|
|
|
|
|
|
|
class FolderScanner:
|
|
|
@@ -73,9 +80,11 @@ class FolderScanner:
|
|
|
logger.info(f"开始扫描 {folder_count} 个文件夹...")
|
|
|
logger.info(f"期望配置: 决策agent={config.expected_decision_count}, "
|
|
|
f"大纲agent={config.expected_outline_count}, "
|
|
|
- f"指标agent={config.expected_metric_count}")
|
|
|
+ f"指标agent={config.expected_metric_count}, "
|
|
|
+ f"数据转换agent={config.expected_transform_count}")
|
|
|
|
|
|
self.results = []
|
|
|
+ missing_folders = [] # 记录缺失的文件夹
|
|
|
|
|
|
for i in range(1, folder_count + 1):
|
|
|
folder_name = f"api_results_{i}"
|
|
|
@@ -83,12 +92,13 @@ class FolderScanner:
|
|
|
|
|
|
if not os.path.exists(folder_path):
|
|
|
logger.warning(f"文件夹不存在: {folder_path}")
|
|
|
+ missing_folders.append(folder_name)
|
|
|
continue
|
|
|
|
|
|
result = self._scan_single_folder(folder_name, folder_path, config)
|
|
|
self.results.append(result)
|
|
|
|
|
|
- return self._generate_report(config)
|
|
|
+ return self._generate_report(config, folder_count, missing_folders)
|
|
|
|
|
|
def _calculate_accuracy(self, actual: int, expected: int) -> float:
|
|
|
"""计算准确率"""
|
|
|
@@ -111,6 +121,7 @@ class FolderScanner:
|
|
|
outline_files = []
|
|
|
metric_files = []
|
|
|
success_metric_files = []
|
|
|
+ transform_files = [] # 新增:数据转换文件
|
|
|
|
|
|
for file in all_files:
|
|
|
if file.endswith('.md'):
|
|
|
@@ -118,6 +129,8 @@ class FolderScanner:
|
|
|
decision_files.append(file)
|
|
|
elif '大纲生成' in file:
|
|
|
outline_files.append(file)
|
|
|
+ elif '数据转换' in file: # 新增:识别数据转换文件
|
|
|
+ transform_files.append(file)
|
|
|
elif file.endswith('.json'):
|
|
|
metric_files.append(file)
|
|
|
if '_success.json' in file:
|
|
|
@@ -128,11 +141,13 @@ class FolderScanner:
|
|
|
actual_outline_count = len(outline_files)
|
|
|
actual_metric_count = len(metric_files)
|
|
|
actual_success_metric_count = len(success_metric_files)
|
|
|
+ actual_transform_count = len(transform_files) # 新增:数据转换数量
|
|
|
|
|
|
# 计算准确率(如果配置了期望值)
|
|
|
decision_accuracy = None
|
|
|
outline_accuracy = None
|
|
|
metric_accuracy = None
|
|
|
+ transform_accuracy = None # 新增:数据转换准确率
|
|
|
|
|
|
if config.expected_decision_count != -1 and config.expected_decision_count > 0:
|
|
|
decision_accuracy = self._calculate_accuracy(actual_decision_count, config.expected_decision_count)
|
|
|
@@ -143,29 +158,41 @@ class FolderScanner:
|
|
|
if config.expected_metric_count != -1 and config.expected_metric_count > 0:
|
|
|
metric_accuracy = self._calculate_accuracy(actual_metric_count, config.expected_metric_count)
|
|
|
|
|
|
+ if config.expected_transform_count != -1 and config.expected_transform_count > 0:
|
|
|
+ transform_accuracy = self._calculate_accuracy(actual_transform_count, config.expected_transform_count)
|
|
|
+
|
|
|
return ScanResult(
|
|
|
folder_name=folder_name,
|
|
|
decision_files=decision_files,
|
|
|
outline_files=outline_files,
|
|
|
metric_files=metric_files,
|
|
|
success_metric_files=success_metric_files,
|
|
|
+ transform_files=transform_files, # 新增
|
|
|
actual_decision_count=actual_decision_count,
|
|
|
actual_outline_count=actual_outline_count,
|
|
|
actual_metric_count=actual_metric_count,
|
|
|
actual_success_metric_count=actual_success_metric_count,
|
|
|
+ actual_transform_count=actual_transform_count, # 新增
|
|
|
decision_accuracy=decision_accuracy,
|
|
|
outline_accuracy=outline_accuracy,
|
|
|
- metric_accuracy=metric_accuracy
|
|
|
+ metric_accuracy=metric_accuracy,
|
|
|
+ transform_accuracy=transform_accuracy # 新增
|
|
|
)
|
|
|
|
|
|
- def _generate_report(self, config: ScanConfig) -> Dict:
|
|
|
+ def _generate_report(self, config: ScanConfig, expected_folder_count: int, missing_folders: List[str]) -> Dict:
|
|
|
"""生成报告"""
|
|
|
total_folders = len(self.results)
|
|
|
|
|
|
+ # 计算文件成功率
|
|
|
+ file_success_rate = 0
|
|
|
+ if expected_folder_count > 0:
|
|
|
+ file_success_rate = (total_folders / expected_folder_count) * 100
|
|
|
+
|
|
|
# 计算平均准确率
|
|
|
avg_decision_accuracy = 0
|
|
|
avg_outline_accuracy = 0
|
|
|
avg_metric_accuracy = 0
|
|
|
+ avg_transform_accuracy = 0 # 新增:数据转换平均准确率
|
|
|
|
|
|
if total_folders > 0:
|
|
|
# 决策agent平均准确率
|
|
|
@@ -183,6 +210,11 @@ class FolderScanner:
|
|
|
if metric_accuracies:
|
|
|
avg_metric_accuracy = sum(metric_accuracies) / len(metric_accuracies)
|
|
|
|
|
|
+ # 数据转换agent平均准确率
|
|
|
+ transform_accuracies = [r.transform_accuracy for r in self.results if r.transform_accuracy is not None]
|
|
|
+ if transform_accuracies:
|
|
|
+ avg_transform_accuracy = sum(transform_accuracies) / len(transform_accuracies)
|
|
|
+
|
|
|
# 计算综合成功率(基于所有配置的校验项)
|
|
|
comprehensive_success_rate = 0
|
|
|
success_items = []
|
|
|
@@ -194,6 +226,8 @@ class FolderScanner:
|
|
|
success_items.append(avg_outline_accuracy)
|
|
|
if config.expected_metric_count != -1:
|
|
|
success_items.append(avg_metric_accuracy)
|
|
|
+ if config.expected_transform_count != -1:
|
|
|
+ success_items.append(avg_transform_accuracy)
|
|
|
|
|
|
if success_items:
|
|
|
comprehensive_success_rate = sum(success_items) / len(success_items)
|
|
|
@@ -204,11 +238,15 @@ class FolderScanner:
|
|
|
|
|
|
report = {
|
|
|
"total_folders_scanned": total_folders,
|
|
|
+ "expected_folder_count": expected_folder_count,
|
|
|
+ "missing_folders": missing_folders,
|
|
|
+ "file_success_rate": f"{file_success_rate:.2f}%",
|
|
|
"accuracy_statistics": {
|
|
|
"comprehensive_success_rate": f"{comprehensive_success_rate:.2f}%", # 综合成功率
|
|
|
"decision_accuracy": f"{avg_decision_accuracy:.2f}%" if config.expected_decision_count != -1 else "未配置",
|
|
|
"outline_accuracy": f"{avg_outline_accuracy:.2f}%" if config.expected_outline_count != -1 else "未配置",
|
|
|
"metric_accuracy": f"{avg_metric_accuracy:.2f}%" if config.expected_metric_count != -1 else "未配置",
|
|
|
+ "transform_accuracy": f"{avg_transform_accuracy:.2f}%" if config.expected_transform_count != -1 else "未配置",
|
|
|
},
|
|
|
"metric_statistics": {
|
|
|
"total_metric_files": total_metric_files,
|
|
|
@@ -218,7 +256,8 @@ class FolderScanner:
|
|
|
"expected_counts": {
|
|
|
"decision": config.expected_decision_count,
|
|
|
"outline": config.expected_outline_count,
|
|
|
- "metric": config.expected_metric_count
|
|
|
+ "metric": config.expected_metric_count,
|
|
|
+ "transform": config.expected_transform_count
|
|
|
},
|
|
|
"details": []
|
|
|
}
|
|
|
@@ -231,7 +270,8 @@ class FolderScanner:
|
|
|
"decision": result.actual_decision_count,
|
|
|
"outline": result.actual_outline_count,
|
|
|
"metric": result.actual_metric_count,
|
|
|
- "success_metric": result.actual_success_metric_count
|
|
|
+ "success_metric": result.actual_success_metric_count,
|
|
|
+ "transform": result.actual_transform_count # 新增
|
|
|
},
|
|
|
"accuracies": {}
|
|
|
}
|
|
|
@@ -243,6 +283,8 @@ class FolderScanner:
|
|
|
detail["accuracies"]["outline"] = f"{result.outline_accuracy:.2f}%"
|
|
|
if result.metric_accuracy is not None:
|
|
|
detail["accuracies"]["metric"] = f"{result.metric_accuracy:.2f}%"
|
|
|
+ if result.transform_accuracy is not None:
|
|
|
+ detail["accuracies"]["transform"] = f"{result.transform_accuracy:.2f}%"
|
|
|
|
|
|
report["details"].append(detail)
|
|
|
|
|
|
@@ -253,7 +295,16 @@ class FolderScanner:
|
|
|
logger.info("=" * 50)
|
|
|
logger.info("文件夹扫描统计报告")
|
|
|
logger.info("=" * 50)
|
|
|
- logger.info(f"扫描文件夹总数: {report['total_folders_scanned']}")
|
|
|
+
|
|
|
+ # 文件成功率统计
|
|
|
+ logger.info(f"期望扫描文件夹数: {report['expected_folder_count']}")
|
|
|
+ logger.info(f"实际扫描文件夹数: {report['total_folders_scanned']}")
|
|
|
+ logger.info(f"文件成功率: {report['file_success_rate']}")
|
|
|
+
|
|
|
+ if report['missing_folders']:
|
|
|
+ logger.info(f"缺失文件夹: {', '.join(report['missing_folders'][:5])}")
|
|
|
+ if len(report['missing_folders']) > 5:
|
|
|
+ logger.info(f" ... 还有{len(report['missing_folders']) - 5}个缺失文件夹")
|
|
|
|
|
|
# 准确率统计
|
|
|
logger.info("\n📊 准确率统计:")
|
|
|
@@ -269,6 +320,8 @@ class FolderScanner:
|
|
|
logger.info(f" 大纲agent准确率: {acc_stats['outline_accuracy']}")
|
|
|
if acc_stats['metric_accuracy'] != "未配置":
|
|
|
logger.info(f" 指标agent准确率: {acc_stats['metric_accuracy']}")
|
|
|
+ if acc_stats['transform_accuracy'] != "未配置":
|
|
|
+ logger.info(f" 数据转换agent准确率: {acc_stats['transform_accuracy']}")
|
|
|
|
|
|
# 指标文件统计
|
|
|
logger.info("\n📈 指标文件统计:")
|
|
|
@@ -278,7 +331,7 @@ class FolderScanner:
|
|
|
logger.info(f" 指标成功率: {metric_stats['success_ratio']}")
|
|
|
|
|
|
# 期望值显示
|
|
|
- logger.info("\n⚙️ 配置期望值:")
|
|
|
+ logger.info("\n⚙ 配置期望值:")
|
|
|
expected = report['expected_counts']
|
|
|
if expected['decision'] != -1:
|
|
|
logger.info(f" 决策agent: {expected['decision']}个")
|
|
|
@@ -286,13 +339,15 @@ class FolderScanner:
|
|
|
logger.info(f" 大纲agent: {expected['outline']}个")
|
|
|
if expected['metric'] != -1:
|
|
|
logger.info(f" 指标agent: {expected['metric']}个")
|
|
|
+ if expected['transform'] != -1:
|
|
|
+ logger.info(f" 数据转换agent: {expected['transform']}个")
|
|
|
|
|
|
logger.info("=" * 50)
|
|
|
|
|
|
# 打印前3个文件夹的详情(避免日志太长)
|
|
|
if report['details']:
|
|
|
logger.info("前3个文件夹详情:")
|
|
|
- for i, detail in enumerate(report['details'][:3]):
|
|
|
+ for i, detail in enumerate(report['details'][:50]):
|
|
|
logger.info(f"\n 📂 {detail['folder']}:")
|
|
|
actual = detail['actual_counts']
|
|
|
|
|
|
@@ -304,6 +359,8 @@ class FolderScanner:
|
|
|
count_parts.append(f"大纲:{actual['outline']}")
|
|
|
if expected['metric'] != -1:
|
|
|
count_parts.append(f"指标:{actual['metric']}")
|
|
|
+ if expected['transform'] != -1:
|
|
|
+ count_parts.append(f"数据转换:{actual['transform']}")
|
|
|
count_parts.append(f"成功指标:{actual['success_metric']}")
|
|
|
|
|
|
logger.info(f" 实际数量 - {', '.join(count_parts)}")
|
|
|
@@ -318,10 +375,12 @@ class FolderScanner:
|
|
|
accuracy_parts.append(f"大纲:{value}")
|
|
|
elif key == 'metric':
|
|
|
accuracy_parts.append(f"指标:{value}")
|
|
|
+ elif key == 'transform':
|
|
|
+ accuracy_parts.append(f"数据转换:{value}")
|
|
|
|
|
|
logger.info(f" 准确率 - {', '.join(accuracy_parts)}")
|
|
|
|
|
|
- if len(report['details']) > 3:
|
|
|
+ if len(report['details']) > 50:
|
|
|
logger.info(f" ... 还有{len(report['details']) - 3}个文件夹未显示")
|
|
|
|
|
|
logger.info("=" * 50)
|
|
|
@@ -337,11 +396,13 @@ def main():
|
|
|
config = ScanConfig(
|
|
|
expected_decision_count=-1, # 期望每个文件夹有3个决策agent文件
|
|
|
expected_outline_count=1, # 期望每个文件夹有1个大纲生成agent文件
|
|
|
- expected_metric_count=20 # 期望每个文件夹有20个指标计算agent文件
|
|
|
+ expected_metric_count=19, # 期望每个文件夹有20个指标计算agent文件
|
|
|
+ expected_transform_count=2, # 期望每个文件夹有2个数据转换agent文件
|
|
|
+ expected_folder_count=10 # 期望扫描100个文件夹
|
|
|
)
|
|
|
|
|
|
# 扫描100个文件夹
|
|
|
- report = scanner.scan_folders(folder_count=2, config=config)
|
|
|
+ report = scanner.scan_folders(folder_count=10, config=config)
|
|
|
|
|
|
# 打印报告
|
|
|
scanner.print_report(report)
|