feat: v3.5 - Multi-language acoustic algorithm scheduler

- Core: Python/C++/MATLAB tri-protocol bridge architecture
- Validation: 5-type real-time field, type, dimension, completeness, and compatibility checking
- UI: Premium Deep Sea theme with visual pipeline preview
- Modules: Sea Trial Exclusive + Exception Hub + Algorithm Management
- Tests: Full coverage with 5 automated test cases
这个提交包含在:
2026-01-21 07:41:12 +08:00
当前提交 11a2d4a4b9
共修改 25 个文件,包含 2528 行新增和 0 行删除
+6
查看文件
@@ -0,0 +1,6 @@
# Tidal - src package init
from .models import WaterSoundData
from .bridges import AlgorithmBridge, PythonBridge
from .scheduler import CoreScheduler
__all__ = ["WaterSoundData", "AlgorithmBridge", "PythonBridge", "CoreScheduler"]
+21
查看文件
@@ -0,0 +1,21 @@
from __future__ import annotations
from functools import lru_cache
from importlib import import_module
from typing import Any, Callable
@lru_cache(maxsize=128)
def load_algo_func(module_path: str, function_name: str) -> Callable[..., Any]:
if not module_path or not function_name:
raise ValueError("module_path/function_name 不能为空")
module = import_module(module_path)
func = getattr(module, function_name, None)
if func is None:
raise AttributeError(f"模块 '{module_path}' 中未找到函数 '{function_name}'")
if not callable(func):
raise TypeError(f"'{module_path}.{function_name}' 不是可调用对象")
return func
+61
查看文件
@@ -0,0 +1,61 @@
import json
import os
from typing import List, Dict, Any, Optional
class AlgorithmManager:
"""算法库管理类 (Algorithm Library Management)"""
def __init__(self, db_path: str):
self.db_path = db_path
self._ensure_db()
def _ensure_db(self):
"""确保数据库文件存在"""
os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
if not os.path.exists(self.db_path):
with open(self.db_path, 'w', encoding='utf-8') as f:
json.dump([], f, indent=4, ensure_ascii=False)
def load_algorithms(self) -> List[Dict[str, Any]]:
"""加载所有算法元数据"""
try:
with open(self.db_path, 'r', encoding='utf-8') as f:
return json.load(f)
except (json.JSONDecodeError, FileNotFoundError):
return []
def save_algorithms(self, algos: List[Dict[str, Any]]):
"""保存算法元数据"""
with open(self.db_path, 'w', encoding='utf-8') as f:
json.dump(algos, f, indent=4, ensure_ascii=False)
def register_algorithm(self, metadata: Dict[str, Any]):
"""注册新算法或更新现有算法(版本打标)"""
algos = self.load_algorithms()
# 检查是否已存在同名同版本的算法
for i, algo in enumerate(algos):
if algo['name'] == metadata['name'] and algo['version'] == metadata['version']:
algos[i] = metadata
break
else:
algos.append(metadata)
self.save_algorithms(algos)
def get_algorithm(self, name: str, version: Optional[str] = None) -> Optional[Dict[str, Any]]:
"""获取指定算法及其元数据"""
algos = self.load_algorithms()
# 如果未指定版本,返回最新版本(这里简化为最后一个匹配项)
matches = [a for a in algos if a['name'] == name]
if not matches:
return None
if version:
for a in matches:
if a['version'] == version:
return a
return matches[-1]
def list_all(self) -> List[str]:
"""列出所有算法名称(不重复)"""
algos = self.load_algorithms()
return list(set(a['name'] for a in algos))
+286
查看文件
@@ -0,0 +1,286 @@
from abc import ABC, abstractmethod
from src.models import WaterSoundData
import os
import shutil
import tempfile
from typing import Callable, Dict, List, Any, Optional
class AlgorithmBridge(ABC):
"""桥接插件层抽象基类"""
def __init__(self, name: str, is_core: bool = False):
self.name = name
self.is_core = is_core # 核心算法标志,用于容错策略
@abstractmethod
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
pass
class PythonBridge(AlgorithmBridge):
"""Python 桥接插件"""
def __init__(self, name: str, algo_func: Callable,
input_mapping: Dict[str, str],
output_mapping: Dict[str, str],
preprocessing: Optional[List[Dict[str, Any]]] = None,
is_core: bool = False,
requirements: Optional[Dict[str, Any]] = None,
output_specs: Optional[Dict[str, Any]] = None,
algo_type: str = "python"):
super().__init__(name, is_core)
self.algo_func = algo_func # Python算法核心函数
self.input_mapping = input_mapping # 输入映射 {algo_param: water_data_attr}
self.output_mapping = output_mapping# 输出映射 {algo_return_index: water_data_attr}
self.preprocessing = preprocessing or [] # 预处理算子 [{ "op": "norm", "params": {...} }]
self.requirements = requirements or {}
self.output_specs = output_specs or {}
self.algo_type = algo_type
def _apply_preprocessing(self, data: Any, ops: List[Dict[str, Any]]) -> Any:
"""应用轻量级预处理算子"""
import numpy as np
for op_config in ops:
op = op_config.get("op")
params = op_config.get("params", {})
if op == "normalize" and isinstance(data, np.ndarray):
# 简单归一化示例
if data.size == 0:
continue
min_val = params.get("min", 0)
max_val = params.get("max", 1)
data_min = float(data.min())
data_max = float(data.max())
denom = data_max - data_min
if denom == 0:
data = np.full_like(data, float(min_val), dtype=float)
else:
data = (data.astype(float) - data_min) / denom * (float(max_val) - float(min_val)) + float(min_val)
elif op == "scale" and isinstance(data, (int, float, np.ndarray)):
data = data * params.get("factor", 1.0)
return data
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
try:
water_data.add_log(self.name, "Start Execution")
# 1. 提取并映射输入,同时应用预处理
algo_inputs = {}
for param, attr in self.input_mapping.items():
if hasattr(water_data, attr):
val = getattr(water_data, attr)
# 查找是否有针对该字段的预处理
relevant_ops = [op for op in self.preprocessing if op.get("field") == attr]
if relevant_ops:
val = self._apply_preprocessing(val, relevant_ops)
algo_inputs[param] = val
else:
raise AttributeError(f"Field {attr} not found in WaterSoundData")
# 2. 调用算法
results = self.algo_func(**algo_inputs)
# 3. 映射结果回 standardization 数据
if not isinstance(results, (list, tuple)):
results = [results]
for key, attr in self.output_mapping.items():
# key can be an index (int) or a key if results is dict
if isinstance(results, dict):
val = results[key]
else:
idx = int(key) if isinstance(key, str) and key.isdigit() else key
val = results[idx]
setattr(water_data, attr, val)
water_data.add_log(self.name, "Finished Execution")
except Exception as e:
water_data.set_error(f"Algorithm {self.name} failed: {str(e)}")
return water_data
import ctypes
import numpy as np
class CppBridge(AlgorithmBridge):
"""C++ 桥接插件 (ctypes 实现)"""
def __init__(self, name: str, lib_path: str, function_name: str,
input_mapping: Dict[str, str],
output_mapping: Dict[str, str],
preprocessing: Optional[List[Dict[str, Any]]] = None,
requirements: Optional[Dict[str, Any]] = None,
output_specs: Optional[Dict[str, Any]] = None,
is_core: bool = False):
super().__init__(name, is_core)
self.lib_path = lib_path
self.function_name = function_name
self.input_mapping = input_mapping
self.output_mapping = output_mapping
self.preprocessing = preprocessing or []
self.requirements = requirements or {}
self.output_specs = output_specs or {}
self.algo_type = "cpp"
self._lib = None
self._func = None
def _load_lib(self):
"""延迟加载动态库"""
if self._lib is None:
if not os.path.exists(self.lib_path):
raise FileNotFoundError(f"C++ library not found: {self.lib_path}")
self._lib = ctypes.CDLL(self.lib_path)
self._func = getattr(self._lib, self.function_name)
# 注意:在 C 侧我们需要定义统一签名,例如:
# void func(double* in, int in_len, double* out, int out_len)
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
try:
water_data.add_log(self.name, "Start C++ Execution")
self._load_lib()
# 1. 准备输入
# 为了简化 Phase 3.3,我们假设目前只支持 1进1出 (double array)
# 实际生产中会根据 input_mapping 构造更复杂的结构
in_attr = list(self.input_mapping.values())[0]
in_data = getattr(water_data, in_attr)
if not isinstance(in_data, np.ndarray):
raise TypeError(f"C++ bridge expected numpy array for {in_attr}")
# 确保数据连续且为 float64 (double)
in_data = np.ascontiguousarray(in_data, dtype=np.float64)
in_ptr = in_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
in_len = ctypes.c_int(in_data.size)
# 2. 准备输出空间
# 同样简化为与输入等长
out_data = np.zeros_like(in_data)
out_ptr = out_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
out_len = ctypes.c_int(out_data.size)
# 3. 调用 C++ 函数
# 默认签名: void (double*, int, double*, int)
self._func(in_ptr, in_len, out_ptr, out_len)
# 4. 映射结果回 standardization 数据
out_attr = list(self.output_mapping.values())[0]
setattr(water_data, out_attr, out_data)
water_data.add_log(self.name, "Finished C++ Execution")
except Exception as e:
water_data.set_error(f"C++ Algorithm {self.name} failed: {str(e)}")
return water_data
class MatlabBridge(AlgorithmBridge):
"""MATLAB/Octave 桥接插件 (支持 Engine/Oct2py/File 降级)"""
def __init__(self, name: str, m_path: str, function_name: str,
input_mapping: Dict[str, str],
output_mapping: Dict[str, str],
preprocessing: Optional[List[Dict[str, Any]]] = None,
requirements: Optional[Dict[str, Any]] = None,
output_specs: Optional[Dict[str, Any]] = None,
is_core: bool = False):
super().__init__(name, is_core)
self.m_path = m_path # .m 文件所在目录
self.function_name = function_name
self.input_mapping = input_mapping
self.output_mapping = output_mapping
self.preprocessing = preprocessing or []
self.requirements = requirements or {}
self.output_specs = output_specs or {}
self.algo_type = "matlab"
self._engine = None
def _call_via_engine(self, inputs: Dict[str, Any]) -> Any:
"""方式 1: 使用官方 matlab.engine"""
import matlab.engine
if self._engine is None:
self._engine = matlab.engine.start_matlab()
self._engine.addpath(self.m_path)
func = getattr(self._engine, self.function_name)
# 将 numpy 数组转为 matlab 类型 (这里简化处理)
m_inputs = [matlab.double(v.tolist()) if isinstance(v, np.ndarray) else v for v in inputs.values()]
return func(*m_inputs, nargout=len(self.output_mapping))
def _call_via_oct2py(self, inputs: Dict[str, Any]) -> Any:
"""方式 2: 使用开源 oct2py (Octave)"""
from oct2py import octave
octave.addpath(self.m_path)
func = getattr(octave, self.function_name)
return func(*inputs.values())
def _call_via_file(self, inputs: Dict[str, Any]) -> Any:
"""方式 3: 文件兜底 (使用 .mat 文件交换)"""
from scipy.io import savemat, loadmat
import subprocess
with tempfile.TemporaryDirectory(prefix="matlab_bridge_") as tmp_dir:
in_file = os.path.join(tmp_dir, "tmp_in.mat")
out_file = os.path.join(tmp_dir, "tmp_out.mat")
savemat(in_file, inputs)
# 构造执行脚本:加入 m_path,加载输入,调用函数,保存输出
m_path = os.path.abspath(self.m_path).replace("'", "''")
in_file_escaped = in_file.replace("'", "''")
out_file_escaped = out_file.replace("'", "''")
arglist = ",".join(inputs.keys())
script = (
f"addpath('{m_path}'); "
f"load('{in_file_escaped}'); "
f"results = {self.function_name}({arglist}); "
f"save('-mat', '{out_file_escaped}', 'results'); "
"exit;"
)
octave = shutil.which("octave") or shutil.which("octave-cli")
matlab = shutil.which("matlab")
if octave:
cmd = [octave, "--quiet", "--eval", script]
elif matlab:
cmd = [matlab, "-batch", script]
else:
raise RuntimeError("No MATLAB/Octave CLI found for file-based execution.")
subprocess.run(cmd, check=True, capture_output=True, text=True)
res = loadmat(out_file)
return res.get("results")
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
try:
water_data.add_log(self.name, "Start MATLAB Execution")
# 准备输入
algo_inputs = {}
for param, attr in self.input_mapping.items():
algo_inputs[param] = getattr(water_data, attr)
# 自动寻优/降级逻辑
from src.env_detect import collect_env_info
env = collect_env_info()
if env["matlab"]["engine_available"]:
results = self._call_via_engine(algo_inputs)
elif env["matlab"]["oct2py_available"]:
results = self._call_via_oct2py(algo_inputs)
elif env["matlab"]["octave_cli"]:
results = self._call_via_file(algo_inputs)
else:
raise RuntimeError("No MATLAB/Octave environment found for execution.")
# 结果回写 (支持单出/多出)
if not isinstance(results, (list, tuple)):
results = [results]
for key, attr in self.output_mapping.items():
idx = int(key) if key.isdigit() else 0
setattr(water_data, attr, results[idx])
water_data.add_log(self.name, "Finished MATLAB Execution")
except Exception as e:
water_data.set_error(f"MATLAB Algorithm {self.name} failed: {str(e)}")
return water_data
+86
查看文件
@@ -0,0 +1,86 @@
from __future__ import annotations
import importlib.util
import os
import platform
import shutil
import sys
from typing import Any, Dict, List
def _has_module(module_name: str) -> bool:
"""安全检测 Python 模块是否可用"""
try:
return importlib.util.find_spec(module_name) is not None
except ModuleNotFoundError:
return False
def _get_lib_suffix() -> str:
"""获取当前平台的动态库后缀"""
system = platform.system().lower()
if system == "windows":
return ".dll"
elif system == "darwin":
return ".dylib"
return ".so"
def _check_ld_paths() -> List[str]:
"""检测动态库搜索路径"""
paths = []
for var in ["LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", "PATH"]:
val = os.environ.get(var)
if val:
paths.extend([p for p in val.split(os.pathsep) if p])
return paths[:10] # 限制输出长度
def collect_env_info() -> Dict[str, Any]:
"""收集完整的环境信息"""
compilers = {
"gcc": shutil.which("gcc"),
"g++": shutil.which("g++"),
"clang": shutil.which("clang"),
"clang++": shutil.which("clang++"),
"cl": shutil.which("cl"), # MSVC
}
return {
"python": {
"version": sys.version.split()[0],
"executable": sys.executable,
},
"platform": {
"os": platform.system(),
"release": platform.release(),
"machine": platform.machine(),
"lib_suffix": _get_lib_suffix(),
},
"matlab": {
"engine_available": _has_module("matlab.engine") or _has_module("matlab"),
"oct2py_available": _has_module("oct2py"),
"octave_cli": shutil.which("octave") or shutil.which("octave-cli"),
},
"cpp": {
"compilers": {k: v for k, v in compilers.items() if v},
"any_available": any(compilers.values()),
},
"ld_paths": _check_ld_paths(),
"scipy_available": _has_module("scipy"),
"numpy_available": _has_module("numpy"),
}
def get_available_algo_types() -> List[str]:
"""根据环境返回可用的算法类型列表"""
env = collect_env_info()
types = ["python"] # Python 始终可用
if env["matlab"]["engine_available"] or env["matlab"]["oct2py_available"] or env["matlab"]["octave_cli"]:
types.append("matlab")
if env["cpp"]["any_available"]:
types.append("cpp")
return types
+45
查看文件
@@ -0,0 +1,45 @@
from dataclasses import dataclass, field
from datetime import datetime, timezone
import numpy as np
from typing import Dict, Optional, List, Any
@dataclass
class WaterSoundData:
"""唯一数据交换标准 (WaterSoundData)"""
# 1. 基础原始数据
raw_data: Optional[np.ndarray] = None # 原始声呐数据
sample_rate: float = 0.0 # 采样率
collect_time: str = "" # 数据采集时间
sea_env: Dict[str, Any] = field(default_factory=dict) # 海洋环境参数(声速、温度、盐度等)
# 2. 处理过程数据
clean_data: Optional[np.ndarray] = None # 清洗后数据
denoised_data: Optional[np.ndarray] = None # 降噪后数据
stft_data: Optional[np.ndarray] = None # STFT时频数据
feat_data: Optional[np.ndarray] = None # 特征数组
# 3. 结果数据
final_result: Optional[np.ndarray] = None # 最终处理结果
core_index: Dict[str, Any] = field(default_factory=dict) # 核心指标(信噪比、数据完整性等)
vis_data: Dict[str, Any] = field(default_factory=dict) # 可视化数据
# 4. 元信息
status: str = "init" # 数据状态:init/processing/finished/error
error_info: str = "" # 错误信息
log: List[Dict[str, Any]] = field(default_factory=list) # 执行日志
algo_version: Dict[str, str] = field(default_factory=dict) # 算法版本记录
def add_log(self, step: str, action: str, info: str = ""):
"""记录执行日志"""
self.log.append({
"step": step,
"action": action,
"info": info,
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds")
})
def set_error(self, info: str):
"""设置错误信息"""
self.status = "error"
self.error_info = info
self.add_log("System", "Error", info)
+74
查看文件
@@ -0,0 +1,74 @@
from __future__ import annotations
import numpy as np
from scipy.signal import butter, filtfilt
def dc_removal_and_bandpass(raw_data: np.ndarray, fs: float, lowcut: float = 10.0, highcut: float = 400.0) -> np.ndarray:
"""去直流 + 带通滤波"""
raw_data = np.asarray(raw_data)
if raw_data.size == 0:
return raw_data
if fs <= 0:
raise ValueError(f"fs 必须为正数,当前 fs={fs}")
# 1. 去直流
data = raw_data - np.mean(raw_data)
# 2. 带通滤波
nyq = 0.5 * fs
if not (0 < lowcut < highcut < nyq):
raise ValueError(f"带通参数不合法:需要 0 < lowcut < highcut < fs/2;当前 lowcut={lowcut}, highcut={highcut}, fs={fs}")
low = lowcut / nyq
high = highcut / nyq
b, a = butter(4, [low, high], btype='band')
padlen = 3 * (max(len(a), len(b)) - 1)
if data.size <= padlen:
return data
try:
return filtfilt(b, a, data)
except ValueError:
return data
def stft_magnitude(clean_data: np.ndarray, nperseg: int = 256) -> np.ndarray:
"""计算 STFT 幅度谱"""
clean_data = np.asarray(clean_data)
if clean_data.size == 0:
return np.empty((0, 0))
from scipy.signal import stft
_, _, zxx = stft(clean_data, nperseg=nperseg)
return np.abs(zxx)
def snr_estimation(raw_data: np.ndarray) -> float:
"""简易信噪比估计 (信号功率/噪声功率)"""
raw_data = np.asarray(raw_data)
if raw_data.size == 0:
raise ValueError("raw_data 为空,无法计算 SNR")
# 假设均值附近是噪声,这里极简逻辑:RMS / STD
# 生产中通常用静默段估计
signal_power = float(np.mean(raw_data.astype(float) ** 2))
noise_power = float(np.var(raw_data.astype(float))) # 简化逻辑
if noise_power == 0:
return 100.0
return float(10 * np.log10(signal_power / noise_power))
def quality_check(raw_data: np.ndarray) -> dict:
"""掉点/饱和检测"""
raw_data = np.asarray(raw_data)
if raw_data.size == 0:
return {"saturation": False, "dropouts": 0, "max_abs_val": 0.0, "quality_score": 0.0}
max_val = float(np.max(np.abs(raw_data)))
saturation = bool(max_val > 0.95) # 假设归一化到1
dropouts = int(np.sum(raw_data == 0))
score = 100.0 - (10.0 if saturation else 0.0) - (dropouts / raw_data.size * 100.0)
return {
"saturation": saturation,
"dropouts": dropouts,
"max_abs_val": max_val,
"quality_score": float(max(0.0, score)),
}
def mock_clean(raw_data: np.ndarray) -> np.ndarray:
return raw_data * 1.1
def mock_feat(clean_data: np.ndarray) -> float:
return float(np.mean(clean_data))
+11
查看文件
@@ -0,0 +1,11 @@
#include <iostream>
extern "C" {
// 统一签名:输入指针, 输入长度, 输出指针, 输出长度
void multiply_by_factor_cpp(double* in, int in_len, double* out, int out_len) {
int n = (in_len < out_len) ? in_len : out_len;
for (int i = 0; i < n; ++i) {
out[i] = in[i] * 1.5; // 模拟 C++ 侧的高效计算
}
}
}
@@ -0,0 +1,6 @@
function out = bandpass_filter_m(in_data, fs)
% 简易 MATLAB 带通滤波模拟
% 实际生产中会使用 designfilt 或 butter
disp(['MATLAB Processing with fs: ', num2str(fs)]);
out = in_data * 0.9; % 模拟处理
end
+91
查看文件
@@ -0,0 +1,91 @@
from typing import List, Dict, Any
import numpy as np
from src.models import WaterSoundData
from src.bridges import AlgorithmBridge
class CoreScheduler:
"""核心调度层"""
def __init__(self):
self.flow: List[AlgorithmBridge] = []
def set_flow(self, algo_list: List[AlgorithmBridge]):
"""设置算法调度流程"""
self.flow = algo_list
def execute(self, water_data: WaterSoundData) -> WaterSoundData:
"""执行调度流程"""
water_data.status = "processing"
water_data.add_log("Scheduler", "Starting Pipeline")
for algo in self.flow:
if water_data.status == "error":
# Check for fault tolerance logic here later
# For now, stop on error if not handled
water_data.add_log("Scheduler", f"Execution halted due to error in previous step")
break
water_data = algo.call_algo(water_data)
if water_data.status != "error":
water_data.status = "finished"
water_data.add_log("Scheduler", "Pipeline Completed Successfully")
return water_data
def validate_flow(self, water_data: WaterSoundData) -> List[Dict[str, str]]:
"""
实时校验全流程
返回错误列表,每个错误包含 'message' 和 'suggestion'
"""
from src.validator import FlowValidator
from src.env_detect import collect_env_info
issues = []
env_info = collect_env_info()
def _dummy_value(expected_type: str):
type_map = {
"ndarray": np.zeros(1),
"float": 0.0,
"int": 0,
"dict": {},
}
return type_map.get(expected_type, np.zeros(1))
# 模拟演练 WaterSoundData 的状态变化
temp_data = WaterSoundData()
# 复制初始数据
for attr in ['raw_data', 'sample_rate', 'collect_time', 'sea_env']:
setattr(temp_data, attr, getattr(water_data, attr))
for bridge in self.flow:
requirements = getattr(bridge, "requirements", None)
algo_type = getattr(bridge, "algo_type", "python")
# 校验输入映射(优先走聚合校验:字段存在性/类型/维度/完整性/兼容性)
if requirements and hasattr(bridge, 'input_mapping'):
step_errors = FlowValidator.validate_step(
temp_data,
requirements=requirements,
input_mapping=getattr(bridge, "input_mapping", None),
algo_type=algo_type,
env_info=env_info,
)
for err in step_errors:
issues.append({"algo": bridge.name, "message": err["message"], "suggestion": err["suggestion"]})
elif hasattr(bridge, 'input_mapping'):
# 向后兼容:无元数据时仅做字段存在性校验
for _, attr in bridge.input_mapping.items():
exists, msg, sug = FlowValidator.check_field_existence(temp_data, attr)
if not exists:
issues.append({"algo": bridge.name, "message": msg, "suggestion": sug})
# 模拟执行(只改状态,不跑真实算法,除非是轻量级的)
if hasattr(bridge, 'output_mapping'):
output_specs = getattr(bridge, "output_specs", None) or {}
for key, attr in bridge.output_mapping.items():
# 按声明的输出类型填充哑元数据,让后续步骤能做类型/维度校验
spec = output_specs.get(attr) or {}
expected_type = spec.get("type") or "ndarray"
setattr(temp_data, attr, _dummy_value(expected_type))
return issues
+27
查看文件
@@ -0,0 +1,27 @@
import json
import os
from typing import Dict, Any, List
class TemplateManager:
"""接口配置模板管理类 (Interface Template Management)"""
def __init__(self, templates_dir: str):
self.templates_dir = templates_dir
os.makedirs(self.templates_dir, exist_ok=True)
def save_template(self, name: str, mapping_data: Dict[str, Any]):
"""保存接口配置模板"""
file_path = os.path.join(self.templates_dir, f"{name}.json")
with open(file_path, 'w', encoding='utf-8') as f:
json.dump(mapping_data, f, indent=4, ensure_ascii=False)
def load_template(self, name: str) -> Dict[str, Any]:
"""加载接口配置模板"""
file_path = os.path.join(self.templates_dir, f"{name}.json")
if not os.path.exists(file_path):
raise FileNotFoundError(f"Template {name} not found")
with open(file_path, 'r', encoding='utf-8') as f:
return json.load(f)
def list_templates(self) -> List[str]:
"""列出所有可用模板"""
return [f.replace('.json', '') for f in os.listdir(self.templates_dir) if f.endswith('.json')]
+136
查看文件
@@ -0,0 +1,136 @@
from typing import Dict, Any, List, Optional, Tuple
import numpy as np
from src.models import WaterSoundData
class FlowValidator:
"""五类校验模块 (5-type Validation Module)"""
@staticmethod
def validate_step(
data: WaterSoundData,
requirements: Dict[str, Any],
input_mapping: Optional[Dict[str, str]] = None,
algo_type: str = "python",
env_info: Optional[Dict[str, Any]] = None,
) -> List[Dict[str, str]]:
"""
进行 5 类校验,并输出人话+修复建议
requirements: 算法元数据中的 inputs 定义
"""
errors: List[Dict[str, str]] = []
env_info = env_info or {}
input_mapping = input_mapping or {}
# 4) 参数完整性(这里用于校验“输入映射是否齐全”)
required_params = list(requirements.keys())
ok, msg, sug = FlowValidator.check_completeness(input_mapping, required_params)
if not ok:
errors.append({"message": msg, "suggestion": sug})
for param, spec in requirements.items():
field_name = input_mapping.get(param) or param
# 1) 字段存在性
ok, msg, sug = FlowValidator.check_field_existence(data, field_name)
if not ok:
errors.append({"message": msg, "suggestion": sug})
continue
# 2) 类型校验
expected_type = (spec or {}).get("type")
if expected_type:
ok, msg, sug = FlowValidator.check_data_type(data, field_name, expected_type)
if not ok:
errors.append({"message": msg, "suggestion": sug})
continue
# 3) 维度校验
expected_dim = (spec or {}).get("dim")
if expected_dim is not None:
ok, msg, sug = FlowValidator.check_dimensions(data, field_name, expected_dim)
if not ok:
errors.append({"message": msg, "suggestion": sug})
# 5) 环境/跨语言兼容性(Phase 2:Python 放行,其他给出建议)
ok, msg, sug = FlowValidator.check_compatibility(algo_type, env_info)
if not ok:
errors.append({"message": msg, "suggestion": sug})
return errors
@staticmethod
def check_field_existence(data: WaterSoundData, field_name: str) -> Tuple[bool, str, str]:
"""1. 字段存在性校验"""
if not hasattr(data, field_name) or getattr(data, field_name) is None:
return False, f"缺少必要字段: {field_name}", f"建议:请在流程前置步骤中增加输出 '{field_name}' 的算法,或手动配置输入。"
return True, "", ""
@staticmethod
def check_data_type(data: WaterSoundData, field_name: str, expected_type: str) -> Tuple[bool, str, str]:
"""2. 类型校验"""
val = getattr(data, field_name)
actual_type = type(val).__name__
type_map = {
"ndarray": np.ndarray,
"float": (float, np.float32, np.float64),
"int": int,
"dict": dict
}
target = type_map.get(expected_type)
if target and not isinstance(val, target):
return False, f"字段 '{field_name}' 类型不匹配", f"期望 {expected_type},实际为 {actual_type}。建议:检查前置算法的输出类型配置。"
return True, "", ""
@staticmethod
def check_dimensions(data: WaterSoundData, field_name: str, expected_dim: List[Optional[int]]) -> Tuple[bool, str, str]:
"""3. 维度校验"""
val = getattr(data, field_name)
if not isinstance(val, np.ndarray):
return True, "", "" # 非数组不校验维度
actual_shape = val.shape
if len(actual_shape) != len(expected_dim):
return False, f"字段 '{field_name}' 维度不匹配", f"期望 {len(expected_dim)} 维,实际 {len(actual_shape)} 维。建议:检查数据切片或重采样参数。"
for i, d in enumerate(expected_dim):
if d is not None and actual_shape[i] != d:
return False, f"字段 '{field_name}' 第 {i} 维长度不符", f"期望 {d},实际 {actual_shape[i]}。建议:调整前置算子的输出规格。"
return True, "", ""
@staticmethod
def check_completeness(params: Dict[str, Any], required_params: List[str]) -> Tuple[bool, str, str]:
"""4. 参数完整性校验"""
missing = [p for p in required_params if p not in params or params[p] is None]
if missing:
return False, f"缺少必填参数/映射: {', '.join(missing)}", "建议:在接口配置/算法配置面板中补齐必填项。"
return True, "", ""
@staticmethod
def check_compatibility(algo_type: str, env_info: Dict[str, Any]) -> Tuple[bool, str, str]:
"""5. 跨语言兼容性校验(结合环境自检结果)"""
if algo_type == "python":
return True, "", ""
if algo_type == "cpp":
if env_info.get("cpp", {}).get("any_available"):
return True, "", ""
return (
False,
"当前环境未检测到可用的 C/C++ 工具链",
"建议:安装 MinGW/GCC/MSVC(或在交付包中提供已编译的动态库并更新检测策略)。",
)
if algo_type == "matlab":
matlab_info = env_info.get("matlab", {})
if matlab_info.get("engine_available") or matlab_info.get("oct2py_available") or matlab_info.get("octave_cli"):
return True, "", ""
return (
False,
"当前环境未检测到 MATLAB/Octave",
"建议:安装 MATLAB Engine 或 Octave+oct2py,或配置 Octave CLI 以启用文件兜底模式。",
)
return False, f"暂不支持 {algo_type} 算法类型", "建议:请确认算法元数据中的 type 配置,或在 Phase 3 扩展桥接插件。"