feat: v3.5 - Multi-language acoustic algorithm scheduler
- Core: Python/C++/MATLAB tri-protocol bridge architecture - Validation: 5-type real-time field, type, dimension, completeness, and compatibility checking - UI: Premium Deep Sea theme with visual pipeline preview - Modules: Sea Trial Exclusive + Exception Hub + Algorithm Management - Tests: Full coverage with 5 automated test cases
这个提交包含在:
@@ -0,0 +1,6 @@
|
||||
# Tidal - src package init
|
||||
from .models import WaterSoundData
|
||||
from .bridges import AlgorithmBridge, PythonBridge
|
||||
from .scheduler import CoreScheduler
|
||||
|
||||
__all__ = ["WaterSoundData", "AlgorithmBridge", "PythonBridge", "CoreScheduler"]
|
||||
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from importlib import import_module
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def load_algo_func(module_path: str, function_name: str) -> Callable[..., Any]:
|
||||
if not module_path or not function_name:
|
||||
raise ValueError("module_path/function_name 不能为空")
|
||||
|
||||
module = import_module(module_path)
|
||||
func = getattr(module, function_name, None)
|
||||
if func is None:
|
||||
raise AttributeError(f"模块 '{module_path}' 中未找到函数 '{function_name}'")
|
||||
if not callable(func):
|
||||
raise TypeError(f"'{module_path}.{function_name}' 不是可调用对象")
|
||||
|
||||
return func
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import json
|
||||
import os
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
class AlgorithmManager:
|
||||
"""算法库管理类 (Algorithm Library Management)"""
|
||||
def __init__(self, db_path: str):
|
||||
self.db_path = db_path
|
||||
self._ensure_db()
|
||||
|
||||
def _ensure_db(self):
|
||||
"""确保数据库文件存在"""
|
||||
os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
|
||||
if not os.path.exists(self.db_path):
|
||||
with open(self.db_path, 'w', encoding='utf-8') as f:
|
||||
json.dump([], f, indent=4, ensure_ascii=False)
|
||||
|
||||
def load_algorithms(self) -> List[Dict[str, Any]]:
|
||||
"""加载所有算法元数据"""
|
||||
try:
|
||||
with open(self.db_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, FileNotFoundError):
|
||||
return []
|
||||
|
||||
def save_algorithms(self, algos: List[Dict[str, Any]]):
|
||||
"""保存算法元数据"""
|
||||
with open(self.db_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(algos, f, indent=4, ensure_ascii=False)
|
||||
|
||||
def register_algorithm(self, metadata: Dict[str, Any]):
|
||||
"""注册新算法或更新现有算法(版本打标)"""
|
||||
algos = self.load_algorithms()
|
||||
# 检查是否已存在同名同版本的算法
|
||||
for i, algo in enumerate(algos):
|
||||
if algo['name'] == metadata['name'] and algo['version'] == metadata['version']:
|
||||
algos[i] = metadata
|
||||
break
|
||||
else:
|
||||
algos.append(metadata)
|
||||
|
||||
self.save_algorithms(algos)
|
||||
|
||||
def get_algorithm(self, name: str, version: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
"""获取指定算法及其元数据"""
|
||||
algos = self.load_algorithms()
|
||||
# 如果未指定版本,返回最新版本(这里简化为最后一个匹配项)
|
||||
matches = [a for a in algos if a['name'] == name]
|
||||
if not matches:
|
||||
return None
|
||||
|
||||
if version:
|
||||
for a in matches:
|
||||
if a['version'] == version:
|
||||
return a
|
||||
return matches[-1]
|
||||
|
||||
def list_all(self) -> List[str]:
|
||||
"""列出所有算法名称(不重复)"""
|
||||
algos = self.load_algorithms()
|
||||
return list(set(a['name'] for a in algos))
|
||||
+286
@@ -0,0 +1,286 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from src.models import WaterSoundData
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from typing import Callable, Dict, List, Any, Optional
|
||||
|
||||
class AlgorithmBridge(ABC):
|
||||
"""桥接插件层抽象基类"""
|
||||
def __init__(self, name: str, is_core: bool = False):
|
||||
self.name = name
|
||||
self.is_core = is_core # 核心算法标志,用于容错策略
|
||||
|
||||
@abstractmethod
|
||||
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
|
||||
pass
|
||||
|
||||
class PythonBridge(AlgorithmBridge):
|
||||
"""Python 桥接插件"""
|
||||
def __init__(self, name: str, algo_func: Callable,
|
||||
input_mapping: Dict[str, str],
|
||||
output_mapping: Dict[str, str],
|
||||
preprocessing: Optional[List[Dict[str, Any]]] = None,
|
||||
is_core: bool = False,
|
||||
requirements: Optional[Dict[str, Any]] = None,
|
||||
output_specs: Optional[Dict[str, Any]] = None,
|
||||
algo_type: str = "python"):
|
||||
super().__init__(name, is_core)
|
||||
self.algo_func = algo_func # Python算法核心函数
|
||||
self.input_mapping = input_mapping # 输入映射 {algo_param: water_data_attr}
|
||||
self.output_mapping = output_mapping# 输出映射 {algo_return_index: water_data_attr}
|
||||
self.preprocessing = preprocessing or [] # 预处理算子 [{ "op": "norm", "params": {...} }]
|
||||
self.requirements = requirements or {}
|
||||
self.output_specs = output_specs or {}
|
||||
self.algo_type = algo_type
|
||||
|
||||
def _apply_preprocessing(self, data: Any, ops: List[Dict[str, Any]]) -> Any:
|
||||
"""应用轻量级预处理算子"""
|
||||
import numpy as np
|
||||
for op_config in ops:
|
||||
op = op_config.get("op")
|
||||
params = op_config.get("params", {})
|
||||
if op == "normalize" and isinstance(data, np.ndarray):
|
||||
# 简单归一化示例
|
||||
if data.size == 0:
|
||||
continue
|
||||
min_val = params.get("min", 0)
|
||||
max_val = params.get("max", 1)
|
||||
data_min = float(data.min())
|
||||
data_max = float(data.max())
|
||||
denom = data_max - data_min
|
||||
if denom == 0:
|
||||
data = np.full_like(data, float(min_val), dtype=float)
|
||||
else:
|
||||
data = (data.astype(float) - data_min) / denom * (float(max_val) - float(min_val)) + float(min_val)
|
||||
elif op == "scale" and isinstance(data, (int, float, np.ndarray)):
|
||||
data = data * params.get("factor", 1.0)
|
||||
return data
|
||||
|
||||
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
|
||||
try:
|
||||
water_data.add_log(self.name, "Start Execution")
|
||||
|
||||
# 1. 提取并映射输入,同时应用预处理
|
||||
algo_inputs = {}
|
||||
for param, attr in self.input_mapping.items():
|
||||
if hasattr(water_data, attr):
|
||||
val = getattr(water_data, attr)
|
||||
# 查找是否有针对该字段的预处理
|
||||
relevant_ops = [op for op in self.preprocessing if op.get("field") == attr]
|
||||
if relevant_ops:
|
||||
val = self._apply_preprocessing(val, relevant_ops)
|
||||
algo_inputs[param] = val
|
||||
else:
|
||||
raise AttributeError(f"Field {attr} not found in WaterSoundData")
|
||||
|
||||
# 2. 调用算法
|
||||
results = self.algo_func(**algo_inputs)
|
||||
|
||||
# 3. 映射结果回 standardization 数据
|
||||
if not isinstance(results, (list, tuple)):
|
||||
results = [results]
|
||||
|
||||
for key, attr in self.output_mapping.items():
|
||||
# key can be an index (int) or a key if results is dict
|
||||
if isinstance(results, dict):
|
||||
val = results[key]
|
||||
else:
|
||||
idx = int(key) if isinstance(key, str) and key.isdigit() else key
|
||||
val = results[idx]
|
||||
setattr(water_data, attr, val)
|
||||
|
||||
water_data.add_log(self.name, "Finished Execution")
|
||||
|
||||
except Exception as e:
|
||||
water_data.set_error(f"Algorithm {self.name} failed: {str(e)}")
|
||||
|
||||
return water_data
|
||||
|
||||
import ctypes
|
||||
import numpy as np
|
||||
|
||||
class CppBridge(AlgorithmBridge):
|
||||
"""C++ 桥接插件 (ctypes 实现)"""
|
||||
def __init__(self, name: str, lib_path: str, function_name: str,
|
||||
input_mapping: Dict[str, str],
|
||||
output_mapping: Dict[str, str],
|
||||
preprocessing: Optional[List[Dict[str, Any]]] = None,
|
||||
requirements: Optional[Dict[str, Any]] = None,
|
||||
output_specs: Optional[Dict[str, Any]] = None,
|
||||
is_core: bool = False):
|
||||
super().__init__(name, is_core)
|
||||
self.lib_path = lib_path
|
||||
self.function_name = function_name
|
||||
self.input_mapping = input_mapping
|
||||
self.output_mapping = output_mapping
|
||||
self.preprocessing = preprocessing or []
|
||||
self.requirements = requirements or {}
|
||||
self.output_specs = output_specs or {}
|
||||
self.algo_type = "cpp"
|
||||
self._lib = None
|
||||
self._func = None
|
||||
|
||||
def _load_lib(self):
|
||||
"""延迟加载动态库"""
|
||||
if self._lib is None:
|
||||
if not os.path.exists(self.lib_path):
|
||||
raise FileNotFoundError(f"C++ library not found: {self.lib_path}")
|
||||
self._lib = ctypes.CDLL(self.lib_path)
|
||||
self._func = getattr(self._lib, self.function_name)
|
||||
# 注意:在 C 侧我们需要定义统一签名,例如:
|
||||
# void func(double* in, int in_len, double* out, int out_len)
|
||||
|
||||
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
|
||||
try:
|
||||
water_data.add_log(self.name, "Start C++ Execution")
|
||||
self._load_lib()
|
||||
|
||||
# 1. 准备输入
|
||||
# 为了简化 Phase 3.3,我们假设目前只支持 1进1出 (double array)
|
||||
# 实际生产中会根据 input_mapping 构造更复杂的结构
|
||||
in_attr = list(self.input_mapping.values())[0]
|
||||
in_data = getattr(water_data, in_attr)
|
||||
|
||||
if not isinstance(in_data, np.ndarray):
|
||||
raise TypeError(f"C++ bridge expected numpy array for {in_attr}")
|
||||
|
||||
# 确保数据连续且为 float64 (double)
|
||||
in_data = np.ascontiguousarray(in_data, dtype=np.float64)
|
||||
in_ptr = in_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
|
||||
in_len = ctypes.c_int(in_data.size)
|
||||
|
||||
# 2. 准备输出空间
|
||||
# 同样简化为与输入等长
|
||||
out_data = np.zeros_like(in_data)
|
||||
out_ptr = out_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
|
||||
out_len = ctypes.c_int(out_data.size)
|
||||
|
||||
# 3. 调用 C++ 函数
|
||||
# 默认签名: void (double*, int, double*, int)
|
||||
self._func(in_ptr, in_len, out_ptr, out_len)
|
||||
|
||||
# 4. 映射结果回 standardization 数据
|
||||
out_attr = list(self.output_mapping.values())[0]
|
||||
setattr(water_data, out_attr, out_data)
|
||||
|
||||
water_data.add_log(self.name, "Finished C++ Execution")
|
||||
|
||||
except Exception as e:
|
||||
water_data.set_error(f"C++ Algorithm {self.name} failed: {str(e)}")
|
||||
|
||||
return water_data
|
||||
|
||||
|
||||
class MatlabBridge(AlgorithmBridge):
|
||||
"""MATLAB/Octave 桥接插件 (支持 Engine/Oct2py/File 降级)"""
|
||||
def __init__(self, name: str, m_path: str, function_name: str,
|
||||
input_mapping: Dict[str, str],
|
||||
output_mapping: Dict[str, str],
|
||||
preprocessing: Optional[List[Dict[str, Any]]] = None,
|
||||
requirements: Optional[Dict[str, Any]] = None,
|
||||
output_specs: Optional[Dict[str, Any]] = None,
|
||||
is_core: bool = False):
|
||||
super().__init__(name, is_core)
|
||||
self.m_path = m_path # .m 文件所在目录
|
||||
self.function_name = function_name
|
||||
self.input_mapping = input_mapping
|
||||
self.output_mapping = output_mapping
|
||||
self.preprocessing = preprocessing or []
|
||||
self.requirements = requirements or {}
|
||||
self.output_specs = output_specs or {}
|
||||
self.algo_type = "matlab"
|
||||
self._engine = None
|
||||
|
||||
def _call_via_engine(self, inputs: Dict[str, Any]) -> Any:
|
||||
"""方式 1: 使用官方 matlab.engine"""
|
||||
import matlab.engine
|
||||
if self._engine is None:
|
||||
self._engine = matlab.engine.start_matlab()
|
||||
self._engine.addpath(self.m_path)
|
||||
|
||||
func = getattr(self._engine, self.function_name)
|
||||
# 将 numpy 数组转为 matlab 类型 (这里简化处理)
|
||||
m_inputs = [matlab.double(v.tolist()) if isinstance(v, np.ndarray) else v for v in inputs.values()]
|
||||
return func(*m_inputs, nargout=len(self.output_mapping))
|
||||
|
||||
def _call_via_oct2py(self, inputs: Dict[str, Any]) -> Any:
|
||||
"""方式 2: 使用开源 oct2py (Octave)"""
|
||||
from oct2py import octave
|
||||
octave.addpath(self.m_path)
|
||||
func = getattr(octave, self.function_name)
|
||||
return func(*inputs.values())
|
||||
|
||||
def _call_via_file(self, inputs: Dict[str, Any]) -> Any:
|
||||
"""方式 3: 文件兜底 (使用 .mat 文件交换)"""
|
||||
from scipy.io import savemat, loadmat
|
||||
import subprocess
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="matlab_bridge_") as tmp_dir:
|
||||
in_file = os.path.join(tmp_dir, "tmp_in.mat")
|
||||
out_file = os.path.join(tmp_dir, "tmp_out.mat")
|
||||
savemat(in_file, inputs)
|
||||
|
||||
# 构造执行脚本:加入 m_path,加载输入,调用函数,保存输出
|
||||
m_path = os.path.abspath(self.m_path).replace("'", "''")
|
||||
in_file_escaped = in_file.replace("'", "''")
|
||||
out_file_escaped = out_file.replace("'", "''")
|
||||
arglist = ",".join(inputs.keys())
|
||||
script = (
|
||||
f"addpath('{m_path}'); "
|
||||
f"load('{in_file_escaped}'); "
|
||||
f"results = {self.function_name}({arglist}); "
|
||||
f"save('-mat', '{out_file_escaped}', 'results'); "
|
||||
"exit;"
|
||||
)
|
||||
|
||||
octave = shutil.which("octave") or shutil.which("octave-cli")
|
||||
matlab = shutil.which("matlab")
|
||||
if octave:
|
||||
cmd = [octave, "--quiet", "--eval", script]
|
||||
elif matlab:
|
||||
cmd = [matlab, "-batch", script]
|
||||
else:
|
||||
raise RuntimeError("No MATLAB/Octave CLI found for file-based execution.")
|
||||
|
||||
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
||||
|
||||
res = loadmat(out_file)
|
||||
return res.get("results")
|
||||
|
||||
def call_algo(self, water_data: WaterSoundData) -> WaterSoundData:
|
||||
try:
|
||||
water_data.add_log(self.name, "Start MATLAB Execution")
|
||||
|
||||
# 准备输入
|
||||
algo_inputs = {}
|
||||
for param, attr in self.input_mapping.items():
|
||||
algo_inputs[param] = getattr(water_data, attr)
|
||||
|
||||
# 自动寻优/降级逻辑
|
||||
from src.env_detect import collect_env_info
|
||||
env = collect_env_info()
|
||||
|
||||
if env["matlab"]["engine_available"]:
|
||||
results = self._call_via_engine(algo_inputs)
|
||||
elif env["matlab"]["oct2py_available"]:
|
||||
results = self._call_via_oct2py(algo_inputs)
|
||||
elif env["matlab"]["octave_cli"]:
|
||||
results = self._call_via_file(algo_inputs)
|
||||
else:
|
||||
raise RuntimeError("No MATLAB/Octave environment found for execution.")
|
||||
|
||||
# 结果回写 (支持单出/多出)
|
||||
if not isinstance(results, (list, tuple)):
|
||||
results = [results]
|
||||
|
||||
for key, attr in self.output_mapping.items():
|
||||
idx = int(key) if key.isdigit() else 0
|
||||
setattr(water_data, attr, results[idx])
|
||||
|
||||
water_data.add_log(self.name, "Finished MATLAB Execution")
|
||||
|
||||
except Exception as e:
|
||||
water_data.set_error(f"MATLAB Algorithm {self.name} failed: {str(e)}")
|
||||
|
||||
return water_data
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
def _has_module(module_name: str) -> bool:
|
||||
"""安全检测 Python 模块是否可用"""
|
||||
try:
|
||||
return importlib.util.find_spec(module_name) is not None
|
||||
except ModuleNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def _get_lib_suffix() -> str:
|
||||
"""获取当前平台的动态库后缀"""
|
||||
system = platform.system().lower()
|
||||
if system == "windows":
|
||||
return ".dll"
|
||||
elif system == "darwin":
|
||||
return ".dylib"
|
||||
return ".so"
|
||||
|
||||
|
||||
def _check_ld_paths() -> List[str]:
|
||||
"""检测动态库搜索路径"""
|
||||
paths = []
|
||||
for var in ["LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", "PATH"]:
|
||||
val = os.environ.get(var)
|
||||
if val:
|
||||
paths.extend([p for p in val.split(os.pathsep) if p])
|
||||
return paths[:10] # 限制输出长度
|
||||
|
||||
|
||||
def collect_env_info() -> Dict[str, Any]:
|
||||
"""收集完整的环境信息"""
|
||||
compilers = {
|
||||
"gcc": shutil.which("gcc"),
|
||||
"g++": shutil.which("g++"),
|
||||
"clang": shutil.which("clang"),
|
||||
"clang++": shutil.which("clang++"),
|
||||
"cl": shutil.which("cl"), # MSVC
|
||||
}
|
||||
|
||||
return {
|
||||
"python": {
|
||||
"version": sys.version.split()[0],
|
||||
"executable": sys.executable,
|
||||
},
|
||||
"platform": {
|
||||
"os": platform.system(),
|
||||
"release": platform.release(),
|
||||
"machine": platform.machine(),
|
||||
"lib_suffix": _get_lib_suffix(),
|
||||
},
|
||||
"matlab": {
|
||||
"engine_available": _has_module("matlab.engine") or _has_module("matlab"),
|
||||
"oct2py_available": _has_module("oct2py"),
|
||||
"octave_cli": shutil.which("octave") or shutil.which("octave-cli"),
|
||||
},
|
||||
"cpp": {
|
||||
"compilers": {k: v for k, v in compilers.items() if v},
|
||||
"any_available": any(compilers.values()),
|
||||
},
|
||||
"ld_paths": _check_ld_paths(),
|
||||
"scipy_available": _has_module("scipy"),
|
||||
"numpy_available": _has_module("numpy"),
|
||||
}
|
||||
|
||||
|
||||
def get_available_algo_types() -> List[str]:
|
||||
"""根据环境返回可用的算法类型列表"""
|
||||
env = collect_env_info()
|
||||
types = ["python"] # Python 始终可用
|
||||
|
||||
if env["matlab"]["engine_available"] or env["matlab"]["oct2py_available"] or env["matlab"]["octave_cli"]:
|
||||
types.append("matlab")
|
||||
|
||||
if env["cpp"]["any_available"]:
|
||||
types.append("cpp")
|
||||
|
||||
return types
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
import numpy as np
|
||||
from typing import Dict, Optional, List, Any
|
||||
|
||||
@dataclass
|
||||
class WaterSoundData:
|
||||
"""唯一数据交换标准 (WaterSoundData)"""
|
||||
# 1. 基础原始数据
|
||||
raw_data: Optional[np.ndarray] = None # 原始声呐数据
|
||||
sample_rate: float = 0.0 # 采样率
|
||||
collect_time: str = "" # 数据采集时间
|
||||
sea_env: Dict[str, Any] = field(default_factory=dict) # 海洋环境参数(声速、温度、盐度等)
|
||||
|
||||
# 2. 处理过程数据
|
||||
clean_data: Optional[np.ndarray] = None # 清洗后数据
|
||||
denoised_data: Optional[np.ndarray] = None # 降噪后数据
|
||||
stft_data: Optional[np.ndarray] = None # STFT时频数据
|
||||
feat_data: Optional[np.ndarray] = None # 特征数组
|
||||
|
||||
# 3. 结果数据
|
||||
final_result: Optional[np.ndarray] = None # 最终处理结果
|
||||
core_index: Dict[str, Any] = field(default_factory=dict) # 核心指标(信噪比、数据完整性等)
|
||||
vis_data: Dict[str, Any] = field(default_factory=dict) # 可视化数据
|
||||
|
||||
# 4. 元信息
|
||||
status: str = "init" # 数据状态:init/processing/finished/error
|
||||
error_info: str = "" # 错误信息
|
||||
log: List[Dict[str, Any]] = field(default_factory=list) # 执行日志
|
||||
algo_version: Dict[str, str] = field(default_factory=dict) # 算法版本记录
|
||||
|
||||
def add_log(self, step: str, action: str, info: str = ""):
|
||||
"""记录执行日志"""
|
||||
self.log.append({
|
||||
"step": step,
|
||||
"action": action,
|
||||
"info": info,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
})
|
||||
|
||||
def set_error(self, info: str):
|
||||
"""设置错误信息"""
|
||||
self.status = "error"
|
||||
self.error_info = info
|
||||
self.add_log("System", "Error", info)
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
import numpy as np
|
||||
from scipy.signal import butter, filtfilt
|
||||
|
||||
def dc_removal_and_bandpass(raw_data: np.ndarray, fs: float, lowcut: float = 10.0, highcut: float = 400.0) -> np.ndarray:
|
||||
"""去直流 + 带通滤波"""
|
||||
raw_data = np.asarray(raw_data)
|
||||
if raw_data.size == 0:
|
||||
return raw_data
|
||||
if fs <= 0:
|
||||
raise ValueError(f"fs 必须为正数,当前 fs={fs}")
|
||||
|
||||
# 1. 去直流
|
||||
data = raw_data - np.mean(raw_data)
|
||||
# 2. 带通滤波
|
||||
nyq = 0.5 * fs
|
||||
if not (0 < lowcut < highcut < nyq):
|
||||
raise ValueError(f"带通参数不合法:需要 0 < lowcut < highcut < fs/2;当前 lowcut={lowcut}, highcut={highcut}, fs={fs}")
|
||||
low = lowcut / nyq
|
||||
high = highcut / nyq
|
||||
b, a = butter(4, [low, high], btype='band')
|
||||
padlen = 3 * (max(len(a), len(b)) - 1)
|
||||
if data.size <= padlen:
|
||||
return data
|
||||
try:
|
||||
return filtfilt(b, a, data)
|
||||
except ValueError:
|
||||
return data
|
||||
|
||||
def stft_magnitude(clean_data: np.ndarray, nperseg: int = 256) -> np.ndarray:
|
||||
"""计算 STFT 幅度谱"""
|
||||
clean_data = np.asarray(clean_data)
|
||||
if clean_data.size == 0:
|
||||
return np.empty((0, 0))
|
||||
from scipy.signal import stft
|
||||
_, _, zxx = stft(clean_data, nperseg=nperseg)
|
||||
return np.abs(zxx)
|
||||
|
||||
def snr_estimation(raw_data: np.ndarray) -> float:
|
||||
"""简易信噪比估计 (信号功率/噪声功率)"""
|
||||
raw_data = np.asarray(raw_data)
|
||||
if raw_data.size == 0:
|
||||
raise ValueError("raw_data 为空,无法计算 SNR")
|
||||
|
||||
# 假设均值附近是噪声,这里极简逻辑:RMS / STD
|
||||
# 生产中通常用静默段估计
|
||||
signal_power = float(np.mean(raw_data.astype(float) ** 2))
|
||||
noise_power = float(np.var(raw_data.astype(float))) # 简化逻辑
|
||||
if noise_power == 0:
|
||||
return 100.0
|
||||
return float(10 * np.log10(signal_power / noise_power))
|
||||
|
||||
def quality_check(raw_data: np.ndarray) -> dict:
|
||||
"""掉点/饱和检测"""
|
||||
raw_data = np.asarray(raw_data)
|
||||
if raw_data.size == 0:
|
||||
return {"saturation": False, "dropouts": 0, "max_abs_val": 0.0, "quality_score": 0.0}
|
||||
|
||||
max_val = float(np.max(np.abs(raw_data)))
|
||||
saturation = bool(max_val > 0.95) # 假设归一化到1
|
||||
dropouts = int(np.sum(raw_data == 0))
|
||||
score = 100.0 - (10.0 if saturation else 0.0) - (dropouts / raw_data.size * 100.0)
|
||||
return {
|
||||
"saturation": saturation,
|
||||
"dropouts": dropouts,
|
||||
"max_abs_val": max_val,
|
||||
"quality_score": float(max(0.0, score)),
|
||||
}
|
||||
|
||||
def mock_clean(raw_data: np.ndarray) -> np.ndarray:
|
||||
return raw_data * 1.1
|
||||
|
||||
def mock_feat(clean_data: np.ndarray) -> float:
|
||||
return float(np.mean(clean_data))
|
||||
@@ -0,0 +1,11 @@
|
||||
#include <iostream>
|
||||
|
||||
extern "C" {
|
||||
// 统一签名:输入指针, 输入长度, 输出指针, 输出长度
|
||||
void multiply_by_factor_cpp(double* in, int in_len, double* out, int out_len) {
|
||||
int n = (in_len < out_len) ? in_len : out_len;
|
||||
for (int i = 0; i < n; ++i) {
|
||||
out[i] = in[i] * 1.5; // 模拟 C++ 侧的高效计算
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
function out = bandpass_filter_m(in_data, fs)
|
||||
% 简易 MATLAB 带通滤波模拟
|
||||
% 实际生产中会使用 designfilt 或 butter
|
||||
disp(['MATLAB Processing with fs: ', num2str(fs)]);
|
||||
out = in_data * 0.9; % 模拟处理
|
||||
end
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
from typing import List, Dict, Any
|
||||
import numpy as np
|
||||
from src.models import WaterSoundData
|
||||
from src.bridges import AlgorithmBridge
|
||||
|
||||
class CoreScheduler:
|
||||
"""核心调度层"""
|
||||
def __init__(self):
|
||||
self.flow: List[AlgorithmBridge] = []
|
||||
|
||||
def set_flow(self, algo_list: List[AlgorithmBridge]):
|
||||
"""设置算法调度流程"""
|
||||
self.flow = algo_list
|
||||
|
||||
def execute(self, water_data: WaterSoundData) -> WaterSoundData:
|
||||
"""执行调度流程"""
|
||||
water_data.status = "processing"
|
||||
water_data.add_log("Scheduler", "Starting Pipeline")
|
||||
|
||||
for algo in self.flow:
|
||||
if water_data.status == "error":
|
||||
# Check for fault tolerance logic here later
|
||||
# For now, stop on error if not handled
|
||||
water_data.add_log("Scheduler", f"Execution halted due to error in previous step")
|
||||
break
|
||||
|
||||
water_data = algo.call_algo(water_data)
|
||||
|
||||
if water_data.status != "error":
|
||||
water_data.status = "finished"
|
||||
water_data.add_log("Scheduler", "Pipeline Completed Successfully")
|
||||
|
||||
return water_data
|
||||
|
||||
def validate_flow(self, water_data: WaterSoundData) -> List[Dict[str, str]]:
|
||||
"""
|
||||
实时校验全流程
|
||||
返回错误列表,每个错误包含 'message' 和 'suggestion'
|
||||
"""
|
||||
from src.validator import FlowValidator
|
||||
from src.env_detect import collect_env_info
|
||||
issues = []
|
||||
env_info = collect_env_info()
|
||||
|
||||
def _dummy_value(expected_type: str):
|
||||
type_map = {
|
||||
"ndarray": np.zeros(1),
|
||||
"float": 0.0,
|
||||
"int": 0,
|
||||
"dict": {},
|
||||
}
|
||||
return type_map.get(expected_type, np.zeros(1))
|
||||
|
||||
# 模拟演练 WaterSoundData 的状态变化
|
||||
temp_data = WaterSoundData()
|
||||
# 复制初始数据
|
||||
for attr in ['raw_data', 'sample_rate', 'collect_time', 'sea_env']:
|
||||
setattr(temp_data, attr, getattr(water_data, attr))
|
||||
|
||||
for bridge in self.flow:
|
||||
requirements = getattr(bridge, "requirements", None)
|
||||
algo_type = getattr(bridge, "algo_type", "python")
|
||||
|
||||
# 校验输入映射(优先走聚合校验:字段存在性/类型/维度/完整性/兼容性)
|
||||
if requirements and hasattr(bridge, 'input_mapping'):
|
||||
step_errors = FlowValidator.validate_step(
|
||||
temp_data,
|
||||
requirements=requirements,
|
||||
input_mapping=getattr(bridge, "input_mapping", None),
|
||||
algo_type=algo_type,
|
||||
env_info=env_info,
|
||||
)
|
||||
for err in step_errors:
|
||||
issues.append({"algo": bridge.name, "message": err["message"], "suggestion": err["suggestion"]})
|
||||
elif hasattr(bridge, 'input_mapping'):
|
||||
# 向后兼容:无元数据时仅做字段存在性校验
|
||||
for _, attr in bridge.input_mapping.items():
|
||||
exists, msg, sug = FlowValidator.check_field_existence(temp_data, attr)
|
||||
if not exists:
|
||||
issues.append({"algo": bridge.name, "message": msg, "suggestion": sug})
|
||||
|
||||
# 模拟执行(只改状态,不跑真实算法,除非是轻量级的)
|
||||
if hasattr(bridge, 'output_mapping'):
|
||||
output_specs = getattr(bridge, "output_specs", None) or {}
|
||||
for key, attr in bridge.output_mapping.items():
|
||||
# 按声明的输出类型填充哑元数据,让后续步骤能做类型/维度校验
|
||||
spec = output_specs.get(attr) or {}
|
||||
expected_type = spec.get("type") or "ndarray"
|
||||
setattr(temp_data, attr, _dummy_value(expected_type))
|
||||
|
||||
return issues
|
||||
@@ -0,0 +1,27 @@
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
class TemplateManager:
|
||||
"""接口配置模板管理类 (Interface Template Management)"""
|
||||
def __init__(self, templates_dir: str):
|
||||
self.templates_dir = templates_dir
|
||||
os.makedirs(self.templates_dir, exist_ok=True)
|
||||
|
||||
def save_template(self, name: str, mapping_data: Dict[str, Any]):
|
||||
"""保存接口配置模板"""
|
||||
file_path = os.path.join(self.templates_dir, f"{name}.json")
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(mapping_data, f, indent=4, ensure_ascii=False)
|
||||
|
||||
def load_template(self, name: str) -> Dict[str, Any]:
|
||||
"""加载接口配置模板"""
|
||||
file_path = os.path.join(self.templates_dir, f"{name}.json")
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"Template {name} not found")
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
def list_templates(self) -> List[str]:
|
||||
"""列出所有可用模板"""
|
||||
return [f.replace('.json', '') for f in os.listdir(self.templates_dir) if f.endswith('.json')]
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
import numpy as np
|
||||
from src.models import WaterSoundData
|
||||
|
||||
class FlowValidator:
|
||||
"""五类校验模块 (5-type Validation Module)"""
|
||||
|
||||
@staticmethod
|
||||
def validate_step(
|
||||
data: WaterSoundData,
|
||||
requirements: Dict[str, Any],
|
||||
input_mapping: Optional[Dict[str, str]] = None,
|
||||
algo_type: str = "python",
|
||||
env_info: Optional[Dict[str, Any]] = None,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""
|
||||
进行 5 类校验,并输出人话+修复建议
|
||||
requirements: 算法元数据中的 inputs 定义
|
||||
"""
|
||||
errors: List[Dict[str, str]] = []
|
||||
env_info = env_info or {}
|
||||
input_mapping = input_mapping or {}
|
||||
|
||||
# 4) 参数完整性(这里用于校验“输入映射是否齐全”)
|
||||
required_params = list(requirements.keys())
|
||||
ok, msg, sug = FlowValidator.check_completeness(input_mapping, required_params)
|
||||
if not ok:
|
||||
errors.append({"message": msg, "suggestion": sug})
|
||||
|
||||
for param, spec in requirements.items():
|
||||
field_name = input_mapping.get(param) or param
|
||||
|
||||
# 1) 字段存在性
|
||||
ok, msg, sug = FlowValidator.check_field_existence(data, field_name)
|
||||
if not ok:
|
||||
errors.append({"message": msg, "suggestion": sug})
|
||||
continue
|
||||
|
||||
# 2) 类型校验
|
||||
expected_type = (spec or {}).get("type")
|
||||
if expected_type:
|
||||
ok, msg, sug = FlowValidator.check_data_type(data, field_name, expected_type)
|
||||
if not ok:
|
||||
errors.append({"message": msg, "suggestion": sug})
|
||||
continue
|
||||
|
||||
# 3) 维度校验
|
||||
expected_dim = (spec or {}).get("dim")
|
||||
if expected_dim is not None:
|
||||
ok, msg, sug = FlowValidator.check_dimensions(data, field_name, expected_dim)
|
||||
if not ok:
|
||||
errors.append({"message": msg, "suggestion": sug})
|
||||
|
||||
# 5) 环境/跨语言兼容性(Phase 2:Python 放行,其他给出建议)
|
||||
ok, msg, sug = FlowValidator.check_compatibility(algo_type, env_info)
|
||||
if not ok:
|
||||
errors.append({"message": msg, "suggestion": sug})
|
||||
|
||||
return errors
|
||||
|
||||
@staticmethod
|
||||
def check_field_existence(data: WaterSoundData, field_name: str) -> Tuple[bool, str, str]:
|
||||
"""1. 字段存在性校验"""
|
||||
if not hasattr(data, field_name) or getattr(data, field_name) is None:
|
||||
return False, f"缺少必要字段: {field_name}", f"建议:请在流程前置步骤中增加输出 '{field_name}' 的算法,或手动配置输入。"
|
||||
return True, "", ""
|
||||
|
||||
@staticmethod
|
||||
def check_data_type(data: WaterSoundData, field_name: str, expected_type: str) -> Tuple[bool, str, str]:
|
||||
"""2. 类型校验"""
|
||||
val = getattr(data, field_name)
|
||||
actual_type = type(val).__name__
|
||||
|
||||
type_map = {
|
||||
"ndarray": np.ndarray,
|
||||
"float": (float, np.float32, np.float64),
|
||||
"int": int,
|
||||
"dict": dict
|
||||
}
|
||||
|
||||
target = type_map.get(expected_type)
|
||||
if target and not isinstance(val, target):
|
||||
return False, f"字段 '{field_name}' 类型不匹配", f"期望 {expected_type},实际为 {actual_type}。建议:检查前置算法的输出类型配置。"
|
||||
return True, "", ""
|
||||
|
||||
@staticmethod
|
||||
def check_dimensions(data: WaterSoundData, field_name: str, expected_dim: List[Optional[int]]) -> Tuple[bool, str, str]:
|
||||
"""3. 维度校验"""
|
||||
val = getattr(data, field_name)
|
||||
if not isinstance(val, np.ndarray):
|
||||
return True, "", "" # 非数组不校验维度
|
||||
|
||||
actual_shape = val.shape
|
||||
if len(actual_shape) != len(expected_dim):
|
||||
return False, f"字段 '{field_name}' 维度不匹配", f"期望 {len(expected_dim)} 维,实际 {len(actual_shape)} 维。建议:检查数据切片或重采样参数。"
|
||||
|
||||
for i, d in enumerate(expected_dim):
|
||||
if d is not None and actual_shape[i] != d:
|
||||
return False, f"字段 '{field_name}' 第 {i} 维长度不符", f"期望 {d},实际 {actual_shape[i]}。建议:调整前置算子的输出规格。"
|
||||
|
||||
return True, "", ""
|
||||
|
||||
@staticmethod
|
||||
def check_completeness(params: Dict[str, Any], required_params: List[str]) -> Tuple[bool, str, str]:
|
||||
"""4. 参数完整性校验"""
|
||||
missing = [p for p in required_params if p not in params or params[p] is None]
|
||||
if missing:
|
||||
return False, f"缺少必填参数/映射: {', '.join(missing)}", "建议:在接口配置/算法配置面板中补齐必填项。"
|
||||
return True, "", ""
|
||||
|
||||
@staticmethod
|
||||
def check_compatibility(algo_type: str, env_info: Dict[str, Any]) -> Tuple[bool, str, str]:
|
||||
"""5. 跨语言兼容性校验(结合环境自检结果)"""
|
||||
if algo_type == "python":
|
||||
return True, "", ""
|
||||
|
||||
if algo_type == "cpp":
|
||||
if env_info.get("cpp", {}).get("any_available"):
|
||||
return True, "", ""
|
||||
return (
|
||||
False,
|
||||
"当前环境未检测到可用的 C/C++ 工具链",
|
||||
"建议:安装 MinGW/GCC/MSVC(或在交付包中提供已编译的动态库并更新检测策略)。",
|
||||
)
|
||||
|
||||
if algo_type == "matlab":
|
||||
matlab_info = env_info.get("matlab", {})
|
||||
if matlab_info.get("engine_available") or matlab_info.get("oct2py_available") or matlab_info.get("octave_cli"):
|
||||
return True, "", ""
|
||||
return (
|
||||
False,
|
||||
"当前环境未检测到 MATLAB/Octave",
|
||||
"建议:安装 MATLAB Engine 或 Octave+oct2py,或配置 Octave CLI 以启用文件兜底模式。",
|
||||
)
|
||||
|
||||
return False, f"暂不支持 {algo_type} 算法类型", "建议:请确认算法元数据中的 type 配置,或在 Phase 3 扩展桥接插件。"
|
||||
在新工单中引用
屏蔽一个用户