diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 74a76fa..5547c66 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -53,6 +53,7 @@ graph TD | **PythonBridge** | 动态 `importlib` 加载 | 原生性能,易于调试 | | **CppBridge** | `ctypes` 指针深度透传 / 可选子进程隔离(`src/cpp_isolated_runner.py`) | 计算密集型算子首选,零拷贝;隔离模式可避免段错误拖垮主进程 | | **MatlabBridge** | 三级降级 (Engine/Oct2py/CLI) | 跨平台及离线环境的最大兼容 | +| **WebBridge** | HTTP/HTTPS 调用 + slot 串行 + 安全策略(`src/security.py`) | 依赖隔离,远程重型服务可被资产化并自由编排 | ### D. 数据模型层 (src/models.py) - **WaterSoundData (WSD)**:系统唯一的标准数据交换格式(核心为 `raw_data` + `meta`)。 diff --git a/docs/walkthrough_final.md b/docs/walkthrough_final.md index 5641d24..89ccab2 100644 --- a/docs/walkthrough_final.md +++ b/docs/walkthrough_final.md @@ -9,12 +9,14 @@ ## 2) 多协议桥接 - Python:动态加载并执行(`src/algo_loader.py`、`src/bridges.py`) -- C++:`.so/.dll` 动态库加载 + `ctypes` 调用(`src/bridges.py`、示例 `src/sample_algos_cpp/`) +- C++:`.so/.dll` 动态库加载 + `ctypes` 调用;可选 `isolate=true` 子进程隔离(`src/bridges.py`、`src/cpp_isolated_runner.py`、示例 `src/sample_algos_cpp/`) - MATLAB/Octave:Engine → oct2py → CLI+matfile 三级降级(`src/bridges.py`、示例 `src/sample_algos_matlab/`) +- WebBridge:HTTP/HTTPS 远程服务桥接(`src/bridges.py`、`src/security.py`、`src/resource_slots.py`) ## 3) 工业级自检与五维校验 -- 环境自检:编译器、MATLAB/Octave 能力检测(`src/env_detect.py`) +- 环境自检:区分 C++ 运行时能力与编译能力,并检测 MATLAB/Octave 能力(`src/env_detect.py`) - 五维校验:字段/类型/维度/完整性/环境兼容(`src/validator.py`、调度前拦截 `src/scheduler.py`) +- 插件加固:动态 Python 模块与跨语言插件可启用 `src/data` 白名单 + SHA256 校验(`src/integrity.py`、`src/algo_loader.py`、`src/bridges.py`) ## 4) 海试专属与异常中心 - 海试一键巡检:内置链路 + CSV 报告导出(`main.py`) @@ -30,4 +32,3 @@ python tests/test_core.py ```bash streamlit run main.py ``` - diff --git a/main.py b/main.py index dc7c23a..d4da566 100644 --- a/main.py +++ b/main.py @@ -390,6 +390,7 @@ def show_scheduling(): scheduler = CoreScheduler() bridges = [] for algo in st.session_state.pipeline: + algo_type = algo.get("type", "python") tpl = None if auto_apply_template: try: @@ -421,7 +422,6 @@ def show_scheduling(): preprocessing = [] from src.bridges import PythonBridge, CppBridge - algo_type = algo.get("type", "python") if algo_type == "python": try: diff --git a/requirements.txt b/requirements.txt index 39f5ea1..49f7138 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ streamlit>=1.28.0 numpy>=1.21.0 scipy>=1.7.0 +requests>=2.0.0 diff --git a/setup.py b/setup.py index e035bac..7ff2d79 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ setup( "streamlit>=1.28.0", "numpy>=1.21.0", "scipy>=1.7.0", + "requests>=2.0.0", ], entry_points={ "console_scripts": [ diff --git a/src/algo_loader.py b/src/algo_loader.py index 1715e1c..3014893 100644 --- a/src/algo_loader.py +++ b/src/algo_loader.py @@ -2,15 +2,29 @@ from __future__ import annotations from functools import lru_cache from importlib import import_module -from typing import Any, Callable +from typing import Any, Callable, Optional + +from src.integrity import ensure_file_sha256, ensure_within_allowed_dirs @lru_cache(maxsize=128) -def load_algo_func(module_path: str, function_name: str) -> Callable[..., Any]: +def load_algo_func( + module_path: str, + function_name: str, + expected_sha256: Optional[str] = None, +) -> Callable[..., Any]: if not module_path or not function_name: raise ValueError("module_path/function_name 不能为空") module = import_module(module_path) + module_file = getattr(module, "__file__", None) + if module_file: + ensure_within_allowed_dirs(module_file) + if expected_sha256: + ensure_file_sha256(module_file, expected_sha256, label=f"python module {module_path}") + elif expected_sha256: + raise PermissionError(f"模块 '{module_path}' 缺少 __file__,无法进行 SHA256 校验") + func = getattr(module, function_name, None) if func is None: raise AttributeError(f"模块 '{module_path}' 中未找到函数 '{function_name}'") @@ -18,4 +32,3 @@ def load_algo_func(module_path: str, function_name: str) -> Callable[..., Any]: raise TypeError(f"'{module_path}.{function_name}' 不是可调用对象") return func - diff --git a/src/bridges.py b/src/bridges.py index 2af1fc6..563c601 100644 --- a/src/bridges.py +++ b/src/bridges.py @@ -487,7 +487,7 @@ class WebBridge(AlgorithmBridge): # 3. 业务逻辑检查:关键参数枚举校验 validate_critical_enums(request_params) - # 3. 清洗输出参数(生成唯一 run_id) + # 4. 清洗输出参数(生成唯一 run_id) run_id = generate_run_id() if self.sanitize_output: request_params = sanitize_output_params(request_params, run_id) diff --git a/src/cpp_isolated_runner.py b/src/cpp_isolated_runner.py new file mode 100644 index 0000000..41e6ce6 --- /dev/null +++ b/src/cpp_isolated_runner.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import argparse +import ctypes +import json +import os +import sys +from typing import Any, Dict, List, Optional + +import numpy as np + +from src.integrity import ensure_file_sha256, ensure_within_allowed_dirs + + +def _apply_abi(func: Any, cfg: Dict[str, Any]) -> None: + # Default ABI: void (double*, int, double*, int) + func.argtypes = [ + ctypes.POINTER(ctypes.c_double), + ctypes.c_int, + ctypes.POINTER(ctypes.c_double), + ctypes.c_int, + ] + func.restype = None + + argtypes = cfg.get("argtypes") + if isinstance(argtypes, list) and argtypes: + mapping = {"double_ptr": ctypes.POINTER(ctypes.c_double), "int": ctypes.c_int} + func.argtypes = [mapping.get(t, ctypes.c_void_p) for t in argtypes] + + restype = cfg.get("restype") + if isinstance(restype, str) and restype: + mapping = {"int": ctypes.c_int, "void": None} + func.restype = mapping.get(restype, None) + + +def run(config_path: str, input_path: str, output_path: str) -> int: + with open(config_path, "r", encoding="utf-8") as f: + cfg: Dict[str, Any] = json.load(f) + + lib_path = cfg.get("lib_path") or "" + function_name = cfg.get("function_name") or "" + expected_sha256 = cfg.get("expected_sha256") or None + + if not lib_path or not function_name: + raise ValueError("config 缺少 lib_path/function_name") + + abs_lib_path = ensure_within_allowed_dirs(lib_path) + if not os.path.exists(abs_lib_path): + raise FileNotFoundError(f"C++ library not found: {abs_lib_path}") + + if expected_sha256: + ensure_file_sha256(abs_lib_path, expected_sha256, label="cpp library (isolated)") + + in_data = np.load(input_path, allow_pickle=False) + if not isinstance(in_data, np.ndarray): + raise TypeError("input_npy 不是 ndarray") + in_data = np.ascontiguousarray(in_data, dtype=np.float64) + + out_data = np.zeros_like(in_data) + + lib = ctypes.CDLL(abs_lib_path) + func = getattr(lib, function_name) + _apply_abi(func, cfg) + + in_ptr = in_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double)) + out_ptr = out_data.ctypes.data_as(ctypes.POINTER(ctypes.c_double)) + in_len = ctypes.c_int(in_data.size) + out_len = ctypes.c_int(out_data.size) + + func(in_ptr, in_len, out_ptr, out_len) + + np.save(output_path, out_data, allow_pickle=False) + return 0 + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Run C++ bridge call in isolated subprocess") + parser.add_argument("--config", required=True, help="Path to JSON config") + parser.add_argument("--input", required=True, help="Path to input .npy") + parser.add_argument("--output", required=True, help="Path to output .npy") + args = parser.parse_args(argv) + + try: + return run(args.config, args.input, args.output) + except Exception as e: + print(f"[cpp_isolated_runner] error: {e}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/env_detect.py b/src/env_detect.py index 493819a..b5a931a 100644 --- a/src/env_detect.py +++ b/src/env_detect.py @@ -5,7 +5,7 @@ import os import platform import shutil import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple def _has_module(module_name: str) -> bool: diff --git a/src/integrity.py b/src/integrity.py new file mode 100644 index 0000000..7d6a09a --- /dev/null +++ b/src/integrity.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import hashlib +import os +from typing import Tuple + + +DEFAULT_ALLOWED_DIRS: Tuple[str, ...] = ("src", "data") + + +def project_root() -> str: + return os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def resolve_project_path(path: str) -> str: + if not path: + raise ValueError("path 不能为空") + + if os.path.isabs(path): + return os.path.realpath(path) + return os.path.realpath(os.path.join(project_root(), path)) + + +def ensure_within_allowed_dirs(path: str, allowed_dirs: Tuple[str, ...] = DEFAULT_ALLOWED_DIRS) -> str: + abs_path = resolve_project_path(path) + root = project_root() + allowed_abs = [os.path.realpath(os.path.join(root, d)) for d in allowed_dirs] + + if not any(os.path.commonpath([abs_path, d]) == d for d in allowed_abs): + raise PermissionError( + f"Security Block: Path {abs_path} is outside allowed directories: {', '.join(allowed_dirs)}" + ) + return abs_path + + +def sha256_file(path: str) -> str: + abs_path = resolve_project_path(path) + digest = hashlib.sha256() + with open(abs_path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def ensure_file_sha256(path: str, expected_sha256: str, label: str = "file") -> str: + if not expected_sha256: + return sha256_file(path) + + expected = expected_sha256.strip().lower() + if len(expected) != 64 or any(c not in "0123456789abcdef" for c in expected): + raise ValueError(f"{label} expected_sha256 非法: {expected_sha256!r}") + + actual = sha256_file(path).lower() + if actual != expected: + raise PermissionError( + f"SHA256 mismatch for {label}: expected {expected}, got {actual} (path={resolve_project_path(path)})" + ) + return actual +