feat: Worker 进程拆分(F20)
- 新增 worker 进程入口(npm run worker)只跑作业不监听端口 - trialrun compose 增加 worker profile(server 可 JOBS_ENABLED=false) - 更新 runbook/.env.example/README 说明与 v3 CSV 状态推进
这个提交包含在:
@@ -54,9 +54,14 @@ DELEGATION_REVOKE_CRON="0 * * * *"
|
|||||||
|
|
||||||
# ===== 作业系统(pg-boss,可回滚到 cron)=====
|
# ===== 作业系统(pg-boss,可回滚到 cron)=====
|
||||||
JOB_DRIVER="pgboss" # cron|pgboss
|
JOB_DRIVER="pgboss" # cron|pgboss
|
||||||
JOBS_ENABLED="true" # false 时整体关闭作业(排障/回滚)
|
JOBS_ENABLED="true" # false 时整体关闭作业(排障/回滚/或启用 worker 拆分)
|
||||||
PG_BOSS_SCHEMA="pgboss" # pg-boss 在 PG 内的 schema
|
PG_BOSS_SCHEMA="pgboss" # pg-boss 在 PG 内的 schema
|
||||||
|
|
||||||
|
# ===== Worker 拆分(可选,F20)=====
|
||||||
|
# 如需启用“API 进程 + Worker 进程”:
|
||||||
|
# 1) 设置 JOBS_ENABLED="false"(server 不跑作业)
|
||||||
|
# 2) 启动 compose 时加:--profile worker(会启动 worker 容器,worker 内强制 JOBS_ENABLED=true)
|
||||||
|
|
||||||
# ===== 试运行引导管理员(E20)=====
|
# ===== 试运行引导管理员(E20)=====
|
||||||
TRIALRUN_ADMIN_EMAIL="[email protected]"
|
TRIALRUN_ADMIN_EMAIL="[email protected]"
|
||||||
TRIALRUN_ADMIN_PHONE="13800000000"
|
TRIALRUN_ADMIN_PHONE="13800000000"
|
||||||
|
|||||||
@@ -64,6 +64,44 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./data/storage:${STORAGE_DIR}
|
- ./data/storage:${STORAGE_DIR}
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build:
|
||||||
|
context: ../..
|
||||||
|
dockerfile: deploy/trialrun/Dockerfile
|
||||||
|
container_name: hyb-oa-trialrun-worker
|
||||||
|
restart: unless-stopped
|
||||||
|
profiles: ["worker"]
|
||||||
|
depends_on:
|
||||||
|
db:
|
||||||
|
condition: service_healthy
|
||||||
|
command: ["npm", "run", "worker"]
|
||||||
|
environment:
|
||||||
|
NODE_ENV: production
|
||||||
|
APP_ORIGIN: ${APP_ORIGIN}
|
||||||
|
JWT_SECRET: ${JWT_SECRET}
|
||||||
|
DATABASE_URL: ${DATABASE_URL}
|
||||||
|
STORAGE_DIR: ${STORAGE_DIR}
|
||||||
|
SERVE_CLIENT_DIST: "false"
|
||||||
|
CLIENT_DIST_DIR: ${CLIENT_DIST_DIR}
|
||||||
|
|
||||||
|
OCR_PROVIDER: ${OCR_PROVIDER}
|
||||||
|
OCR_MAX_ATTEMPTS: ${OCR_MAX_ATTEMPTS}
|
||||||
|
OCR_POLL_CRON: ${OCR_POLL_CRON}
|
||||||
|
OCR_BATCH_SIZE: ${OCR_BATCH_SIZE}
|
||||||
|
OCR_LOCK_TTL_SECONDS: ${OCR_LOCK_TTL_SECONDS}
|
||||||
|
OCR_TIMEOUT_MS: ${OCR_TIMEOUT_MS}
|
||||||
|
TENCENT_SECRET_ID: ${TENCENT_SECRET_ID}
|
||||||
|
TENCENT_SECRET_KEY: ${TENCENT_SECRET_KEY}
|
||||||
|
TENCENT_OCR_REGION: ${TENCENT_OCR_REGION}
|
||||||
|
|
||||||
|
DELEGATION_REVOKE_CRON: ${DELEGATION_REVOKE_CRON}
|
||||||
|
|
||||||
|
JOB_DRIVER: ${JOB_DRIVER}
|
||||||
|
JOBS_ENABLED: "true"
|
||||||
|
PG_BOSS_SCHEMA: ${PG_BOSS_SCHEMA}
|
||||||
|
volumes:
|
||||||
|
- ./data/storage:${STORAGE_DIR}
|
||||||
|
|
||||||
init:
|
init:
|
||||||
build:
|
build:
|
||||||
context: ../..
|
context: ../..
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ cp deploy/trialrun/.env.example deploy/trialrun/.env
|
|||||||
作业系统建议:
|
作业系统建议:
|
||||||
- 试运行默认 `JOB_DRIVER=pgboss`(基于 Postgres 的队列调度,单机/多进程语义更稳)
|
- 试运行默认 `JOB_DRIVER=pgboss`(基于 Postgres 的队列调度,单机/多进程语义更稳)
|
||||||
- 如需快速回滚作业调度:将 `JOB_DRIVER` 改回 `cron`
|
- 如需快速回滚作业调度:将 `JOB_DRIVER` 改回 `cron`
|
||||||
|
- 可选 Worker 拆分(F20):将 `JOBS_ENABLED=false` 并启用 `--profile worker`(worker 进程会强制跑作业)
|
||||||
|
|
||||||
### 1.2 启动数据库与服务
|
### 1.2 启动数据库与服务
|
||||||
```bash
|
```bash
|
||||||
@@ -29,6 +30,14 @@ docker compose -f deploy/trialrun/docker-compose.yml --env-file deploy/trialrun/
|
|||||||
docker compose -f deploy/trialrun/docker-compose.yml --env-file deploy/trialrun/.env up -d --build server
|
docker compose -f deploy/trialrun/docker-compose.yml --env-file deploy/trialrun/.env up -d --build server
|
||||||
```
|
```
|
||||||
|
|
||||||
|
可选:启用 Worker 进程(推荐在试运行/多进程模式使用):
|
||||||
|
```bash
|
||||||
|
cd /path/to/hyb-oa
|
||||||
|
# 1) 在 deploy/trialrun/.env 中设置:JOBS_ENABLED="false"
|
||||||
|
# 2) 启用 worker profile(会额外启动 worker 容器)
|
||||||
|
docker compose -f deploy/trialrun/docker-compose.yml --env-file deploy/trialrun/.env --profile worker up -d --build server worker
|
||||||
|
```
|
||||||
|
|
||||||
### 1.3 初始化(迁移 + 配置 seed + 试运行管理员)
|
### 1.3 初始化(迁移 + 配置 seed + 试运行管理员)
|
||||||
```bash
|
```bash
|
||||||
cd /path/to/hyb-oa
|
cd /path/to/hyb-oa
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ F19,作业系统升级(pg-boss 基础设施),"引入 pg-boss(Postgres 队
|
|||||||
F19b,作业迁移(OCR -> pg-boss),"将 OCR 异步处理迁移到 pg-boss:保持并发锁语义、幂等与回归脚本不变","mock/tencent 两种 provider 下 OCR 仍可跑;并发下单条仅处理一次;回归全绿","cd server && ./scripts/regression-ocr.sh && ./scripts/regression-all.sh",manual,DONE,DONE,DONE,"server/src/jobs/pgBossJobs.js | server/src/jobs/ocrJob.js | server/src/jobs/jobSystem.js | server/src/config.js | server/README.md","[""F19""]","pg-boss schedule 支持秒级 cron;OCR_PROVIDER=off 时自动 unschedule ocr.tick"
|
F19b,作业迁移(OCR -> pg-boss),"将 OCR 异步处理迁移到 pg-boss:保持并发锁语义、幂等与回归脚本不变","mock/tencent 两种 provider 下 OCR 仍可跑;并发下单条仅处理一次;回归全绿","cd server && ./scripts/regression-ocr.sh && ./scripts/regression-all.sh",manual,DONE,DONE,DONE,"server/src/jobs/pgBossJobs.js | server/src/jobs/ocrJob.js | server/src/jobs/jobSystem.js | server/src/config.js | server/README.md","[""F19""]","pg-boss schedule 支持秒级 cron;OCR_PROVIDER=off 时自动 unschedule ocr.tick"
|
||||||
F19c,作业迁移(委派回收 -> pg-boss),"将委派过期回收迁移到 pg-boss:保持 FOR UPDATE SKIP LOCKED + UPDATE RETURNING 原子认领语义","多进程并发下审计不重复写;回收逻辑不变;回归全绿","cd server && RUN_DELEGATION_CONCURRENCY_CHECK=1 ./scripts/regression-workflow-actions.sh",manual,DONE,DONE,DONE,"server/src/jobs/pgBossJobs.js | server/src/jobs/delegationRevokeJob.js | server/src/jobs/jobSystem.js | server/README.md","[""F19""]","仍保留原子认领+审计只基于 RETURNING 结果集"
|
F19c,作业迁移(委派回收 -> pg-boss),"将委派过期回收迁移到 pg-boss:保持 FOR UPDATE SKIP LOCKED + UPDATE RETURNING 原子认领语义","多进程并发下审计不重复写;回收逻辑不变;回归全绿","cd server && RUN_DELEGATION_CONCURRENCY_CHECK=1 ./scripts/regression-workflow-actions.sh",manual,DONE,DONE,DONE,"server/src/jobs/pgBossJobs.js | server/src/jobs/delegationRevokeJob.js | server/src/jobs/jobSystem.js | server/README.md","[""F19""]","仍保留原子认领+审计只基于 RETURNING 结果集"
|
||||||
F19d,作业迁移(超期归档 -> pg-boss),"将 90 天游离归档迁移到 pg-boss(替代 node-cron),统一重试/并发/可观测字段","作业可重复执行幂等;多进程不重复处理;回归全绿","cd server && ./scripts/regression-ocr.sh && ./scripts/regression-all.sh",manual,DONE,DONE,DONE,"server/src/jobs/pgBossJobs.js | server/src/jobs/invoiceArchiveJob.js | server/src/jobs/jobSystem.js | server/README.md","[""F19"",""F12""]","invoice.archive-unused 默认按 INVOICE_ARCHIVE_CRON 调度;回滚:JOB_DRIVER=cron"
|
F19d,作业迁移(超期归档 -> pg-boss),"将 90 天游离归档迁移到 pg-boss(替代 node-cron),统一重试/并发/可观测字段","作业可重复执行幂等;多进程不重复处理;回归全绿","cd server && ./scripts/regression-ocr.sh && ./scripts/regression-all.sh",manual,DONE,DONE,DONE,"server/src/jobs/pgBossJobs.js | server/src/jobs/invoiceArchiveJob.js | server/src/jobs/jobSystem.js | server/README.md","[""F19"",""F12""]","invoice.archive-unused 默认按 INVOICE_ARCHIVE_CRON 调度;回滚:JOB_DRIVER=cron"
|
||||||
F20,Worker进程拆分(可选),"提供 API 进程与 Worker 进程拆分的部署方式(compose 开关),保持单机也可跑","开启后:API 不跑作业;Worker 只跑作业;两者共享 DATABASE_URL;功能不回归","manual: docker compose 起两个进程并跑回归脚本",manual,TODO,TODO,TODO,"deploy/trialrun/docker-compose.yml | server/src/index.js | server/README.md","[""F19""]","仍保持单机;为未来横向扩展铺路"
|
F20,Worker进程拆分(可选),"提供 API 进程与 Worker 进程拆分的部署方式(compose 开关),保持单机也可跑","开启后:API 不跑作业;Worker 只跑作业;两者共享 DATABASE_URL;功能不回归","manual: docker compose 起两个进程并跑回归脚本",manual,DONE,DONE,DONE,"deploy/trialrun/docker-compose.yml | server/src/index.js | server/README.md","[""F19""]","仍保持单机;为未来横向扩展铺路"
|
||||||
F21,对象存储(MinIO/S3兼容),"存储从本地目录演进到 S3 兼容(先 MinIO),保持 signed-url 契约不变或提供兼容层","上传/预览/签名URL 在 MinIO 下可用;回归与 E2E 通过;迁移路径清晰可回滚","manual: trialrun 环境启用 MinIO 并完成一次上传预览",manual,TODO,TODO,TODO,"server/src/services/storage/* | deploy/trialrun/* | server/README.md","[""F20""]","优先用 @aws-sdk/client-s3;单机优先 MinIO"
|
F21,对象存储(MinIO/S3兼容),"存储从本地目录演进到 S3 兼容(先 MinIO),保持 signed-url 契约不变或提供兼容层","上传/预览/签名URL 在 MinIO 下可用;回归与 E2E 通过;迁移路径清晰可回滚","manual: trialrun 环境启用 MinIO 并完成一次上传预览",manual,TODO,TODO,TODO,"server/src/services/storage/* | deploy/trialrun/* | server/README.md","[""F20""]","优先用 @aws-sdk/client-s3;单机优先 MinIO"
|
||||||
F22,可观测性增强(metrics/日志),"新增 /metrics(Prometheus)与更完整的结构化日志字段规范(不落敏感)","metrics 可抓取关键指标(HTTP/作业/DB);日志具备 requestId/actorId/targetId 等字段","manual: curl /metrics 并核对字段;触发作业观察指标变化",manual,TODO,TODO,TODO,"server/src/app.js | server/src/observability/* | deploy/trialrun/*","[""F19""]","优先用 prom-client;OTel 放更后"
|
F22,可观测性增强(metrics/日志),"新增 /metrics(Prometheus)与更完整的结构化日志字段规范(不落敏感)","metrics 可抓取关键指标(HTTP/作业/DB);日志具备 requestId/actorId/targetId 等字段","manual: curl /metrics 并核对字段;触发作业观察指标变化",manual,TODO,TODO,TODO,"server/src/app.js | server/src/observability/* | deploy/trialrun/*","[""F19""]","优先用 prom-client;OTel 放更后"
|
||||||
F23,SSO/OIDC(可选,P7+),"支持 OIDC 登录(企业 SSO),保留现有密码登录作为 fallback","可通过 OIDC 登录并绑定员工;权限模型不变;可配置开关","manual: 配置 OIDC provider 后走一次登录回调",manual,TODO,TODO,TODO,"server/src/routes/auth.js | server/src/services/oidc/* | server/README.md","[""F1""]","优先复用 openid-client/passport 策略"
|
F23,SSO/OIDC(可选,P7+),"支持 OIDC 登录(企业 SSO),保留现有密码登录作为 fallback","可通过 OIDC 登录并绑定员工;权限模型不变;可配置开关","manual: 配置 OIDC provider 后走一次登录回调",manual,TODO,TODO,TODO,"server/src/routes/auth.js | server/src/services/oidc/* | server/README.md","[""F1""]","优先复用 openid-client/passport 策略"
|
||||||
|
|||||||
|
+21
@@ -92,6 +92,27 @@ npm run seed:trialrun
|
|||||||
- 回滚策略:
|
- 回滚策略:
|
||||||
- 将 `JOB_DRIVER` 切回 `cron`(无需 schema 迁移回滚,风险低)
|
- 将 `JOB_DRIVER` 切回 `cron`(无需 schema 迁移回滚,风险低)
|
||||||
|
|
||||||
|
### 2.3.4 Worker 进程拆分(可选,F20)
|
||||||
|
> 目的:单机也能用“API 进程 + Worker 进程”模式运行,便于未来横向扩展;不改变对外 API。
|
||||||
|
|
||||||
|
- API 进程:只提供 HTTP(可选托管 `client/dist`),**不跑作业**
|
||||||
|
- Worker 进程:只跑作业(OCR/委派回收/超期归档),**不监听端口**
|
||||||
|
|
||||||
|
本地双进程示例:
|
||||||
|
```bash
|
||||||
|
# 终端 A:API(不跑作业)
|
||||||
|
cd /home/shunli/Projects/hyb-oa/server
|
||||||
|
JOBS_ENABLED=false npm run dev
|
||||||
|
|
||||||
|
# 终端 B:Worker(只跑作业)
|
||||||
|
cd /home/shunli/Projects/hyb-oa/server
|
||||||
|
JOBS_ENABLED=true npm run worker
|
||||||
|
```
|
||||||
|
|
||||||
|
试运行 compose(推荐,见 `deploy/trialrun/runbook.md`):
|
||||||
|
- 启用 worker profile:`--profile worker`
|
||||||
|
- 同时将 server 的 `JOBS_ENABLED=false`(worker 进程内强制为 true)
|
||||||
|
|
||||||
### 2.4 启动服务
|
### 2.4 启动服务
|
||||||
```bash
|
```bash
|
||||||
cd /home/shunli/Projects/hyb-oa/server
|
cd /home/shunli/Projects/hyb-oa/server
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
"postinstall": "prisma generate",
|
"postinstall": "prisma generate",
|
||||||
"dev": "node src/index.js",
|
"dev": "node src/index.js",
|
||||||
"start": "node src/index.js",
|
"start": "node src/index.js",
|
||||||
|
"worker": "node src/worker.js",
|
||||||
"test": "echo \"Error: no test specified\" && exit 1",
|
"test": "echo \"Error: no test specified\" && exit 1",
|
||||||
"prisma:format": "prisma format",
|
"prisma:format": "prisma format",
|
||||||
"prisma:validate": "prisma validate",
|
"prisma:validate": "prisma validate",
|
||||||
|
|||||||
@@ -2,6 +2,13 @@ const { buildApp } = require("./app");
|
|||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
const app = await buildApp();
|
const app = await buildApp();
|
||||||
|
app.log.info(
|
||||||
|
{
|
||||||
|
jobDriver: app.config.jobDriver,
|
||||||
|
jobsEnabled: app.config.jobsEnabled,
|
||||||
|
},
|
||||||
|
"API process started",
|
||||||
|
);
|
||||||
|
|
||||||
const { host, port } = app.config;
|
const { host, port } = app.config;
|
||||||
await app.listen({ host, port });
|
await app.listen({ host, port });
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
const { buildApp } = require("./app");
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const app = await buildApp();
|
||||||
|
app.log.info(
|
||||||
|
{
|
||||||
|
jobDriver: app.config.jobDriver,
|
||||||
|
jobsEnabled: app.config.jobsEnabled,
|
||||||
|
},
|
||||||
|
"Worker process started",
|
||||||
|
);
|
||||||
|
|
||||||
|
let shuttingDown = false;
|
||||||
|
const shutdown = async (signal) => {
|
||||||
|
if (shuttingDown) return;
|
||||||
|
shuttingDown = true;
|
||||||
|
|
||||||
|
app.log.info({ signal }, "Worker graceful shutdown start");
|
||||||
|
|
||||||
|
const timeoutMs = 10_000;
|
||||||
|
const forceTimer = setTimeout(() => {
|
||||||
|
app.log.error({ timeoutMs }, "Worker graceful shutdown timeout, force exit");
|
||||||
|
process.exit(1);
|
||||||
|
}, timeoutMs);
|
||||||
|
forceTimer.unref?.();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await app.close();
|
||||||
|
app.log.info("Worker graceful shutdown completed");
|
||||||
|
process.exit(0);
|
||||||
|
} catch (error) {
|
||||||
|
app.log.error({ err: error }, "Worker graceful shutdown failed");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
||||||
|
process.on("SIGINT", () => shutdown("SIGINT"));
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
// eslint-disable-next-line no-console
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
||||||
在新工单中引用
屏蔽一个用户