wangsli/hyb-oa
1
0
派生 0
代码 工单 合并请求 工作流 5 软件包 项目 发布 百科 活动

feat(server): /metrics 可观测与日志上下文（F22）

浏览代码 
- 新增 Prometheus 指标端点 /metrics（HTTP + DB + 作业统计）
- request.log 绑定 requestId/actorId，避免日志难追踪
- cron/pg-boss 作业增加 metrics 计数（runs/processed）
- 更新 env/runbook/README 与 v3 CSV 状态推进
这个提交包含在：
wangsli
2026-02-09 20:30:41 +08:00
父节点 55924c8ff8
当前提交 6ecebf8ad6
共修改 17 个文件，包含 353 行新增和 25 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件
server/.env.example
+3
查看文件
@@ -8,6 +8,9 @@ STORAGE_DIR="generated/storage"
SIGNED_URL_TTL_SECONDS="600"
MAX_UPLOAD_SIZE_BYTES="10485760"
# ===== Metrics（Prometheus）=====
# METRICS_ENABLED="true"
# ===== 单域托管 client/dist（试运行/生产建议）=====
# SERVE_CLIENT_DIST="true"
# CLIENT_DIST_DIR="../client/dist"
server/README.md
+2
查看文件
@@ -145,11 +145,13 @@ cd /home/shunli/Projects/hyb-oa/server
```bash
curl -fsS http://127.0.0.1:3000/health
curl -fsS http://127.0.0.1:3000/ready
curl -fsS http://127.0.0.1:3000/metrics | head
```
说明：
- `/health`：liveness（不查 DB）
- `/ready`：readiness（`SELECT 1` 探活 DB，失败返回 503）
- `/metrics`：Prometheus 指标（可通过 `METRICS_ENABLED=false` 关闭）
---
server/package-lock.json
自动生成
+38
查看文件
@@ -28,6 +28,7 @@
"pg": "^8.18.0",
"pg-boss": "^10.3.0",
"prisma": "^7.3.0",
"prom-client": "^15.1.0",
"tencentcloud-sdk-nodejs": "^4.1.14",
"zod": "^4.3.6",
"zxcvbn": "^4.4.2"
@@ -503,6 +504,15 @@
"node": ">=16"
}
},
"node_modules/@opentelemetry/api": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
"integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
"license": "Apache-2.0",
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/@pinojs/redact": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
@@ -975,6 +985,12 @@
"node": "*"
}
},
"node_modules/bintrees": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz",
"integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==",
"license": "MIT"
},
"node_modules/bl": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
@@ -3046,6 +3062,19 @@
],
"license": "MIT"
},
"node_modules/prom-client": {
"version": "15.1.3",
"resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz",
"integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==",
"license": "Apache-2.0",
"dependencies": {
"@opentelemetry/api": "^1.4.0",
"tdigest": "^0.1.1"
},
"engines": {
"node": "^16 || ^18 || >=20"
}
},
"node_modules/proper-lockfile": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz",
@@ -3540,6 +3569,15 @@
"node": ">=6"
}
},
"node_modules/tdigest": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz",
"integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==",
"license": "MIT",
"dependencies": {
"bintrees": "1.0.2"
}
},
"node_modules/tencentcloud-sdk-nodejs": {
"version": "4.1.184",
"resolved": "https://registry.npmjs.org/tencentcloud-sdk-nodejs/-/tencentcloud-sdk-nodejs-4.1.184.tgz",
server/package.json
+1
查看文件
@@ -41,6 +41,7 @@
"fastify-plugin": "^5.1.0",
"nodemailer": "^6.10.1",
"node-cron": "^4.2.1",
"prom-client": "^15.1.0",
"pg-boss": "^10.3.0",
"pg": "^8.18.0",
"prisma": "^7.3.0",
server/src/app.js
+21
查看文件
@@ -30,6 +30,7 @@ const approvalRulesRoutes = require("./routes/approvalRules");
const notificationsRoutes = require("./routes/notifications");
const exportsRoutes = require("./routes/exports");
const { startJobs } = require("./jobs/jobSystem");
const { initMetrics } = require("./observability");
const { AppError } = require("./utils/AppError");
async function buildApp() {
@@ -112,6 +113,9 @@ async function buildApp() {
app.addHook("onRequest", async (request, reply) => {
reply.header("x-request-id", request.id);
if (request.log && typeof request.log.child === "function") {
request.log = request.log.child({ requestId: request.id });
}
});
app.setNotFoundHandler(async (request, reply) => {
@@ -189,6 +193,23 @@ async function buildApp() {
}
});
if (config.metricsEnabled) {
initMetrics(app);
app.get("/metrics", async (request, reply) => {
try {
await app.prisma.$queryRaw`SELECT 1`;
app.metrics.dbReady.set(1);
} catch {
app.metrics.dbReady.set(0);
}
const payload = await app.metrics.registry.metrics();
reply.header("content-type", app.metrics.registry.contentType);
return reply.send(payload);
});
app.log.info("Metrics endpoint enabled at /metrics");
}
await app.register(authRoutes, { prefix: "/api/auth" });
await app.register(meRoutes, { prefix: "/api" });
await app.register(reimbursementRoutes, { prefix: "/api" });
server/src/config.js
+2
查看文件
@@ -22,6 +22,8 @@ function getConfig() {
port: Number(process.env.PORT ?? 3000),
host: process.env.HOST ?? "0.0.0.0",
metricsEnabled: process.env.METRICS_ENABLED !== "false",
appOrigin: process.env.APP_ORIGIN ?? "",
jwtSecret: requiredEnv("JWT_SECRET"),
server/src/jobs/delegationRevokeJob.js
+21 -1
查看文件
@@ -92,9 +92,29 @@ function startDelegationRevokeJob(fastify) {
running = true;
try {
await revokeExpiredDelegations(fastify.prisma, fastify.log);
const revoked = await revokeExpiredDelegations(fastify.prisma, fastify.log);
fastify.metrics?.jobRunsTotal?.inc?.({
job: "delegation.revoke-expired",
driver: "cron",
outcome: "success",
});
if (revoked > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{
job: "delegation.revoke-expired",
driver: "cron",
kind: "revoked",
},
revoked,
);
}
} catch (error) {
fastify.log.error({ err: error }, "Delegation revoke job failed");
fastify.metrics?.jobRunsTotal?.inc?.({
job: "delegation.revoke-expired",
driver: "cron",
outcome: "failure",
});
} finally {
running = false;
}
server/src/jobs/invoiceArchiveJob.js
+21 -2
查看文件
@@ -76,12 +76,32 @@ function startInvoiceArchiveJob(fastify) {
running = true;
try {
await archiveUnusedInvoices(fastify.prisma, fastify.log, {
const archived = await archiveUnusedInvoices(fastify.prisma, fastify.log, {
batchSize: fastify.config.invoiceArchiveBatchSize,
unusedDays: fastify.config.invoiceArchiveUnusedDays,
});
fastify.metrics?.jobRunsTotal?.inc?.({
job: "invoice.archive-unused",
driver: "cron",
outcome: "success",
});
if (archived > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{
job: "invoice.archive-unused",
driver: "cron",
kind: "archived",
},
archived,
);
}
} catch (error) {
fastify.log.error({ err: error }, "Invoice archive job failed");
fastify.metrics?.jobRunsTotal?.inc?.({
job: "invoice.archive-unused",
driver: "cron",
outcome: "failure",
});
} finally {
running = false;
}
@@ -107,4 +127,3 @@ module.exports = {
archiveUnusedInvoices,
startInvoiceArchiveJob,
};
server/src/jobs/ocrJob.js
+28
查看文件
@@ -301,6 +301,29 @@ function startOcrJob(fastify) {
running = true;
try {
const result = await runOcrOnce(fastify.prisma, fastify.config, fastify.log, workerId);
fastify.metrics?.jobRunsTotal?.inc?.({
job: "ocr.tick",
driver: "cron",
outcome: "success",
});
if (result.available > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{ job: "ocr.tick", driver: "cron", kind: "available" },
result.available,
);
}
if (result.duplicate > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{ job: "ocr.tick", driver: "cron", kind: "duplicate" },
result.duplicate,
);
}
if (result.failed > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{ job: "ocr.tick", driver: "cron", kind: "failed" },
result.failed,
);
}
if (result.claimed > 0) {
fastify.log.info(
{
@@ -312,6 +335,11 @@ function startOcrJob(fastify) {
}
} catch (error) {
fastify.log.error({ err: error }, "OCR job tick failed");
fastify.metrics?.jobRunsTotal?.inc?.({
job: "ocr.tick",
driver: "cron",
outcome: "failure",
});
} finally {
running = false;
}
server/src/jobs/pgBossJobs.js
+93 -21
查看文件
@@ -38,26 +38,64 @@ async function startPgBossJobs(fastify) {
await boss.work(JOB_DELEGATION_REVOKE, { batchSize: 1 }, async (jobs) => {
const job = jobs?.[0];
const revoked = await revokeExpiredDelegations(fastify.prisma, fastify.log);
if (revoked > 0) {
fastify.log.info(
{ jobId: job?.id, revoked },
"pg-boss delegation revoke tick processed",
);
try {
const revoked = await revokeExpiredDelegations(fastify.prisma, fastify.log);
fastify.metrics?.jobRunsTotal?.inc?.({
job: JOB_DELEGATION_REVOKE,
driver: "pgboss",
outcome: "success",
});
if (revoked > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{ job: JOB_DELEGATION_REVOKE, driver: "pgboss", kind: "revoked" },
revoked,
);
fastify.log.info(
{ jobId: job?.id, revoked },
"pg-boss delegation revoke tick processed",
);
}
} catch (error) {
fastify.metrics?.jobRunsTotal?.inc?.({
job: JOB_DELEGATION_REVOKE,
driver: "pgboss",
outcome: "failure",
});
fastify.log.error({ err: error, jobId: job?.id }, "pg-boss delegation revoke tick failed");
throw error;
}
});
await boss.work(JOB_INVOICE_ARCHIVE, { batchSize: 1 }, async (jobs) => {
const job = jobs?.[0];
const archived = await archiveUnusedInvoices(fastify.prisma, fastify.log, {
batchSize: fastify.config.invoiceArchiveBatchSize,
unusedDays: fastify.config.invoiceArchiveUnusedDays,
});
if (archived > 0) {
fastify.log.info(
{ jobId: job?.id, archived, unusedDays: fastify.config.invoiceArchiveUnusedDays },
"pg-boss invoice archive tick processed",
);
try {
const archived = await archiveUnusedInvoices(fastify.prisma, fastify.log, {
batchSize: fastify.config.invoiceArchiveBatchSize,
unusedDays: fastify.config.invoiceArchiveUnusedDays,
});
fastify.metrics?.jobRunsTotal?.inc?.({
job: JOB_INVOICE_ARCHIVE,
driver: "pgboss",
outcome: "success",
});
if (archived > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{ job: JOB_INVOICE_ARCHIVE, driver: "pgboss", kind: "archived" },
archived,
);
fastify.log.info(
{ jobId: job?.id, archived, unusedDays: fastify.config.invoiceArchiveUnusedDays },
"pg-boss invoice archive tick processed",
);
}
} catch (error) {
fastify.metrics?.jobRunsTotal?.inc?.({
job: JOB_INVOICE_ARCHIVE,
driver: "pgboss",
outcome: "failure",
});
fastify.log.error({ err: error, jobId: job?.id }, "pg-boss invoice archive tick failed");
throw error;
}
});
@@ -68,12 +106,46 @@ async function startPgBossJobs(fastify) {
const job = jobs?.[0];
const workerId = `pgboss:ocr:${process.pid}`;
const stats = await runOcrOnce(fastify.prisma, fastify.config, fastify.log, workerId);
if (stats.claimed > 0) {
fastify.log.info(
{ jobId: job?.id, workerId, ...stats },
"pg-boss OCR tick processed",
);
try {
const stats = await runOcrOnce(fastify.prisma, fastify.config, fastify.log, workerId);
fastify.metrics?.jobRunsTotal?.inc?.({
job: JOB_OCR_TICK,
driver: "pgboss",
outcome: "success",
});
if (stats.available > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{ job: JOB_OCR_TICK, driver: "pgboss", kind: "available" },
stats.available,
);
}
if (stats.duplicate > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{ job: JOB_OCR_TICK, driver: "pgboss", kind: "duplicate" },
stats.duplicate,
);
}
if (stats.failed > 0) {
fastify.metrics?.jobProcessedTotal?.inc?.(
{ job: JOB_OCR_TICK, driver: "pgboss", kind: "failed" },
stats.failed,
);
}
if (stats.claimed > 0) {
fastify.log.info(
{ jobId: job?.id, workerId, ...stats },
"pg-boss OCR tick processed",
);
}
} catch (error) {
fastify.metrics?.jobRunsTotal?.inc?.({
job: JOB_OCR_TICK,
driver: "pgboss",
outcome: "failure",
});
fastify.log.error({ err: error, jobId: job?.id }, "pg-boss OCR tick failed");
throw error;
}
});
server/src/observability/index.js
+6
查看文件
@@ -0,0 +1,6 @@
const { initMetrics } = require("./metrics");
module.exports = {
initMetrics,
};
server/src/observability/metrics.js
+105
查看文件
@@ -0,0 +1,105 @@
const client = require("prom-client");
function resolveRouteLabel(request) {
const url = request?.routeOptions?.url;
if (typeof url === "string" && url.trim()) {
return url;
}
const routerPath = request?.routerPath;
if (typeof routerPath === "string" && routerPath.trim()) {
return routerPath;
}
const rawUrl = request?.raw?.url;
if (typeof rawUrl === "string" && rawUrl.trim()) {
return rawUrl.split("?")[0];
}
return "unknown";
}
function initMetrics(app) {
const registry = new client.Registry();
client.collectDefaultMetrics({
register: registry,
});
const httpRequestsTotal = new client.Counter({
name: "hyb_http_requests_total",
help: "Total number of HTTP requests",
labelNames: ["method", "route", "status_code"],
registers: [registry],
});
const httpRequestDurationSeconds = new client.Histogram({
name: "hyb_http_request_duration_seconds",
help: "HTTP request duration in seconds",
labelNames: ["method", "route", "status_code"],
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
registers: [registry],
});
const jobRunsTotal = new client.Counter({
name: "hyb_job_runs_total",
help: "Total number of background job runs",
labelNames: ["job", "driver", "outcome"],
registers: [registry],
});
const jobProcessedTotal = new client.Counter({
name: "hyb_job_processed_total",
help: "Total number of processed items per job",
labelNames: ["job", "driver", "kind"],
registers: [registry],
});
const dbReady = new client.Gauge({
name: "hyb_db_ready",
help: "Database readiness probe result (1=ready, 0=not ready)",
registers: [registry],
});
app.decorate("metrics", {
registry,
httpRequestsTotal,
httpRequestDurationSeconds,
jobRunsTotal,
jobProcessedTotal,
dbReady,
});
app.decorateRequest("metricsStart", null);
app.addHook("onRequest", async (request) => {
request.metricsStart = process.hrtime.bigint();
});
app.addHook("onResponse", async (request, reply) => {
const start = request.metricsStart;
if (typeof start !== "bigint") {
return;
}
const end = process.hrtime.bigint();
const durationSeconds = Number(end - start) / 1e9;
const method = request.method;
const route = resolveRouteLabel(request);
const statusCode = String(reply.statusCode);
httpRequestsTotal.inc({ method, route, status_code: statusCode });
httpRequestDurationSeconds.observe(
{ method, route, status_code: statusCode },
durationSeconds,
);
});
return app.metrics;
}
module.exports = {
initMetrics,
};
server/src/plugins/auth.js
+4
查看文件
@@ -99,6 +99,10 @@ async function authPlugin(fastify) {
roleCodes,
permissionCodes,
};
if (request.log && typeof request.log.child === "function") {
request.log = request.log.child({ actorId: employee.id });
}
});
}