diff --git a/Dockerfile b/Dockerfile index a36b62184..215a35806 100644 --- a/Dockerfile +++ b/Dockerfile @@ -215,6 +215,8 @@ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh EXPOSE ${TCP_PORT} EXPOSE ${UDP_PORT} VOLUME /dc3-driver/dc3-driver-listening-virtual/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-listening-virtual/dc3/data/driver/listening-virtual +VOLUME /dc3-driver/dc3-driver-listening-virtual/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-listening-virtual.jar"] @@ -229,6 +231,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-modbus-tcp/dc3/logs/driver/modbus-tcp/gc COPY --from=builder /build/dc3-driver/dc3-driver-modbus-tcp/target/dc3-driver-modbus-tcp.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-modbus-tcp/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-modbus-tcp/dc3/data/driver/modbus-tcp +VOLUME /dc3-driver/dc3-driver-modbus-tcp/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-modbus-tcp.jar"] @@ -243,6 +247,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-modbus-rtu/dc3/logs/driver/modbus-rtu/gc COPY --from=builder /build/dc3-driver/dc3-driver-modbus-rtu/target/dc3-driver-modbus-rtu.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-modbus-rtu/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-modbus-rtu/dc3/data/driver/modbus-rtu +VOLUME /dc3-driver/dc3-driver-modbus-rtu/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-modbus-rtu.jar"] @@ -257,6 +263,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-mqtt/dc3/logs/driver/mqtt/gc COPY --from=builder /build/dc3-driver/dc3-driver-mqtt/target/dc3-driver-mqtt.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-mqtt/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-mqtt/dc3/data/driver/mqtt +VOLUME /dc3-driver/dc3-driver-mqtt/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-mqtt.jar"] @@ -271,6 +279,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-opc-da/dc3/logs/driver/opc-da/gc COPY --from=builder /build/dc3-driver/dc3-driver-opc-da/target/dc3-driver-opc-da.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-opc-da/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-opc-da/dc3/data/driver/opc-da +VOLUME /dc3-driver/dc3-driver-opc-da/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-opc-da.jar"] @@ -285,6 +295,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-opc-ua/dc3/logs/driver/opc-ua/gc COPY --from=builder /build/dc3-driver/dc3-driver-opc-ua/target/dc3-driver-opc-ua.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-opc-ua/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-opc-ua/dc3/data/driver/opc-ua +VOLUME /dc3-driver/dc3-driver-opc-ua/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-opc-ua.jar"] @@ -299,6 +311,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-plcs7/dc3/logs/driver/plcs7/gc COPY --from=builder /build/dc3-driver/dc3-driver-plcs7/target/dc3-driver-plcs7.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-plcs7/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-plcs7/dc3/data/driver/plcs7 +VOLUME /dc3-driver/dc3-driver-plcs7/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-plcs7.jar"] @@ -313,6 +327,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-virtual/dc3/logs/driver/virtual/gc COPY --from=builder /build/dc3-driver/dc3-driver-virtual/target/dc3-driver-virtual.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-virtual/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-virtual/dc3/data/driver/virtual +VOLUME /dc3-driver/dc3-driver-virtual/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-virtual.jar"] # ---------- dc3-driver-bacnet-ip ---------- @@ -325,6 +341,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-bacnet-ip/dc3/logs/driver/bacnet-ip/gc COPY --from=builder /build/dc3-driver/dc3-driver-bacnet-ip/target/dc3-driver-bacnet-ip.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-bacnet-ip/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-bacnet-ip/dc3/data/driver/bacnet-ip +VOLUME /dc3-driver/dc3-driver-bacnet-ip/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-bacnet-ip.jar"] @@ -338,6 +356,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-ble/dc3/logs/driver/ble/gc COPY --from=builder /build/dc3-driver/dc3-driver-ble/target/dc3-driver-ble.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-ble/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-ble/dc3/data/driver/ble +VOLUME /dc3-driver/dc3-driver-ble/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-ble.jar"] @@ -351,6 +371,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-can/dc3/logs/driver/can/gc COPY --from=builder /build/dc3-driver/dc3-driver-can/target/dc3-driver-can.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-can/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-can/dc3/data/driver/can +VOLUME /dc3-driver/dc3-driver-can/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-can.jar"] @@ -364,6 +386,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-coap/dc3/logs/driver/coap/gc COPY --from=builder /build/dc3-driver/dc3-driver-coap/target/dc3-driver-coap.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-coap/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-coap/dc3/data/driver/coap +VOLUME /dc3-driver/dc3-driver-coap/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-coap.jar"] @@ -377,6 +401,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-dlms/dc3/logs/driver/dlms/gc COPY --from=builder /build/dc3-driver/dc3-driver-dlms/target/dc3-driver-dlms.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-dlms/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-dlms/dc3/data/driver/dlms +VOLUME /dc3-driver/dc3-driver-dlms/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-dlms.jar"] @@ -390,6 +416,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-ethernet-ip/dc3/logs/driver/ethernet-ip/gc COPY --from=builder /build/dc3-driver/dc3-driver-ethernet-ip/target/dc3-driver-ethernet-ip.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-ethernet-ip/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-ethernet-ip/dc3/data/driver/ethernet-ip +VOLUME /dc3-driver/dc3-driver-ethernet-ip/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-ethernet-ip.jar"] @@ -403,6 +431,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-fins/dc3/logs/driver/fins/gc COPY --from=builder /build/dc3-driver/dc3-driver-fins/target/dc3-driver-fins.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-fins/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-fins/dc3/data/driver/fins +VOLUME /dc3-driver/dc3-driver-fins/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-fins.jar"] @@ -416,6 +446,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-http/dc3/logs/driver/http/gc COPY --from=builder /build/dc3-driver/dc3-driver-http/target/dc3-driver-http.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-http/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-http/dc3/data/driver/http +VOLUME /dc3-driver/dc3-driver-http/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-http.jar"] @@ -429,6 +461,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-iec104/dc3/logs/driver/iec104/gc COPY --from=builder /build/dc3-driver/dc3-driver-iec104/target/dc3-driver-iec104.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-iec104/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-iec104/dc3/data/driver/iec104 +VOLUME /dc3-driver/dc3-driver-iec104/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-iec104.jar"] @@ -442,6 +476,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-lwm2m/dc3/logs/driver/lwm2m/gc COPY --from=builder /build/dc3-driver/dc3-driver-lwm2m/target/dc3-driver-lwm2m.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-lwm2m/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-lwm2m/dc3/data/driver/lwm2m +VOLUME /dc3-driver/dc3-driver-lwm2m/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-lwm2m.jar"] @@ -455,6 +491,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-melsec/dc3/logs/driver/melsec/gc COPY --from=builder /build/dc3-driver/dc3-driver-melsec/target/dc3-driver-melsec.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-melsec/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-melsec/dc3/data/driver/melsec +VOLUME /dc3-driver/dc3-driver-melsec/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-melsec.jar"] @@ -468,6 +506,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-mysql/dc3/logs/driver/mysql/gc COPY --from=builder /build/dc3-driver/dc3-driver-mysql/target/dc3-driver-mysql.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-mysql/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-mysql/dc3/data/driver/mysql +VOLUME /dc3-driver/dc3-driver-mysql/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-mysql.jar"] @@ -481,6 +521,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-oracle/dc3/logs/driver/oracle/gc COPY --from=builder /build/dc3-driver/dc3-driver-oracle/target/dc3-driver-oracle.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-oracle/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-oracle/dc3/data/driver/oracle +VOLUME /dc3-driver/dc3-driver-oracle/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-oracle.jar"] @@ -494,6 +536,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-postgresql/dc3/logs/driver/postgresql/gc COPY --from=builder /build/dc3-driver/dc3-driver-postgresql/target/dc3-driver-postgresql.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-postgresql/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-postgresql/dc3/data/driver/postgresql +VOLUME /dc3-driver/dc3-driver-postgresql/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-postgresql.jar"] @@ -507,6 +551,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-serial/dc3/logs/driver/serial/gc COPY --from=builder /build/dc3-driver/dc3-driver-serial/target/dc3-driver-serial.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-serial/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-serial/dc3/data/driver/serial +VOLUME /dc3-driver/dc3-driver-serial/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-serial.jar"] @@ -520,6 +566,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-sl651/dc3/logs/driver/sl651/gc COPY --from=builder /build/dc3-driver/dc3-driver-sl651/target/dc3-driver-sl651.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-sl651/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-sl651/dc3/data/driver/sl651 +VOLUME /dc3-driver/dc3-driver-sl651/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-sl651.jar"] @@ -533,6 +581,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-snmp/dc3/logs/driver/snmp/gc COPY --from=builder /build/dc3-driver/dc3-driver-snmp/target/dc3-driver-snmp.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-snmp/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-snmp/dc3/data/driver/snmp +VOLUME /dc3-driver/dc3-driver-snmp/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-snmp.jar"] @@ -546,6 +596,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-sqlserver/dc3/logs/driver/sqlserver/gc COPY --from=builder /build/dc3-driver/dc3-driver-sqlserver/target/dc3-driver-sqlserver.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-sqlserver/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-sqlserver/dc3/data/driver/sqlserver +VOLUME /dc3-driver/dc3-driver-sqlserver/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-sqlserver.jar"] @@ -559,6 +611,8 @@ RUN mkdir -p /dc3-driver/dc3-driver-tcp-udp/dc3/logs/driver/tcp-udp/gc COPY --from=builder /build/dc3-driver/dc3-driver-tcp-udp/target/dc3-driver-tcp-udp.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-tcp-udp/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-tcp-udp/dc3/data/driver/tcp-udp +VOLUME /dc3-driver/dc3-driver-tcp-udp/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-tcp-udp.jar"] @@ -572,5 +626,7 @@ RUN mkdir -p /dc3-driver/dc3-driver-zigbee/dc3/logs/driver/zigbee/gc COPY --from=builder /build/dc3-driver/dc3-driver-zigbee/target/dc3-driver-zigbee.jar ./ RUN cp /usr/share/dc3/entrypoint.sh ./entrypoint.sh VOLUME /dc3-driver/dc3-driver-zigbee/dc3/logs +RUN mkdir -p /dc3-driver/dc3-driver-zigbee/dc3/data/driver/zigbee +VOLUME /dc3-driver/dc3-driver-zigbee/dc3/data ENTRYPOINT ["./entrypoint.sh"] CMD ["dc3-driver-zigbee.jar"] diff --git a/Makefile b/Makefile index 37299e388..35c0cd961 100644 --- a/Makefile +++ b/Makefile @@ -134,6 +134,12 @@ help: @printf ' %s\n' 'make up GROUP=core' @printf ' %s\n' 'make up STACK=optional SERVICES="prometheus grafana"' @printf ' %s\n' 'make logs SERVICES="gateway agentic"' + @printf '%s\n' '' + @printf '%s\n' 'Documentation:' + @printf ' %-24s %s\n' 'make docs-dev' 'Start VitePress dev server (localhost:5173)' + @printf ' %-24s %s\n' 'make docs-build' 'Build static site to docs/.vitepress/dist/' + @printf ' %-24s %s\n' 'make docs-preview' 'Preview the built site' + @printf ' %-24s %s\n' 'make docs-version VERSION=X' 'Build versioned docs (archived, noindex)' env: @printf 'ENV_FILE=%s\n' "$(ENV_FILE)" @@ -266,4 +272,19 @@ changelog: openapi: @OPENAPI_BASE="$(OPENAPI_BASE)" dc3/bin/export_openapi.sh $(OPENAPI_OUT) +.PHONY: docs-dev docs-build docs-preview docs-version + +docs-dev: + cd docs && pnpm dev + +docs-build: + cd docs && pnpm build + +docs-preview: + cd docs && pnpm preview + +docs-version: + @test -n "$(VERSION)" || { echo 'Usage: make docs-version VERSION=2025.9'; exit 1; } + cd docs && DC3_DOCS_VERSION=$(VERSION) DC3_DOCS_IS_LATEST=false pnpm build + diff --git a/dc3-common/dc3-common-constant/src/main/java/io/github/pnoker/common/constant/driver/ScheduleConstant.java b/dc3-common/dc3-common-constant/src/main/java/io/github/pnoker/common/constant/driver/ScheduleConstant.java index 6742e9213..b18d3300b 100644 --- a/dc3-common/dc3-common-constant/src/main/java/io/github/pnoker/common/constant/driver/ScheduleConstant.java +++ b/dc3-common/dc3-common-constant/src/main/java/io/github/pnoker/common/constant/driver/ScheduleConstant.java @@ -54,6 +54,11 @@ public class ScheduleConstant { */ public static final String DEVICE_HEALTH_SCHEDULE_JOB = "device-health-schedule-job"; + /** + * Buffer republish schedule job + */ + public static final String BUFFER_REPUBLISH_SCHEDULE_JOB = "buffer-republish-schedule-job"; + /** * Driver health schedule cron */ diff --git a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/biz/impl/PointValueServiceImpl.java b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/biz/impl/PointValueServiceImpl.java index ad73ea9cb..655ec8c12 100644 --- a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/biz/impl/PointValueServiceImpl.java +++ b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/biz/impl/PointValueServiceImpl.java @@ -104,7 +104,13 @@ public class PointValueServiceImpl implements PointValueService { }).collect(Collectors.groupingBy(PointValueBO::getDeviceId)); group.forEach(this::savePointValuesToRepository); - alarmRuleTriggerService.processPointValues(pointValueBOList); + try { + alarmRuleTriggerService.processPointValues(pointValueBOList); + } catch (Exception e) { + // Alarm evaluation runs after persistence: a failure here must not trigger a + // re-queue that would re-insert the already-persisted rows. + log.warn("Alarm rule evaluation failed, size={}, skipped", pointValueBOList.size(), e); + } } @Override @@ -232,14 +238,15 @@ public class PointValueServiceImpl implements PointValueService { // local hot cache pointValueLocalCacheService.savePointValue(deviceId, pointValueBOList); - // other repository + // Repository persistence — wrap any failure (incl. checked IOException) so the + // ingest buffer can re-queue the batch for retry instead of silently dropping it. RepositoryService repositoryService = getFirstRepositoryService(); List> splitPointValueBOList = ListUtils.partition(pointValueBOList, 100); for (List splitPointValueBO : splitPointValueBOList) { repositoryService.savePointValues(splitPointValueBO); } } catch (Exception e) { - log.error("Save point values failed, deviceId={}, size={}", deviceId, pointValueBOList.size(), e); + throw new RepositoryException(e); } } diff --git a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/biz/impl/ScheduleForDataServiceImpl.java b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/biz/impl/ScheduleForDataServiceImpl.java index 4e7bea46f..495e188b5 100644 --- a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/biz/impl/ScheduleForDataServiceImpl.java +++ b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/biz/impl/ScheduleForDataServiceImpl.java @@ -19,44 +19,38 @@ package io.github.pnoker.common.data.biz.impl; import io.github.pnoker.common.constant.driver.ScheduleConstant; import io.github.pnoker.common.data.biz.ScheduleForDataService; -import io.github.pnoker.common.data.entity.property.PointBatchProperties; import io.github.pnoker.common.data.job.HourlyJobForData; -import io.github.pnoker.common.data.job.PointValueJob; import io.github.pnoker.common.exception.ServiceException; import io.github.pnoker.common.quartz.QuartzService; import lombok.RequiredArgsConstructor; -import org.quartz.DateBuilder; import org.quartz.SchedulerException; import org.springframework.stereotype.Service; /** * Business service implementation for data-center scheduled jobs. * + *

Point-value ingestion no longer has a Quartz tick here — it is driven by + * {@link io.github.pnoker.common.data.buffer.PointValueIngestBuffer}'s worker threads. Only the + * hourly maintenance job remains. + * * @author pnoker - * @version 2025.9.0 + * @version 2026.7.8 * @since 2016.10.1 */ @Service @RequiredArgsConstructor public class ScheduleForDataServiceImpl implements ScheduleForDataService { - private final PointBatchProperties pointBatchProperties; - private final QuartzService quartzService; /** - * Initialize data scheduling + * Initialize data scheduling. */ @Override public void initial() { try { - quartzService.createJobWithInterval(ScheduleConstant.DATA_SCHEDULE_GROUP, "data-point-value-schedule-job", - pointBatchProperties.getInterval(), DateBuilder.IntervalUnit.SECOND, PointValueJob.class); - - // Custom scheduling quartzService.createJobWithCron(ScheduleConstant.DATA_SCHEDULE_GROUP, "hourly-job", "0 0 0/1 * * ?", HourlyJobForData.class); - quartzService.startScheduler(); } catch (SchedulerException e) { throw new ServiceException("Failed to initialize data scheduler", e); diff --git a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/buffer/PointValueIngestBuffer.java b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/buffer/PointValueIngestBuffer.java new file mode 100644 index 000000000..20197d973 --- /dev/null +++ b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/buffer/PointValueIngestBuffer.java @@ -0,0 +1,192 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.data.buffer; + +import io.github.pnoker.common.data.biz.PointValueService; +import io.github.pnoker.common.data.entity.property.PointBatchProperties; +import io.github.pnoker.common.entity.bo.PointValueBO; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Bounded in-memory buffer that decouples point-value consumption from repository persistence. + * + *

Replaces the legacy speed-threshold dual path (single-row save vs. unbounded list + Quartz + * tick). Every received point value enters a bounded {@link ArrayBlockingQueue}; worker threads + * drain it on a size-or-time trigger and call {@link PointValueService#save(List)}. When the + * queue is full {@link #offer} returns {@code false} so the receiver can nack-requeue and + * back-pressure RabbitMQ instead of OOM-ing. A failed save re-queues the batch for retry. + * + * @author pnoker + * @version 2026.7.8 + * @since 2026.7.8 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class PointValueIngestBuffer { + + private final PointBatchProperties pointBatchProperties; + private final PointValueService pointValueService; + + private ArrayBlockingQueue queue; + private ExecutorService worker; + private volatile boolean running; + private final AtomicLong droppedCount = new AtomicLong(0); + + /** + * Start the worker pool and begin draining. + */ + @PostConstruct + void start() { + queue = new ArrayBlockingQueue<>(pointBatchProperties.getQueueCapacity()); + int workers = pointBatchProperties.getWorkerCount(); + worker = Executors.newFixedThreadPool(workers, r -> { + Thread thread = new Thread(r, "dc3-point-value-ingest"); + thread.setDaemon(true); + return thread; + }); + running = true; + for (int i = 0; i < workers; i++) { + worker.submit(this::drainLoop); + } + log.info("PointValueIngestBuffer started, queueCapacity={}, batchSize={}, flushIntervalMillis={}, workerCount={}", + pointBatchProperties.getQueueCapacity(), pointBatchProperties.getBatchSize(), + pointBatchProperties.getFlushIntervalMillis(), workers); + } + + /** + * Stop workers and flush whatever remains in the queue. + */ + @PreDestroy + void stop() { + running = false; + if (Objects.nonNull(worker)) { + worker.shutdown(); + try { + worker.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + worker.shutdownNow(); + } + } + if (Objects.nonNull(queue)) { + List remaining = new ArrayList<>(); + queue.drainTo(remaining); + if (!remaining.isEmpty()) { + log.warn("PointValueIngestBuffer flushing {} remaining records on shutdown", remaining.size()); + try { + pointValueService.save(remaining); + } catch (Exception e) { + log.error("PointValueIngestBuffer failed to flush {} remaining records on shutdown", + remaining.size(), e); + } + } + } + } + + /** + * Enqueue a point value without blocking. + * + * @param pointValueBO the value to buffer + * @return {@code true} if accepted, {@code false} if the queue is full (caller should nack-requeue) + */ + public boolean offer(PointValueBO pointValueBO) { + return queue.offer(pointValueBO); + } + + /** + * @return number of point values currently buffered, awaiting the next flush + */ + public int pendingCount() { + return Objects.nonNull(queue) ? queue.size() : 0; + } + + /** + * @return cumulative count of records dropped because the queue was full on re-queue + */ + public long droppedCount() { + return droppedCount.get(); + } + + /** + * Worker loop: block on the first record (up to the flush interval), then non-blockingly + * drain up to {@code batchSize-1} more, then persist. Triggers on either a full batch or + * the flush-interval timeout. + */ + private void drainLoop() { + int batchSize = pointBatchProperties.getBatchSize(); + long flushMillis = pointBatchProperties.getFlushIntervalMillis(); + while (running) { + try { + PointValueBO first = queue.poll(flushMillis, TimeUnit.MILLISECONDS); + if (Objects.isNull(first)) { + continue; + } + List batch = new ArrayList<>(batchSize); + batch.add(first); + queue.drainTo(batch, batchSize - 1); + saveWithRetry(batch); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } catch (Exception e) { + log.error("PointValueIngestBuffer drain loop error", e); + } + } + } + + /** + * Persist a batch; on failure, re-queue it for retry so a transient DB outage does not + * lose data. + */ + private void saveWithRetry(List batch) { + try { + pointValueService.save(batch); + } catch (Exception e) { + log.error("Save point values batch failed, size={}, re-queuing for retry", batch.size(), e); + requeue(batch); + } + } + + /** + * Re-queue a failed batch entry by entry. If the queue is full (sustained DB outage with + * continued inflow), drop the record and count it so it surfaces in monitoring. + */ + private void requeue(List batch) { + for (PointValueBO pointValueBO : batch) { + if (!queue.offer(pointValueBO)) { + long dropped = droppedCount.incrementAndGet(); + log.error("PointValueIngestBuffer re-queue full, dropping record, deviceId={}, pointId={}, totalDropped={}", + pointValueBO.getDeviceId(), pointValueBO.getPointId(), dropped); + } + } + } +} diff --git a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/entity/property/PointBatchProperties.java b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/entity/property/PointBatchProperties.java index 267275103..0f2daa39c 100644 --- a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/entity/property/PointBatchProperties.java +++ b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/entity/property/PointBatchProperties.java @@ -24,11 +24,13 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.validation.annotation.Validated; /** - * Point-value batch processing thresholds. + * Point-value ingest buffer tuning. Replaces the legacy speed/interval threshold pair: every + * received point value enters a bounded queue and is flushed to the repository by worker + * threads on a size-or-time trigger. * * @author pnoker - * @version 2026.5.10 - * @since 2026.5.10 + * @version 2026.7.8 + * @since 2026.7.8 */ @Getter @Setter @@ -36,10 +38,15 @@ import org.springframework.validation.annotation.Validated; @ConfigurationProperties(prefix = "dc3.data.point.batch") public class PointBatchProperties { - @Min(value = 1, message = "Point batch speed must be greater than 0") - private int speed = 100; + @Min(value = 1, message = "Point batch queue capacity must be greater than 0") + private int queueCapacity = 100000; - @Min(value = 1, message = "Point batch interval must be greater than 0") - private int interval = 5; + @Min(value = 1, message = "Point batch size must be greater than 0") + private int batchSize = 1000; + @Min(value = 1, message = "Point batch flush interval must be greater than 0") + private long flushIntervalMillis = 500; + + @Min(value = 1, message = "Point batch worker count must be greater than 0") + private int workerCount = 4; } diff --git a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/job/PointValueJob.java b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/job/PointValueJob.java deleted file mode 100644 index 840868a3d..000000000 --- a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/job/PointValueJob.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2016-present the IoT DC3 original author or authors. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.github.pnoker.common.data.job; - -import io.github.pnoker.common.data.biz.PointValueService; -import io.github.pnoker.common.data.entity.property.PointBatchProperties; -import io.github.pnoker.common.entity.bo.PointValueBO; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; -import org.springframework.scheduling.quartz.QuartzJobBean; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/** - * Quartz job for point value batch processing. - * - * @author pnoker - * @version 2025.9.0 - * @since 2016.10.1 - */ -@Slf4j -@Component -@RequiredArgsConstructor -@DisallowConcurrentExecution -public class PointValueJob extends QuartzJobBean { - - private static final ReentrantReadWriteLock VALUE_LOCK = new ReentrantReadWriteLock(); - - private static final AtomicLong VALUE_COUNT = new AtomicLong(0); - - private static final AtomicLong VALUE_SPEED = new AtomicLong(0); - - private static final List POINT_VALUE_LIST = new ArrayList<>(); - - private final PointBatchProperties pointBatchProperties; - - private final PointValueService pointValueService; - - private final ExecutorService virtualThreadExecutor; - - /** - * Return the number of point values currently buffered, awaiting the next batch flush. - * - * @return the buffered point value count - */ - public static int getPointValuesSize() { - VALUE_LOCK.readLock().lock(); - try { - return POINT_VALUE_LIST.size(); - } finally { - VALUE_LOCK.readLock().unlock(); - } - } - - /** - * Clear all buffered point values without flushing them to the repository. - */ - public static void clearPointValues() { - VALUE_LOCK.writeLock().lock(); - try { - POINT_VALUE_LIST.clear(); - } finally { - VALUE_LOCK.writeLock().unlock(); - } - } - - /** - * Append a single point value to the in-memory batch buffer. - * - * @param pointValueBO the point value to buffer - */ - public static void addPointValues(PointValueBO pointValueBO) { - VALUE_LOCK.writeLock().lock(); - try { - POINT_VALUE_LIST.add(pointValueBO); - } finally { - VALUE_LOCK.writeLock().unlock(); - } - } - - /** - * Increment the received-point-value counter, sampled each batch tick to derive the - * receive speed. - */ - public static void recordPointValue() { - VALUE_COUNT.getAndIncrement(); - } - - /** - * Return the cumulative count of point values received since the last reset. - * - * @return the received point value count - */ - public static long getValueCount() { - return VALUE_COUNT.get(); - } - - /** - * Return the point value receive speed (values per second) computed at the last batch - * tick. - * - * @return the receive speed in values per second - */ - public static long getValueSpeed() { - return VALUE_SPEED.get(); - } - - /** - * Reset both the received-count and receive-speed counters to zero. - */ - public static void resetMetrics() { - VALUE_COUNT.set(0); - VALUE_SPEED.set(0); - } - - @Override - protected void executeInternal(JobExecutionContext jobExecutionContext) throws JobExecutionException { - // Statistical point value receive rate - long speed = VALUE_COUNT.getAndSet(0) / pointBatchProperties.getInterval(); - VALUE_SPEED.set(speed); - if (speed >= pointBatchProperties.getSpeed()) { - log.debug("Point value receiver speed: {} /s, value size: {}, interval: {}", speed, getPointValuesSize(), - pointBatchProperties.getInterval()); - } - - // Swap out the accumulated buffer under the lock; run the save on a private - // snapshot outside the lock so concurrent addPointValues callers are not blocked - // by DB I/O. - List snapshot; - VALUE_LOCK.writeLock().lock(); - try { - if (POINT_VALUE_LIST.isEmpty()) { - return; - } - snapshot = new ArrayList<>(POINT_VALUE_LIST); - POINT_VALUE_LIST.clear(); - } finally { - VALUE_LOCK.writeLock().unlock(); - } - - virtualThreadExecutor.execute(() -> pointValueService.save(snapshot)); - } - -} diff --git a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/rabbit/PointValueReceiver.java b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/rabbit/PointValueReceiver.java index 2d777896c..9142042e6 100644 --- a/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/rabbit/PointValueReceiver.java +++ b/dc3-common/dc3-common-data/src/main/java/io/github/pnoker/common/data/rabbit/PointValueReceiver.java @@ -18,11 +18,8 @@ package io.github.pnoker.common.data.rabbit; import com.rabbitmq.client.Channel; -import io.github.pnoker.common.data.biz.PointValueService; -import io.github.pnoker.common.data.entity.property.PointBatchProperties; -import io.github.pnoker.common.data.job.PointValueJob; +import io.github.pnoker.common.data.buffer.PointValueIngestBuffer; import io.github.pnoker.common.entity.bo.PointValueBO; -import io.github.pnoker.common.utils.JsonUtil; import io.github.pnoker.common.utils.RabbitAckUtil; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -36,8 +33,12 @@ import java.util.Objects; /** * RabbitMQ receiver for point value ingestion events. * + *

Every valid message is handed to {@link PointValueIngestBuffer}; ack when accepted, + * nack-requeue when the buffer is full so RabbitMQ back-pressures instead of the center + * OOM-ing. Uses the high-throughput container factory (wider prefetch / concurrency). + * * @author pnoker - * @version 2025.9.0 + * @version 2026.7.8 * @since 2016.10.1 */ @Slf4j @@ -45,21 +46,19 @@ import java.util.Objects; @RequiredArgsConstructor public class PointValueReceiver { - private final PointBatchProperties pointBatchProperties; - - private final PointValueService pointValueService; + private final PointValueIngestBuffer pointValueIngestBuffer; /** - * Consume a point value message: route it to the batch buffer when the receive speed - * exceeds the threshold, otherwise save it directly. Manual ack on success, requeue - * on failure. + * Consume a point value message: validate, offer to the ingest buffer, ack on success or + * nack-requeue when the buffer is full (back-pressure). Invalid messages are rejected. * * @param channel the RabbitMQ channel for manual ack * @param message the raw message carrying the delivery tag * @param pointValueBO the deserialized point value */ @RabbitHandler - @RabbitListener(queues = "#{pointValueQueue.name}") + @RabbitListener(queues = "#{pointValueQueue.name}", + containerFactory = "highThroughputRabbitListenerContainerFactory") public void pointValueReceive(Channel channel, Message message, PointValueBO pointValueBO) { long deliveryTag = message.getMessageProperties().getDeliveryTag(); try { @@ -69,27 +68,19 @@ public class PointValueReceiver { RabbitAckUtil.reject(channel, deliveryTag); return; } - PointValueJob.recordPointValue(); - log.debug("Receive point value from: {}, {}", message.getMessageProperties().getReceivedRoutingKey(), - JsonUtil.toJsonString(pointValueBO)); - - // Judge whether to process data in batch according to the data transmission - // speed - if (PointValueJob.getValueSpeed() < pointBatchProperties.getSpeed()) { - // Save point value to local latest-value cache and repository storage - pointValueService.save(pointValueBO); + if (pointValueIngestBuffer.offer(pointValueBO)) { + RabbitAckUtil.ack(channel, deliveryTag); } else { - // Save point value to schedule - PointValueJob.addPointValues(pointValueBO); + log.warn("Point value ingest buffer full, nack-requeue to back-pressure, deviceId={}, pointId={}", + pointValueBO.getDeviceId(), pointValueBO.getPointId()); + RabbitAckUtil.nack(channel, deliveryTag, true); } - RabbitAckUtil.ack(channel, deliveryTag); } catch (Exception e) { - log.error("Point value consume failed, deviceId={}, pointId={}, deliveryTag={}, routingKey={}", + log.error("Point value consume failed, deviceId={}, pointId={}, deliveryTag={}", Objects.nonNull(pointValueBO) ? pointValueBO.getDeviceId() : null, Objects.nonNull(pointValueBO) ? pointValueBO.getPointId() : null, - deliveryTag, message.getMessageProperties().getReceivedRoutingKey(), e); + deliveryTag, e); RabbitAckUtil.nack(channel, deliveryTag, true); } } - } diff --git a/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/biz/impl/ScheduleForDataServiceImplTest.java b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/biz/impl/ScheduleForDataServiceImplTest.java index 45c4b07c2..e1bcda487 100644 --- a/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/biz/impl/ScheduleForDataServiceImplTest.java +++ b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/biz/impl/ScheduleForDataServiceImplTest.java @@ -18,9 +18,7 @@ package io.github.pnoker.common.data.biz.impl; import io.github.pnoker.common.constant.driver.ScheduleConstant; -import io.github.pnoker.common.data.entity.property.PointBatchProperties; import io.github.pnoker.common.data.job.HourlyJobForData; -import io.github.pnoker.common.data.job.PointValueJob; import io.github.pnoker.common.exception.ServiceException; import io.github.pnoker.common.quartz.QuartzService; import org.junit.jupiter.api.BeforeEach; @@ -28,7 +26,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.quartz.DateBuilder; import org.quartz.SchedulerException; import static org.mockito.ArgumentMatchers.any; @@ -37,32 +34,31 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +/** + * Point-value ingestion no longer has a Quartz tick (it is driven by PointValueIngestBuffer), + * so only the hourly cron job registration is asserted here. + * + * @author pnoker + * @version 2026.7.8 + * @since 2026.7.8 + */ @ExtendWith(MockitoExtension.class) class ScheduleForDataServiceImplTest { @Mock private QuartzService quartzService; - private PointBatchProperties properties; private ScheduleForDataServiceImpl service; @BeforeEach void setUp() { - properties = new PointBatchProperties(); - properties.setInterval(5); - service = new ScheduleForDataServiceImpl(properties, quartzService); + service = new ScheduleForDataServiceImpl(quartzService); } @Test - void initialRegistersIntervalAndCronJobsAndStartsScheduler() throws Exception { + void initialRegistersHourlyJobAndStartsScheduler() throws Exception { service.initial(); - verify(quartzService).createJobWithInterval( - eq(ScheduleConstant.DATA_SCHEDULE_GROUP), - eq("data-point-value-schedule-job"), - eq(5), - eq(DateBuilder.IntervalUnit.SECOND), - eq(PointValueJob.class)); verify(quartzService).createJobWithCron( eq(ScheduleConstant.DATA_SCHEDULE_GROUP), eq("hourly-job"), @@ -74,7 +70,7 @@ class ScheduleForDataServiceImplTest { @Test void initialThrowsServiceExceptionOnSchedulerFailure() throws Exception { doThrow(new SchedulerException("scheduler down")).when(quartzService) - .createJobWithInterval(any(), any(), any(int.class), any(), any()); + .createJobWithCron(any(), any(), any(), any()); org.assertj.core.api.Assertions.assertThatThrownBy(() -> service.initial()) .isInstanceOf(ServiceException.class) .hasMessageContaining("Failed to initialize data scheduler") diff --git a/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/buffer/PointValueIngestBufferTest.java b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/buffer/PointValueIngestBufferTest.java new file mode 100644 index 000000000..7ddb8c268 --- /dev/null +++ b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/buffer/PointValueIngestBufferTest.java @@ -0,0 +1,130 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.data.buffer; + +import io.github.pnoker.common.data.biz.PointValueService; +import io.github.pnoker.common.data.entity.property.PointBatchProperties; +import io.github.pnoker.common.entity.bo.PointValueBO; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; + +/** + * Verifies the ingest buffer offer/flush/re-queue/back-pressure lifecycle with a mocked + * {@link PointValueService}. + * + * @author pnoker + * @version 2026.7.8 + * @since 2026.7.8 + */ +@ExtendWith(MockitoExtension.class) +class PointValueIngestBufferTest { + + @Mock + private PointValueService pointValueService; + + private PointBatchProperties properties; + private PointValueIngestBuffer buffer; + + @BeforeEach + void setUp() { + properties = new PointBatchProperties(); + properties.setQueueCapacity(1000); + properties.setBatchSize(2); + properties.setFlushIntervalMillis(200); + properties.setWorkerCount(1); + buffer = new PointValueIngestBuffer(properties, pointValueService); + buffer.start(); + } + + @AfterEach + void tearDown() { + buffer.stop(); + } + + @Test + void flushesBatchedValues() { + buffer.offer(bo(1)); + buffer.offer(bo(2)); + verify(pointValueService, timeout(1000)).save(anyList()); + } + + @Test + void flushesSingleRecordPromptly() { + // Below batchSize — the first record still triggers a flush via the poll-then-drain loop. + buffer.offer(bo(1)); + verify(pointValueService, timeout(1000)).save(anyList()); + } + + @Test + void requeuesBatchOnSaveFailure() { + doThrow(new RuntimeException("db down")).when(pointValueService).save(anyList()); + buffer.offer(bo(1)); + // Failed save re-queues the batch, which is then re-drained and re-saved. + verify(pointValueService, timeout(1000).atLeast(2)).save(anyList()); + } + + @Test + void offerReturnsFalseWhenQueueFull() throws Exception { + PointBatchProperties small = new PointBatchProperties(); + small.setQueueCapacity(2); + small.setBatchSize(100); + small.setFlushIntervalMillis(10_000); + small.setWorkerCount(1); + PointValueIngestBuffer full = new PointValueIngestBuffer(small, pointValueService); + full.start(); + try { + CountDownLatch firstTaken = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + doAnswer(inv -> { + firstTaken.countDown(); + release.await(); + return null; + }).when(pointValueService).save(anyList()); + + assertThat(full.offer(bo(1))).isTrue(); + // Wait until the worker has taken the first record and is blocked inside save(), + // so the queue is empty and we can deterministically fill it to capacity. + assertThat(firstTaken.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(full.offer(bo(2))).isTrue(); + assertThat(full.offer(bo(3))).isTrue(); + // Queue capacity is 2 — the 4th offer must be rejected (back-pressure signal). + assertThat(full.offer(bo(4))).isFalse(); + release.countDown(); + } finally { + full.stop(); + } + } + + private PointValueBO bo(int i) { + return PointValueBO.builder().deviceId((long) i).pointId((long) i).rawValue("v" + i).build(); + } +} diff --git a/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/job/HourlyJobForDataTest.java b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/job/HourlyJobForDataTest.java new file mode 100644 index 000000000..c54412737 --- /dev/null +++ b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/job/HourlyJobForDataTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.data.job; + +import org.junit.jupiter.api.Test; +import org.quartz.JobExecutionContext; + +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.Mockito.mock; + +/** + * Migrated from the deleted PointValueJobTest — the hourly maintenance job has nothing to do + * with the ingest buffer and gets its own focused test. + * + * @author pnoker + * @version 2026.7.8 + * @since 2026.7.8 + */ +class HourlyJobForDataTest { + + @Test + void executesWithoutThrowing() { + HourlyJobForData hourly = new HourlyJobForData(); + assertThatNoException().isThrownBy(() -> hourly.executeInternal(mock(JobExecutionContext.class))); + } +} diff --git a/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/job/PointValueJobTest.java b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/job/PointValueJobTest.java deleted file mode 100644 index 4a421d138..000000000 --- a/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/job/PointValueJobTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2016-present the IoT DC3 original author or authors. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.github.pnoker.common.data.job; - -import io.github.pnoker.common.data.biz.PointValueService; -import io.github.pnoker.common.data.entity.property.PointBatchProperties; -import io.github.pnoker.common.entity.bo.PointValueBO; -import org.awaitility.Awaitility; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.quartz.JobExecutionContext; - -import java.time.Duration; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatNoException; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.verify; - -@ExtendWith(MockitoExtension.class) -class PointValueJobTest { - - @Mock - private PointValueService pointValueService; - - @Mock - private JobExecutionContext jobExecutionContext; - - private PointValueJob job; - private ExecutorService executor; - private PointBatchProperties properties; - - @BeforeEach - void setUp() { - properties = new PointBatchProperties(); - properties.setSpeed(100); - properties.setInterval(5); - executor = Executors.newSingleThreadExecutor(); - - job = new PointValueJob(properties, pointValueService, executor); - - PointValueJob.resetMetrics(); - PointValueJob.clearPointValues(); - } - - @AfterEach - void tearDown() { - executor.shutdownNow(); - PointValueJob.resetMetrics(); - PointValueJob.clearPointValues(); - } - - @Test - void executeRotatesValueCountIntoSpeed() throws Exception { - for (int i = 0; i < 50; i++) { - PointValueJob.recordPointValue(); - } - job.executeInternal(jobExecutionContext); - assertThat(PointValueJob.getValueSpeed()).isEqualTo(10); - assertThat(PointValueJob.getValueCount()).isEqualTo(0); - } - - @Test - void executeWithEmptyBufferDoesNotCallSave() throws Exception { - job.executeInternal(jobExecutionContext); - // Yield once so any spurious async submission would land before the assertion. - verify(pointValueService, never().description("save must not run for an empty buffer")).save(any(List.class)); - } - - @Test - void executeFlushesBufferedValuesAsynchronously() throws Exception { - PointValueJob.addPointValues(PointValueBO.builder().deviceId(10L).pointId(20L).rawValue("v").build()); - PointValueJob.addPointValues(PointValueBO.builder().deviceId(10L).pointId(21L).rawValue("v").build()); - - job.executeInternal(jobExecutionContext); - - // Saved on the executor — wait for the async submission to land. - verify(pointValueService, timeout(Duration.ofSeconds(2).toMillis())).save(any(List.class)); - // Buffer is drained even before the async save completes. - Awaitility.await().atMost(Duration.ofSeconds(2)).pollInterval(Duration.ofMillis(50)) - .untilAsserted(() -> assertThat(PointValueJob.getPointValuesSize()).isZero()); - } - - @Test - void addPointValuesAppendsAndClearResetsBuffer() { - PointValueJob.addPointValues(PointValueBO.builder().deviceId(10L).pointId(20L).rawValue("v").build()); - assertThat(PointValueJob.getPointValuesSize()).isEqualTo(1); - PointValueJob.clearPointValues(); - assertThat(PointValueJob.getPointValuesSize()).isZero(); - } - - @Test - void hourlyJobForDataExecutesWithoutThrowing() { - HourlyJobForData hourly = new HourlyJobForData(); - assertThatNoException().isThrownBy(() -> hourly.executeInternal(jobExecutionContext)); - } -} diff --git a/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/rabbit/PointValueReceiverTest.java b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/rabbit/PointValueReceiverTest.java index 21f6909d2..cd177fa54 100644 --- a/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/rabbit/PointValueReceiverTest.java +++ b/dc3-common/dc3-common-data/src/test/java/io/github/pnoker/common/data/rabbit/PointValueReceiverTest.java @@ -18,11 +18,8 @@ package io.github.pnoker.common.data.rabbit; import com.rabbitmq.client.Channel; -import io.github.pnoker.common.data.biz.PointValueService; -import io.github.pnoker.common.data.entity.property.PointBatchProperties; -import io.github.pnoker.common.data.job.PointValueJob; +import io.github.pnoker.common.data.buffer.PointValueIngestBuffer; import io.github.pnoker.common.entity.bo.PointValueBO; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -31,91 +28,70 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.amqp.core.Message; import org.springframework.amqp.core.MessageProperties; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +/** + * Verifies the receiver routes messages to the ingest buffer and applies back-pressure + * (nack-requeue) when the buffer is full. + * + * @author pnoker + * @version 2026.7.8 + * @since 2026.7.8 + */ @ExtendWith(MockitoExtension.class) class PointValueReceiverTest { @Mock - private PointValueService pointValueService; + private PointValueIngestBuffer buffer; @Mock private Channel channel; private PointValueReceiver receiver; private Message message; - private PointBatchProperties properties; @BeforeEach void setUp() { - properties = new PointBatchProperties(); - properties.setSpeed(100); - properties.setInterval(5); - receiver = new PointValueReceiver(properties, pointValueService); - + receiver = new PointValueReceiver(buffer); MessageProperties props = new MessageProperties(); props.setDeliveryTag(7L); message = new Message(new byte[0], props); - // Reset shared state used by the rate-throttling branch - PointValueJob.resetMetrics(); - PointValueJob.clearPointValues(); - } - - @AfterEach - void resetSharedState() { - PointValueJob.resetMetrics(); - PointValueJob.clearPointValues(); } @Test void rejectsNullPayload() throws Exception { receiver.pointValueReceive(channel, message, null); - verifyNoInteractions(pointValueService); + verifyNoInteractions(buffer); verify(channel).basicReject(eq(7L), eq(false)); - verify(channel, never()).basicAck(eq(7L), eq(false)); } @Test void rejectsPayloadWithoutDeviceId() throws Exception { PointValueBO bo = PointValueBO.builder().pointId(20L).build(); receiver.pointValueReceive(channel, message, bo); - verifyNoInteractions(pointValueService); + verifyNoInteractions(buffer); verify(channel).basicReject(eq(7L), eq(false)); } @Test - void belowSpeedThresholdSavesImmediatelyAndAcks() throws Exception { + void offersAndAcks() throws Exception { PointValueBO bo = PointValueBO.builder().deviceId(10L).pointId(20L).rawValue("v").build(); - // VALUE_SPEED defaults to 0 in setUp, below the 100 threshold + when(buffer.offer(bo)).thenReturn(true); receiver.pointValueReceive(channel, message, bo); - verify(pointValueService).save(bo); + verify(buffer).offer(bo); verify(channel).basicAck(eq(7L), eq(false)); - // Counter is incremented per message — pinned so the job-side rate calculation - // stays accurate. - assertThat(PointValueJob.getValueCount()).isEqualTo(1); } @Test - void aboveSpeedThresholdBuffersToScheduleAndAcks() throws Exception { - properties.setSpeed(0); + void nacksAndRequeuesWhenBufferFull() throws Exception { PointValueBO bo = PointValueBO.builder().deviceId(10L).pointId(20L).rawValue("v").build(); + when(buffer.offer(bo)).thenReturn(false); receiver.pointValueReceive(channel, message, bo); - verify(pointValueService, never()).save(any(PointValueBO.class)); - verify(channel).basicAck(eq(7L), eq(false)); - assertThat(PointValueJob.getPointValuesSize()).isEqualTo(1); - } - - @Test - void nacksAndRequeuesOnServiceFailure() throws Exception { - PointValueBO bo = PointValueBO.builder().deviceId(10L).pointId(20L).rawValue("v").build(); - org.mockito.Mockito.doThrow(new RuntimeException("downstream offline")) - .when(pointValueService).save(any(PointValueBO.class)); - receiver.pointValueReceive(channel, message, bo); + verify(buffer).offer(bo); verify(channel).basicNack(eq(7L), eq(false), eq(true)); verify(channel, never()).basicAck(eq(7L), eq(false)); } diff --git a/dc3-common/dc3-common-driver/pom.xml b/dc3-common/dc3-common-driver/pom.xml index aefc8d428..c1c1cad1f 100644 --- a/dc3-common/dc3-common-driver/pom.xml +++ b/dc3-common/dc3-common-driver/pom.xml @@ -77,6 +77,16 @@ caffeine + + + org.xerial + sqlite-jdbc + + + com.zaxxer + HikariCP + + io.github.pnoker diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferService.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferService.java new file mode 100644 index 000000000..1072cd58b --- /dev/null +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferService.java @@ -0,0 +1,66 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.driver.buffer; + +import io.github.pnoker.common.driver.entity.bean.PointValue; + +/** + * Local SQLite-backed buffer for point values that could not be delivered to RabbitMQ. + * + *

Failed/NACKed readings are persisted and republished by a Quartz job once the + * broker recovers, so a RabbitMQ outage no longer loses collected data. + * + * @author pnoker + * @version 2026.5.22 + * @since 2026.6.2 + */ +public interface BufferService { + + /** + * Initialize the SQLite database: create parent directories, open the connection + * pool, and create the buffer table. Idempotent; a no-op when the buffer is disabled. + */ + void initialize(); + + /** + * Persist a point value that failed to publish, keyed by the publisher-confirm + * correlation id so a later NACK republish overwrites the same row (INSERT OR REPLACE). + * + * @param pointValue the failed point value + * @param routingKey RabbitMQ routing key to republish with + * @param correlationId publisher-confirm correlation id, used as the buffer row primary key + * @param attempt ordinal of the send attempt that just failed + */ + void offer(PointValue pointValue, String routingKey, String correlationId, int attempt); + + /** + * Republish up to {@code batchSize} due buffered point values, deleting the ones that + * leave the channel cleanly and back-offing the ones that throw. + */ + void republishBatch(); + + /** + * @return whether the buffer is enabled in configuration + */ + boolean isEnabled(); + + /** + * @return current number of records awaiting republish (0 when disabled) + */ + long pendingCount(); +} diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferServiceImpl.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferServiceImpl.java new file mode 100644 index 000000000..0a5f14b42 --- /dev/null +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferServiceImpl.java @@ -0,0 +1,195 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.driver.buffer; + +import io.github.pnoker.common.constant.driver.RabbitConstant; +import io.github.pnoker.common.driver.entity.bean.PointValue; +import io.github.pnoker.common.driver.entity.property.DriverProperties; +import io.github.pnoker.common.utils.JsonUtil; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.AmqpException; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Objects; + +/** + * SQLite-backed {@link BufferService}. Persists point values that failed to reach RabbitMQ + * (synchronous {@link AmqpException} or asynchronous publisher NACK) and republishes them + * from a Quartz job with exponential backoff. When the buffer file exceeds the configured + * size cap the oldest records are evicted to keep the newest readings. + * + *

Republish is optimistic: a record that leaves the channel without throwing is deleted, + * and a later NACK re-queues it through the confirm callback using the same correlation id. + * + * @author pnoker + * @version 2026.5.22 + * @since 2026.6.2 + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class BufferServiceImpl implements BufferService { + + private final DriverProperties driverProperties; + private final RabbitTemplate rabbitTemplate; + + private PointValueBuffer buffer; + + @Override + public void initialize() { + DriverProperties.BufferProperties config = driverProperties.getBuffer(); + if (Objects.isNull(config) || !Boolean.TRUE.equals(config.getEnabled())) { + log.info("Point value buffer disabled, skip initialization"); + return; + } + this.buffer = new PointValueBuffer(config.getDbPath()); + this.buffer.initialize(); + } + + @Override + public boolean isEnabled() { + DriverProperties.BufferProperties config = driverProperties.getBuffer(); + return Objects.nonNull(config) && Boolean.TRUE.equals(config.getEnabled()) && Objects.nonNull(buffer); + } + + @Override + public long pendingCount() { + return Objects.nonNull(buffer) ? buffer.count() : 0; + } + + @Override + public void offer(PointValue pointValue, String routingKey, String correlationId, int attempt) { + if (!isEnabled()) { + return; + } + DriverProperties.BufferProperties config = driverProperties.getBuffer(); + long now = epochSecond(); + BufferedPointValue record = new BufferedPointValue( + correlationId, + pointValue.getDeviceId(), + pointValue.getPointId(), + pointValue.getDriverId(), + pointValue.getTenantId(), + JsonUtil.toJsonString(pointValue), + routingKey, + attempt, + now + backoffSeconds(attempt, config), + now + ); + buffer.upsert(record); + if (log.isDebugEnabled()) { + log.debug("Buffered point value, id={}, deviceId={}, pointId={}, attempt={}, queueSize={}", + correlationId, pointValue.getDeviceId(), pointValue.getPointId(), attempt, buffer.count()); + } + enforceCapacity(config); + } + + @Override + public void republishBatch() { + if (!isEnabled()) { + return; + } + DriverProperties.BufferProperties config = driverProperties.getBuffer(); + List records = buffer.selectPending(config.getBatchSize(), epochSecond()); + if (records.isEmpty()) { + return; + } + log.debug("Republishing {} buffered point values", records.size()); + for (BufferedPointValue record : records) { + republishOne(record, config); + } + enforceCapacity(config); + } + + /** + * Republish a single buffered record. Records that have exhausted {@code maxRetry} are + * dropped as poison with an ERROR log; the rest are re-sent with an incremented attempt + * counter carried in the correlation so a NACK re-queue stores the right ordinal. + */ + private void republishOne(BufferedPointValue record, DriverProperties.BufferProperties config) { + if (record.attempt() >= config.getMaxRetry()) { + log.error("Buffer record exceeded max retry ({}), dropping poison, id={}, deviceId={}, pointId={}", + config.getMaxRetry(), record.id(), record.deviceId(), record.pointId()); + buffer.delete(record.id()); + return; + } + PointValue pointValue; + try { + pointValue = JsonUtil.parseObject(record.payloadJson(), PointValue.class); + } catch (Exception e) { + log.error("Buffer record payload corrupted, dropping, id={}, deviceId={}, pointId={}", + record.id(), record.deviceId(), record.pointId(), e); + buffer.delete(record.id()); + return; + } + int nextAttempt = record.attempt() + 1; + PointValueCorrelation correlation = new PointValueCorrelation( + record.id(), record.deviceId(), record.pointId(), nextAttempt, + record.payloadJson(), record.routingKey()); + try { + rabbitTemplate.convertAndSend(RabbitConstant.TOPIC_EXCHANGE_VALUE, record.routingKey(), pointValue, correlation); + // Optimistic delete: the message left the channel. A later NACK re-queues it + // via the ConfirmCallback using the same correlation id. + buffer.delete(record.id()); + } catch (AmqpException e) { + long backoff = backoffSeconds(nextAttempt, config); + log.warn("Buffer republish rejected, id={}, attempt={}, retrying in {}s", + record.id(), nextAttempt, backoff); + buffer.markRetry(record.id(), nextAttempt, epochSecond() + backoff); + } + } + + /** + * When the SQLite file exceeds the configured size cap, evict the oldest batch. SQLite + * reuses freed pages, so the file does not shrink without a VACUUM — that is acceptable + * for a bounded buffer that cycles through records. + */ + private void enforceCapacity(DriverProperties.BufferProperties config) { + long maxBytes = config.getMaxSizeMb() * 1024L * 1024L; + if (buffer.fileSize() <= maxBytes) { + return; + } + int evicted = buffer.deleteOldest(config.getBatchSize()); + log.warn("Buffer capacity exceeded ({}B > {}B), evicted {} oldest records", + buffer.fileSize(), maxBytes, evicted); + } + + /** + * Exponential backoff in seconds: {@code backoffSeconds * 2^(attempt-1)}, capped at + * {@code maxBackoffSeconds}. + */ + private long backoffSeconds(int attempt, DriverProperties.BufferProperties config) { + long delay = (long) (config.getBackoffSeconds() * Math.pow(2, attempt - 1)); + return Math.min(delay, config.getMaxBackoffSeconds()); + } + + private static long epochSecond() { + return System.currentTimeMillis() / 1000; + } + + @PreDestroy + void destroy() { + if (Objects.nonNull(buffer)) { + buffer.close(); + } + } +} diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferedPointValue.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferedPointValue.java new file mode 100644 index 000000000..0667e8e9a --- /dev/null +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/BufferedPointValue.java @@ -0,0 +1,42 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.driver.buffer; + +/** + * DAO-level row mapping for a buffered point value awaiting republish. + * + *

Times are epoch seconds so the SQLite store stays free of date-format/timezone + * pitfalls. + * + * @author pnoker + * @version 2026.5.22 + * @since 2026.6.2 + */ +public record BufferedPointValue( + String id, + Long deviceId, + Long pointId, + Long driverId, + Long tenantId, + String payloadJson, + String routingKey, + int attempt, + long nextAttemptAt, + long createdAt +) { +} diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/PointValueBuffer.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/PointValueBuffer.java new file mode 100644 index 000000000..8793ea991 --- /dev/null +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/PointValueBuffer.java @@ -0,0 +1,270 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.driver.buffer; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import io.github.pnoker.common.exception.ServiceException; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * SQLite-backed DAO for the local point-value buffer. Owns a single-connection HikariCP + * pool (SQLite is a single-writer database) in WAL mode and exposes the small CRUD + * surface the buffer service needs. + * + *

All times are epoch seconds; the schema deliberately avoids date types so the store + * is free of timezone/format pitfalls. + * + * @author pnoker + * @version 2026.5.22 + * @since 2026.6.2 + */ +@Slf4j +public class PointValueBuffer { + + private static final String CREATE_TABLE_SQL = """ + CREATE TABLE IF NOT EXISTS point_value_buffer ( + id TEXT PRIMARY KEY, + device_id INTEGER NOT NULL, + point_id INTEGER NOT NULL, + driver_id INTEGER, + tenant_id INTEGER, + payload_json TEXT NOT NULL, + routing_key TEXT NOT NULL, + attempt INTEGER NOT NULL DEFAULT 0, + next_attempt_at INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) + """; + private static final String CREATE_INDEX_NEXT_SQL = + "CREATE INDEX IF NOT EXISTS idx_buffer_next_attempt ON point_value_buffer(next_attempt_at)"; + private static final String CREATE_INDEX_CREATED_SQL = + "CREATE INDEX IF NOT EXISTS idx_buffer_created ON point_value_buffer(created_at)"; + + private static final String UPSERT_SQL = """ + INSERT OR REPLACE INTO point_value_buffer + (id, device_id, point_id, driver_id, tenant_id, payload_json, routing_key, attempt, next_attempt_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """; + private static final String SELECT_PENDING_SQL = """ + SELECT id, device_id, point_id, driver_id, tenant_id, payload_json, routing_key, attempt, next_attempt_at, created_at + FROM point_value_buffer + WHERE next_attempt_at <= ? + ORDER BY next_attempt_at ASC + LIMIT ? + """; + private static final String DELETE_SQL = "DELETE FROM point_value_buffer WHERE id = ?"; + private static final String MARK_RETRY_SQL = + "UPDATE point_value_buffer SET attempt = ?, next_attempt_at = ? WHERE id = ?"; + private static final String DELETE_OLDEST_SQL = """ + DELETE FROM point_value_buffer WHERE id IN ( + SELECT id FROM point_value_buffer ORDER BY created_at ASC LIMIT ? + ) + """; + private static final String COUNT_SQL = "SELECT COUNT(*) FROM point_value_buffer"; + + private final String dbPath; + private HikariDataSource dataSource; + + public PointValueBuffer(String dbPath) { + this.dbPath = dbPath; + } + + /** + * Open the connection pool, apply WAL, and create the buffer table. Idempotent. + */ + public void initialize() { + File parent = new File(dbPath).getParentFile(); + if (Objects.nonNull(parent) && !parent.exists() && !parent.mkdirs()) { + throw new ServiceException("Failed to create buffer db parent directory: " + parent.getAbsolutePath()); + } + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:sqlite:" + dbPath); + config.setDriverClassName("org.sqlite.JDBC"); + config.setMaximumPoolSize(1); + config.setMinimumIdle(1); + config.setConnectionInitSql("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;"); + config.setPoolName("dc3-driver-buffer"); + this.dataSource = new HikariDataSource(config); + createTableIfNotExists(); + log.info("Point value buffer initialized, dbPath={}", dbPath); + } + + private void createTableIfNotExists() { + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + stmt.execute(CREATE_TABLE_SQL); + stmt.execute(CREATE_INDEX_NEXT_SQL); + stmt.execute(CREATE_INDEX_CREATED_SQL); + } catch (SQLException e) { + throw new ServiceException("Failed to init point-value buffer table", e); + } + } + + /** + * Insert or replace a buffered record keyed by the correlation id. + */ + public void upsert(BufferedPointValue record) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(UPSERT_SQL)) { + ps.setString(1, record.id()); + setLong(ps, 2, record.deviceId()); + setLong(ps, 3, record.pointId()); + setLong(ps, 4, record.driverId()); + setLong(ps, 5, record.tenantId()); + ps.setString(6, record.payloadJson()); + ps.setString(7, record.routingKey()); + ps.setInt(8, record.attempt()); + ps.setLong(9, record.nextAttemptAt()); + ps.setLong(10, record.createdAt()); + ps.executeUpdate(); + } catch (SQLException e) { + log.error("Buffer upsert failed, id={}, attempt={}", record.id(), record.attempt(), e); + } + } + + /** + * Return up to {@code batchSize} records due for republish (next_attempt_at <= now), + * oldest-first. + */ + public List selectPending(int batchSize, long nowEpochSec) { + List records = new ArrayList<>(batchSize); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(SELECT_PENDING_SQL)) { + ps.setLong(1, nowEpochSec); + ps.setInt(2, batchSize); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + records.add(new BufferedPointValue( + rs.getString("id"), + rs.getLong("device_id"), + rs.getLong("point_id"), + getNullableLong(rs, "driver_id"), + getNullableLong(rs, "tenant_id"), + rs.getString("payload_json"), + rs.getString("routing_key"), + rs.getInt("attempt"), + rs.getLong("next_attempt_at"), + rs.getLong("created_at") + )); + } + } + } catch (SQLException e) { + log.error("Buffer selectPending failed", e); + } + return records; + } + + /** + * Delete a record after it has left the channel cleanly. + */ + public void delete(String id) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(DELETE_SQL)) { + ps.setString(1, id); + ps.executeUpdate(); + } catch (SQLException e) { + log.error("Buffer delete failed, id={}", id, e); + } + } + + /** + * Bump the attempt counter and push back the next retry time after a failed republish. + */ + public void markRetry(String id, int attempt, long nextAttemptAt) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(MARK_RETRY_SQL)) { + ps.setInt(1, attempt); + ps.setLong(2, nextAttemptAt); + ps.setString(3, id); + ps.executeUpdate(); + } catch (SQLException e) { + log.error("Buffer markRetry failed, id={}, attempt={}", id, attempt, e); + } + } + + /** + * Delete the {@code evictBatch} oldest records (by created_at) for capacity enforcement. + * + * @return number of records deleted + */ + public int deleteOldest(int evictBatch) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(DELETE_OLDEST_SQL)) { + ps.setInt(1, evictBatch); + return ps.executeUpdate(); + } catch (SQLException e) { + log.error("Buffer deleteOldest failed", e); + return 0; + } + } + + /** + * @return current number of buffered records + */ + public long count() { + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(COUNT_SQL)) { + return rs.next() ? rs.getLong(1) : 0; + } catch (SQLException e) { + log.error("Buffer count failed", e); + return 0; + } + } + + /** + * @return on-disk size of the SQLite database file in bytes + */ + public long fileSize() { + File file = new File(dbPath); + return file.exists() ? file.length() : 0; + } + + /** + * Close the connection pool. + */ + public void close() { + if (Objects.nonNull(dataSource) && !dataSource.isClosed()) { + dataSource.close(); + } + } + + private static void setLong(PreparedStatement ps, int index, Long value) throws SQLException { + if (Objects.isNull(value)) { + ps.setNull(index, Types.INTEGER); + } else { + ps.setLong(index, value); + } + } + + private static Long getNullableLong(ResultSet rs, String column) throws SQLException { + long value = rs.getLong(column); + return rs.wasNull() ? null : value; + } +} diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/PointValueCorrelation.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/PointValueCorrelation.java new file mode 100644 index 000000000..80c1c23c0 --- /dev/null +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/buffer/PointValueCorrelation.java @@ -0,0 +1,52 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.driver.buffer; + +import lombok.Getter; +import org.springframework.amqp.rabbit.connection.CorrelationData; + +/** + * Carries the buffer record identity and serialized payload through the publisher-confirm + * callback so a NACK can re-queue the point value without re-reading the message body. + * + *

{@code attempt} is the ordinal of the send attempt that just failed, so a NACK + * republish stores the same counter the synchronous catch path would. + * + * @author pnoker + * @version 2026.5.22 + * @since 2026.6.2 + */ +@Getter +public class PointValueCorrelation extends CorrelationData { + + private final Long deviceId; + private final Long pointId; + private final int attempt; + private final String payloadJson; + private final String routingKey; + + public PointValueCorrelation(String id, Long deviceId, Long pointId, int attempt, + String payloadJson, String routingKey) { + super(id); + this.deviceId = deviceId; + this.pointId = pointId; + this.attempt = attempt; + this.payloadJson = payloadJson; + this.routingKey = routingKey; + } +} diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/entity/property/DriverProperties.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/entity/property/DriverProperties.java index 5280b9a90..a12b0f5d8 100644 --- a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/entity/property/DriverProperties.java +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/entity/property/DriverProperties.java @@ -94,6 +94,12 @@ public class DriverProperties { @Valid private HealthProperties health = new HealthProperties(); + /** + * Local buffer configuration for point-value resume on broker outage. + */ + @Valid + private BufferProperties buffer = new BufferProperties(); + /** * Metadata cache tuning for the driver runtime. */ @@ -283,4 +289,65 @@ public class DriverProperties { } + /** + * Local point-value buffer options. Failed/NACKed point values are persisted to an + * embedded SQLite database and republished by a Quartz job once the broker recovers, + * so a RabbitMQ outage no longer loses collected readings. + */ + @Getter + @Setter + public static class BufferProperties { + + /** + * Whether to persist failed/NACKed point values locally for later republish. + * On by default so drivers resume broker outages out of the box. + */ + private Boolean enabled = true; + + /** + * SQLite database path, relative to the driver working directory. Mirrors the + * {@code dc3/logs} layout so each driver writes its own buffer file. + */ + @NotBlank(message = "Buffer db path can't be empty") + private String dbPath = "dc3/data/driver/buffer.db"; + + /** + * Upper bound on the buffer database size in megabytes. When exceeded the oldest + * records are evicted to keep the newest readings (capacity over completeness). + */ + @Min(1) + private long maxSizeMb = 256; + + /** + * Number of buffered point values republished per Quartz tick. + */ + @Min(1) + private int batchSize = 200; + + /** + * Quartz cron expression for the buffer republish job. + */ + @NotBlank(message = "Buffer republish cron can't be empty") + private String republishCron = "0/10 * * * * ?"; + + /** + * Maximum republish attempts before a buffered record is dropped as poison. + */ + @Min(1) + private int maxRetry = 50; + + /** + * Initial backoff before the first republish retry, in seconds. + */ + @Min(1) + private long backoffSeconds = 10; + + /** + * Upper bound the doubling backoff is capped at, in seconds. + */ + @Min(1) + private long maxBackoffSeconds = 600; + + } + } diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/init/DriverInitRunner.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/init/DriverInitRunner.java index 9f846cdc8..913c5d622 100644 --- a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/init/DriverInitRunner.java +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/init/DriverInitRunner.java @@ -17,6 +17,7 @@ package io.github.pnoker.common.driver.init; +import io.github.pnoker.common.driver.buffer.BufferService; import io.github.pnoker.common.driver.entity.property.DriverProperties; import io.github.pnoker.common.driver.service.DriverCustomService; import io.github.pnoker.common.driver.service.DriverRegisterService; @@ -64,6 +65,9 @@ public class DriverInitRunner implements ApplicationRunner { private final DriverScheduleService driverScheduleService; + /** Local point-value buffer, initialized before registration so readings survive a manager outage. */ + private final BufferService bufferService; + /** * Runs the driver bootstrap sequence on startup: register with the manager center * (with retry), execute custom initialization, then initialize scheduled tasks. @@ -73,6 +77,10 @@ public class DriverInitRunner implements ApplicationRunner { */ @Override public void run(ApplicationArguments args) throws Exception { + // Initialize the local point-value buffer before registration so collected + // readings can be persisted even if the manager center is unreachable. + bufferService.initialize(); + // Initialize driver registration and synchronize basic information with the // platform; tolerate manager center being temporarily unavailable. registerWithRetry(); diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/job/BufferRepublishScheduleJob.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/job/BufferRepublishScheduleJob.java new file mode 100644 index 000000000..66b4d117f --- /dev/null +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/job/BufferRepublishScheduleJob.java @@ -0,0 +1,52 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.driver.job; + +import io.github.pnoker.common.driver.buffer.BufferService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.JobExecutionContext; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.springframework.stereotype.Component; + +/** + * Quartz job that drains the local point-value buffer back to RabbitMQ. Failures are + * logged (not rethrown) so one bad batch does not abort the scheduler. + * + * @author pnoker + * @version 2026.5.22 + * @since 2026.6.2 + */ +@Slf4j +@Component +@RequiredArgsConstructor +@DisallowConcurrentExecution +public class BufferRepublishScheduleJob extends QuartzJobBean { + + private final BufferService bufferService; + + @Override + protected void executeInternal(JobExecutionContext jobExecutionContext) { + try { + bufferService.republishBatch(); + } catch (Exception e) { + log.error("Buffer republish job failed", e); + } + } +} diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/service/impl/DriverScheduleServiceImpl.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/service/impl/DriverScheduleServiceImpl.java index 082647adb..a3b404395 100644 --- a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/service/impl/DriverScheduleServiceImpl.java +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/service/impl/DriverScheduleServiceImpl.java @@ -19,6 +19,7 @@ package io.github.pnoker.common.driver.service.impl; import io.github.pnoker.common.constant.driver.ScheduleConstant; import io.github.pnoker.common.driver.entity.property.DriverProperties; +import io.github.pnoker.common.driver.job.BufferRepublishScheduleJob; import io.github.pnoker.common.driver.job.DeviceHealthScheduleJob; import io.github.pnoker.common.driver.job.DriverCustomScheduleJob; import io.github.pnoker.common.driver.job.DriverHealthScheduleJob; @@ -103,6 +104,17 @@ public class DriverScheduleServiceImpl implements DriverScheduleService { DriverCustomScheduleJob.class); } + // Create and schedule the buffer republish job if enabled + DriverProperties.BufferProperties buffer = driverProperties.getBuffer(); + if (Objects.nonNull(buffer) && Boolean.TRUE.equals(buffer.getEnabled())) { + if (!CronExpression.isValidExpression(buffer.getRepublishCron())) { + throw new CronException("Buffer republish schedule cron expression is invalid"); + } + quartzService.createJobWithCron(ScheduleConstant.DRIVER_SCHEDULE_GROUP, + ScheduleConstant.BUFFER_REPUBLISH_SCHEDULE_JOB, buffer.getRepublishCron(), + BufferRepublishScheduleJob.class); + } + // Start the scheduler after all jobs are configured quartzService.startScheduler(); } catch (SchedulerException e) { diff --git a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/service/impl/DriverSenderServiceImpl.java b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/service/impl/DriverSenderServiceImpl.java index a4c884824..7127123d4 100644 --- a/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/service/impl/DriverSenderServiceImpl.java +++ b/dc3-common/dc3-common-driver/src/main/java/io/github/pnoker/common/driver/service/impl/DriverSenderServiceImpl.java @@ -18,6 +18,8 @@ package io.github.pnoker.common.driver.service.impl; import io.github.pnoker.common.constant.driver.RabbitConstant; +import io.github.pnoker.common.driver.buffer.BufferService; +import io.github.pnoker.common.driver.buffer.PointValueCorrelation; import io.github.pnoker.common.driver.entity.bean.PointValue; import io.github.pnoker.common.driver.entity.bo.DriverBO; import io.github.pnoker.common.driver.entity.property.DriverProperties; @@ -72,12 +74,30 @@ public class DriverSenderServiceImpl implements DriverSenderService { */ private final RabbitTemplate rabbitTemplate; + /** + * Local buffer for point values that fail to reach RabbitMQ, republished once the broker recovers. + */ + private final BufferService bufferService; + @PostConstruct void init() { rabbitTemplate.setConfirmCallback((correlation, ack, reason) -> { - if (!ack && correlation instanceof PointValueCorrelation ctx) { - log.warn("Point value publish NACKed: deviceId={}, pointId={}, reason={}", - ctx.deviceId, ctx.pointId, reason); + if (ack) { + return; + } + if (correlation instanceof PointValueCorrelation ctx) { + log.warn("Point value publish NACKed, buffering for retry: deviceId={}, pointId={}, attempt={}, reason={}", + ctx.getDeviceId(), ctx.getPointId(), ctx.getAttempt(), reason); + try { + PointValue pointValue = JsonUtil.parseObject(ctx.getPayloadJson(), PointValue.class); + bufferService.offer(pointValue, ctx.getRoutingKey(), ctx.getId(), ctx.getAttempt()); + } catch (Exception e) { + log.error("Failed to re-queue NACKed point value, payload corrupted, correlationId={}", + ctx.getId(), e); + } + } else { + log.error("RabbitMQ publisher confirm NACK, correlationId={}, cause={}", + Objects.nonNull(correlation) ? correlation.getId() : null, reason); } }); } @@ -210,13 +230,22 @@ public class DriverSenderServiceImpl implements DriverSenderService { } String routingKey = RabbitConstant.ROUTING_POINT_VALUE_PREFIX + driverProperties.getService(); - CorrelationData correlationData = new PointValueCorrelation( - UUID.randomUUID().toString(), entityDTO.getDeviceId(), entityDTO.getPointId()); + boolean buffering = bufferService.isEnabled(); + CorrelationData correlationData = buffering + ? new PointValueCorrelation(UUID.randomUUID().toString(), entityDTO.getDeviceId(), + entityDTO.getPointId(), 1, JsonUtil.toJsonString(entityDTO), routingKey) + : new CorrelationData(UUID.randomUUID().toString()); try { rabbitTemplate.convertAndSend(RabbitConstant.TOPIC_EXCHANGE_VALUE, routingKey, entityDTO, correlationData); } catch (AmqpException e) { - log.error("Point value publish rejected: deviceId={}, pointId={}", - entityDTO.getDeviceId(), entityDTO.getPointId(), e); + if (buffering) { + log.warn("Point value publish rejected, buffering for retry: deviceId={}, pointId={}", + entityDTO.getDeviceId(), entityDTO.getPointId(), e); + bufferService.offer(entityDTO, routingKey, correlationData.getId(), 1); + } else { + log.error("Point value publish rejected: deviceId={}, pointId={}", + entityDTO.getDeviceId(), entityDTO.getPointId(), e); + } } } @@ -302,25 +331,4 @@ public class DriverSenderServiceImpl implements DriverSenderService { deviceStateSender(deviceState); } - /** - * Carries device/point context through the publisher-confirm callback. - */ - private static class PointValueCorrelation extends CorrelationData { - - /** - * Device owning the point value, included so the NACK callback can log actionable context. - */ - final Long deviceId; - /** - * Point whose value failed to publish, included so the NACK callback can log actionable context. - */ - final Long pointId; - - PointValueCorrelation(String id, Long deviceId, Long pointId) { - super(id); - this.deviceId = deviceId; - this.pointId = pointId; - } - } - } diff --git a/dc3-common/dc3-common-driver/src/test/java/io/github/pnoker/common/driver/buffer/BufferServiceImplTest.java b/dc3-common/dc3-common-driver/src/test/java/io/github/pnoker/common/driver/buffer/BufferServiceImplTest.java new file mode 100644 index 000000000..ee7dac582 --- /dev/null +++ b/dc3-common/dc3-common-driver/src/test/java/io/github/pnoker/common/driver/buffer/BufferServiceImplTest.java @@ -0,0 +1,121 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.driver.buffer; + +import io.github.pnoker.common.driver.entity.bean.PointValue; +import io.github.pnoker.common.driver.entity.property.DriverProperties; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.amqp.AmqpException; +import org.springframework.amqp.rabbit.connection.CorrelationData; +import org.springframework.amqp.rabbit.core.RabbitTemplate; + +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; + +/** + * Verifies the buffer service offer/republish lifecycle with a mocked RabbitTemplate. + * + * @author pnoker + * @version 2026.5.22 + * @since 2026.6.2 + */ +@ExtendWith(MockitoExtension.class) +class BufferServiceImplTest { + + @TempDir + Path tmp; + + @Mock + private RabbitTemplate rabbitTemplate; + + private DriverProperties properties; + private BufferServiceImpl service; + + @BeforeEach + void setUp() { + properties = new DriverProperties(); + properties.getBuffer().setEnabled(true); + properties.getBuffer().setDbPath(tmp.resolve("buffer.db").toString()); + properties.getBuffer().setBatchSize(10); + properties.getBuffer().setMaxRetry(3); + properties.getBuffer().setBackoffSeconds(0); + service = new BufferServiceImpl(properties, rabbitTemplate); + service.initialize(); + } + + @Test + void offerPersistsAndRepublishSendsThenDrains() { + service.offer(pointValue(), "rk", "id-1", 1); + assertThat(service.pendingCount()).as("offer should persist one record").isEqualTo(1); + + service.republishBatch(); + verify(rabbitTemplate).convertAndSend(anyString(), anyString(), any(PointValue.class), any(CorrelationData.class)); + assertThat(service.pendingCount()).as("republish should drain the buffer").isEqualTo(0); + + reset(rabbitTemplate); + service.republishBatch(); + verify(rabbitTemplate, never()).convertAndSend(anyString(), anyString(), any(PointValue.class), any(CorrelationData.class)); + } + + @Test + void republishRequeuesOnAmqpExceptionThenResends() { + service.offer(pointValue(), "rk", "id-1", 1); + + doThrow(new AmqpException("broker down")).when(rabbitTemplate) + .convertAndSend(anyString(), anyString(), any(PointValue.class), any(CorrelationData.class)); + service.republishBatch(); + + reset(rabbitTemplate); + service.republishBatch(); + verify(rabbitTemplate).convertAndSend(anyString(), anyString(), any(PointValue.class), any(CorrelationData.class)); + + reset(rabbitTemplate); + service.republishBatch(); + verify(rabbitTemplate, never()).convertAndSend(anyString(), anyString(), any(PointValue.class), any(CorrelationData.class)); + } + + @Test + void disabledBufferIsNoOp() { + DriverProperties disabledProps = new DriverProperties(); + disabledProps.getBuffer().setEnabled(false); + disabledProps.getBuffer().setDbPath(tmp.resolve("disabled.db").toString()); + BufferServiceImpl disabled = new BufferServiceImpl(disabledProps, rabbitTemplate); + disabled.initialize(); + + assertThat(disabled.isEnabled()).isFalse(); + disabled.offer(pointValue(), "rk", "id-1", 1); + disabled.republishBatch(); + verify(rabbitTemplate, never()).convertAndSend(anyString(), anyString(), any(PointValue.class), any(CorrelationData.class)); + } + + private PointValue pointValue() { + return PointValue.builder().deviceId(1L).pointId(2L).rawValue("42").build(); + } +} diff --git a/dc3-common/dc3-common-driver/src/test/java/io/github/pnoker/common/driver/buffer/PointValueBufferTest.java b/dc3-common/dc3-common-driver/src/test/java/io/github/pnoker/common/driver/buffer/PointValueBufferTest.java new file mode 100644 index 000000000..1b441fab6 --- /dev/null +++ b/dc3-common/dc3-common-driver/src/test/java/io/github/pnoker/common/driver/buffer/PointValueBufferTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.github.pnoker.common.driver.buffer; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies the SQLite DAO against a temporary on-disk database. + * + * @author pnoker + * @version 2026.5.22 + * @since 2026.6.2 + */ +class PointValueBufferTest { + + @Test + void upsertSelectDeleteRoundtrip(@TempDir Path tmp) { + PointValueBuffer buffer = newBuffer(tmp); + long now = epoch(); + buffer.upsert(rec("id-1", 10L, 20L, 1, now, now)); + buffer.upsert(rec("id-2", 11L, 21L, 1, now, now)); + + assertThat(buffer.selectPending(10, now)).hasSize(2); + buffer.delete("id-1"); + assertThat(buffer.selectPending(10, now)).hasSize(1); + buffer.close(); + } + + @Test + void selectPendingSkipsFutureAttempts(@TempDir Path tmp) { + PointValueBuffer buffer = newBuffer(tmp); + long now = epoch(); + buffer.upsert(rec("due", 10L, 20L, 1, now, now)); + buffer.upsert(rec("future", 11L, 21L, 1, now + 3600, now)); + + List pending = buffer.selectPending(10, now); + assertThat(pending).hasSize(1).extracting(BufferedPointValue::id).contains("due"); + buffer.close(); + } + + @Test + void markRetryBumpsAttemptAndBackoff(@TempDir Path tmp) { + PointValueBuffer buffer = newBuffer(tmp); + long now = epoch(); + buffer.upsert(rec("id", 10L, 20L, 1, now, now)); + + buffer.markRetry("id", 2, now + 60); + assertThat(buffer.selectPending(10, now)).isEmpty(); + List later = buffer.selectPending(10, now + 60); + assertThat(later).hasSize(1); + assertThat(later.get(0).attempt()).isEqualTo(2); + buffer.close(); + } + + @Test + void deleteOldestEvictsByCreatedAt(@TempDir Path tmp) { + PointValueBuffer buffer = newBuffer(tmp); + long now = epoch(); + buffer.upsert(rec("old", 10L, 20L, 1, now, now - 100)); + buffer.upsert(rec("new", 11L, 21L, 1, now, now)); + + assertThat(buffer.deleteOldest(1)).isEqualTo(1); + List pending = buffer.selectPending(10, now); + assertThat(pending).hasSize(1).extracting(BufferedPointValue::id).contains("new"); + buffer.close(); + } + + @Test + void upsertReplacesExistingRow(@TempDir Path tmp) { + PointValueBuffer buffer = newBuffer(tmp); + long now = epoch(); + buffer.upsert(rec("id", 10L, 20L, 1, now, now)); + buffer.upsert(rec("id", 10L, 20L, 2, now + 30, now)); + + assertThat(buffer.count()).isEqualTo(1); + assertThat(buffer.selectPending(10, now + 30).get(0).attempt()).isEqualTo(2); + buffer.close(); + } + + private PointValueBuffer newBuffer(Path tmp) { + PointValueBuffer buffer = new PointValueBuffer(tmp.resolve("buffer.db").toString()); + buffer.initialize(); + return buffer; + } + + private BufferedPointValue rec(String id, Long deviceId, Long pointId, int attempt, long nextAttemptAt, long createdAt) { + return new BufferedPointValue(id, deviceId, pointId, 1L, 2L, "{}", "rk", attempt, nextAttemptAt, createdAt); + } + + private static long epoch() { + return System.currentTimeMillis() / 1000; + } +} diff --git a/dc3-common/dc3-common-facade/dc3-common-facade-grpc/pom.xml b/dc3-common/dc3-common-facade/dc3-common-facade-grpc/pom.xml index 111ffd4ef..c78401517 100644 --- a/dc3-common/dc3-common-facade/dc3-common-facade-grpc/pom.xml +++ b/dc3-common/dc3-common-facade/dc3-common-facade-grpc/pom.xml @@ -77,12 +77,21 @@ dc3-common-public - org.springdoc springdoc-openapi-starter-common + + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-context + diff --git a/dc3-common/dc3-common-facade/dc3-common-facade-grpc/src/main/java/io/github/pnoker/common/facade/grpc/config/RequestIdGrpcClientInterceptor.java b/dc3-common/dc3-common-facade/dc3-common-facade-grpc/src/main/java/io/github/pnoker/common/facade/grpc/config/RequestIdGrpcClientInterceptor.java index 40b34fd25..a46f61c92 100644 --- a/dc3-common/dc3-common-facade/dc3-common-facade-grpc/src/main/java/io/github/pnoker/common/facade/grpc/config/RequestIdGrpcClientInterceptor.java +++ b/dc3-common/dc3-common-facade/dc3-common-facade-grpc/src/main/java/io/github/pnoker/common/facade/grpc/config/RequestIdGrpcClientInterceptor.java @@ -24,6 +24,8 @@ import io.grpc.ClientInterceptor; import io.grpc.ForwardingClientCall; import io.grpc.Metadata; import io.grpc.MethodDescriptor; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import org.slf4j.MDC; import org.springframework.grpc.client.GlobalClientInterceptor; import org.springframework.stereotype.Component; @@ -34,9 +36,17 @@ import org.springframework.stereotype.Component; * the same id in its log lines. This closes the traceId gap across the * HTTP → Gateway → gRPC → center-services hop. * - *

The header name mirrors the HTTP {@code X-Request-Id} convention. When the MDC has no - * request id (e.g. a gRPC call triggered outside any HTTP request, such as a Quartz job or - * a driver registration), nothing is attached and the call proceeds unchanged. + *

Production-grade OpenTelemetry Integration: This interceptor now integrates + * with OpenTelemetry. It will use: + *

    + *
  1. MDC requestId (from existing RequestId mechanism)
    2. + *
  2. OpenTelemetry Trace ID (if available and no MDC value)
    3. + *
+ * This ensures full compatibility with both systems while maintaining backward compatibility. + * + *

The header name mirrors the HTTP {@code X-Request-Id} convention. When no id is available + * (e.g. a gRPC call triggered outside any HTTP request, such as a Quartz job or a driver + * registration), nothing is attached and the call proceeds unchanged. * *

Registered as a {@code @GlobalClientInterceptor} bean so spring-grpc applies it to every * client channel automatically. @@ -65,13 +75,25 @@ public class RequestIdGrpcClientInterceptor implements ClientInterceptor { public ClientCall interceptCall(MethodDescriptor method, CallOptions callOptions, Channel next) { String requestId = MDC.get(MDC_REQUEST_ID); + + // If MDC doesn't have requestId, try to get it from OpenTelemetry + if (requestId == null || requestId.isBlank()) { + Span currentSpan = Span.current(); + SpanContext spanContext = currentSpan.getSpanContext(); + if (spanContext.isValid()) { + requestId = spanContext.getTraceId(); + } + } + if (requestId == null || requestId.isBlank()) { return next.newCall(method, callOptions); } + + final String finalRequestId = requestId; return new ForwardingClientCall.SimpleForwardingClientCall<>(next.newCall(method, callOptions)) { @Override public void start(Listener responseListener, Metadata headers) { - headers.put(REQUEST_ID_KEY, requestId); + headers.put(REQUEST_ID_KEY, finalRequestId); super.start(responseListener, headers); } }; diff --git a/dc3-common/dc3-common-facade/dc3-common-facade-grpc/src/main/java/io/github/pnoker/common/facade/grpc/config/RequestIdGrpcServerInterceptor.java b/dc3-common/dc3-common-facade/dc3-common-facade-grpc/src/main/java/io/github/pnoker/common/facade/grpc/config/RequestIdGrpcServerInterceptor.java index 95cd3c3c2..810fa5788 100644 --- a/dc3-common/dc3-common-facade/dc3-common-facade-grpc/src/main/java/io/github/pnoker/common/facade/grpc/config/RequestIdGrpcServerInterceptor.java +++ b/dc3-common/dc3-common-facade/dc3-common-facade-grpc/src/main/java/io/github/pnoker/common/facade/grpc/config/RequestIdGrpcServerInterceptor.java @@ -22,6 +22,8 @@ import io.grpc.Metadata; import io.grpc.ServerCall; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import org.slf4j.MDC; import org.springframework.grpc.server.GlobalServerInterceptor; import org.springframework.stereotype.Component; @@ -34,9 +36,18 @@ import java.util.UUID; * every log line emitted by the service implementation carries the same id as the * originating HTTP request. * - *

When the caller did not send an {@code X-Request-Id} (e.g. a driver registering itself, - * or any gRPC call not initiated from an HTTP request), a fresh UUID is generated so the - * request is still traceable within this service. + *

Production-grade OpenTelemetry Integration: This interceptor now integrates + * with OpenTelemetry. It will use (in order of priority): + *

    + *
  1. X-Request-Id from gRPC metadata (backward compatibility)
    2. + *
  2. OpenTelemetry Trace ID (if available)
    3. + *
  3. A fresh UUID as last resort
    4. + *
+ * This ensures full compatibility with both systems while maintaining backward compatibility. + * + *

When no id is available (e.g. a driver registering itself, or any gRPC call not + * initiated from an HTTP request), a fresh UUID is generated so the request is still + * traceable within this service. * *

MDC is set on {@code onHalfClose} (right before the service method runs) and cleared on * completion or cancellation, to avoid leaking the entry across pooled netty threads. It is @@ -70,9 +81,22 @@ public class RequestIdGrpcServerInterceptor implements ServerInterceptor { // Read the id supplied by the caller (set by RequestIdGrpcClientInterceptor upstream), // or mint one when absent so the call is still self-consistent in this service's logs. String requestId = headers.get(REQUEST_ID_KEY); + + // Priority 1: Use X-Request-Id from header + // Priority 2: Use OpenTelemetry Trace ID if available + if (requestId == null || requestId.isBlank()) { + Span currentSpan = Span.current(); + SpanContext spanContext = currentSpan.getSpanContext(); + if (spanContext.isValid()) { + requestId = spanContext.getTraceId(); + } + } + + // Priority 3: Fall back to UUID if (requestId == null || requestId.isBlank()) { requestId = UUID.randomUUID().toString(); } + return new RequestIdListener<>(next.startCall(call, headers), requestId); } diff --git a/dc3-common/dc3-common-gateway/src/main/java/io/github/pnoker/common/gateway/mcp/McpGatewayController.java b/dc3-common/dc3-common-gateway/src/main/java/io/github/pnoker/common/gateway/mcp/McpGatewayController.java index 263f94237..880532981 100644 --- a/dc3-common/dc3-common-gateway/src/main/java/io/github/pnoker/common/gateway/mcp/McpGatewayController.java +++ b/dc3-common/dc3-common-gateway/src/main/java/io/github/pnoker/common/gateway/mcp/McpGatewayController.java @@ -436,15 +436,15 @@ public class McpGatewayController { * Record a tool-call audit entry (success, denial, or error) with trace id, * duration, argument digest, and client metadata from the exchange. * - * @param context the introspected token context - * @param tool the resolved tool definition - * @param traceId the call trace id - * @param arguments the tool arguments - * @param controls confirmation and idempotency controls - * @param status audit status (success/denied/error) - * @param errorCode error code on failure, empty otherwise - * @param start the call start nanos, for duration - * @param exchange current server exchange, for client metadata and remote ip + * @param context the introspected token context + * @param tool the resolved tool definition + * @param traceId the call trace id + * @param arguments the tool arguments + * @param controls confirmation and idempotency controls + * @param status audit status (success/denied/error) + * @param errorCode error code on failure, empty otherwise + * @param start the call start nanos, for duration + * @param exchange current server exchange, for client metadata and remote ip * @return a mono completing when the audit is recorded */ private Mono audit(McpIntrospectResponseDTO context, McpToolResolveResponseDTO tool, String traceId, diff --git a/dc3-common/dc3-common-log/src/main/resources/logback.xml b/dc3-common/dc3-common-log/src/main/resources/logback.xml index daadeaea2..1f905acb7 100644 --- a/dc3-common/dc3-common-log/src/main/resources/logback.xml +++ b/dc3-common/dc3-common-log/src/main/resources/logback.xml @@ -20,7 +20,7 @@ + value="%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([%X{requestId:-},%X{traceId:-},%X{spanId:-}]){faint} %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){magenta} [%clr(%4line){magenta}] %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n%wEx"/> diff --git a/dc3-common/dc3-common-rabbitmq/pom.xml b/dc3-common/dc3-common-rabbitmq/pom.xml index 01ed5a455..54e2471fd 100644 --- a/dc3-common/dc3-common-rabbitmq/pom.xml +++ b/dc3-common/dc3-common-rabbitmq/pom.xml @@ -83,6 +83,15 @@ io.github.pnoker dc3-common-public + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-context + diff --git a/dc3-common/dc3-common-rabbitmq/src/main/java/io/github/pnoker/common/config/MdcRequestIdListenerAdvice.java b/dc3-common/dc3-common-rabbitmq/src/main/java/io/github/pnoker/common/config/MdcRequestIdListenerAdvice.java index 0ce526eff..842d7c89c 100644 --- a/dc3-common/dc3-common-rabbitmq/src/main/java/io/github/pnoker/common/config/MdcRequestIdListenerAdvice.java +++ b/dc3-common/dc3-common-rabbitmq/src/main/java/io/github/pnoker/common/config/MdcRequestIdListenerAdvice.java @@ -17,6 +17,8 @@ package io.github.pnoker.common.config; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import org.aopalliance.intercept.MethodInterceptor; import org.aopalliance.intercept.MethodInvocation; import org.slf4j.MDC; @@ -29,13 +31,22 @@ import java.util.UUID; * Restores the request id into the MDC for the duration of a RabbitMQ message handling, so the * traceId chain is continuous across the HTTP → gRPC → RabbitMQ hops. * + *

Production-grade OpenTelemetry Integration: This advice now integrates + * with OpenTelemetry. It will use (in order of priority): + *

    + *
  1. X-Request-Id from message header (backward compatibility)
    2. + *
  2. OpenTelemetry Trace ID (if available)
    3. + *
  3. A fresh UUID as last resort
    4. + *
+ * This ensures full compatibility with both systems while maintaining backward compatibility. + * *

The id is carried in the {@value #HEADER_REQUEST_ID} message header, stamped on the * producer side by the {@code beforePublish} post-processor in {@link RabbitConfig}. On the * consumer side, the listener container runs on a pooled thread; this advice reads the header * before the listener runs, publishes it into the MDC, and removes it afterwards — guaranteeing * the entry never leaks to the next message processed on the same pooled thread. * - *

When the header is absent (e.g. a driver registration, a Quartz-triggered publish, or a + *

When no id is available (e.g. a driver registration, a Quartz-triggered publish, or a * producer that predates this wiring), a fresh UUID is minted so the consumer's logs are still * self-consistent within that single message's handling. * @@ -88,10 +99,18 @@ public class MdcRequestIdListenerAdvice implements MethodInterceptor { } /** - * Read the request id from the message header, falling back to a fresh UUID so the consumer - * is always traceable within itself even when the producer did not stamp one. + * Read the request id from the message header, falling back to OpenTelemetry Trace ID, + * and finally a fresh UUID so the consumer is always traceable. + *

+ * Priority order: + *

    + *
  1. X-Request-Id from message header (backward compatibility)
    2. + *
  2. OpenTelemetry Trace ID (if available)
    3. + *
  3. Fresh UUID as last resort
    4. + *
*/ private String readRequestId(Message message) { + // Priority 1: Use X-Request-Id from header if (message != null) { MessageProperties properties = message.getMessageProperties(); if (properties != null) { @@ -101,6 +120,15 @@ public class MdcRequestIdListenerAdvice implements MethodInterceptor { } } } + + // Priority 2: Use OpenTelemetry Trace ID if available + Span currentSpan = Span.current(); + SpanContext spanContext = currentSpan.getSpanContext(); + if (spanContext.isValid()) { + return spanContext.getTraceId(); + } + + // Priority 3: Fall back to UUID return UUID.randomUUID().toString(); } } diff --git a/dc3-common/dc3-common-rabbitmq/src/main/java/io/github/pnoker/common/config/MdcRequestIdMessagePostProcessor.java b/dc3-common/dc3-common-rabbitmq/src/main/java/io/github/pnoker/common/config/MdcRequestIdMessagePostProcessor.java index 98c78830e..2fe6a423b 100644 --- a/dc3-common/dc3-common-rabbitmq/src/main/java/io/github/pnoker/common/config/MdcRequestIdMessagePostProcessor.java +++ b/dc3-common/dc3-common-rabbitmq/src/main/java/io/github/pnoker/common/config/MdcRequestIdMessagePostProcessor.java @@ -17,6 +17,8 @@ package io.github.pnoker.common.config; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import org.slf4j.MDC; import org.springframework.amqp.core.Message; import org.springframework.amqp.core.MessagePostProcessor; @@ -27,7 +29,15 @@ import org.springframework.amqp.core.MessagePostProcessor; * {@link MdcRequestIdListenerAdvice} and keep the traceId chain continuous across the broker * hop. * - *

When MDC has no request id (a publish not driven by an HTTP request — driver + *

Production-grade OpenTelemetry Integration: This post processor now integrates + * with OpenTelemetry. It will use (in order of priority): + *

    + *
  1. MDC requestId (from existing RequestId mechanism)
    2. + *
  2. OpenTelemetry Trace ID (if available and no MDC value)
    3. + *
+ * This ensures full compatibility with both systems while maintaining backward compatibility. + * + *

When no id is available (a publish not driven by an HTTP request — driver * registration, Quartz job, …), nothing is stamped and the consumer mints its own id. * *

Extracted as a named class (rather than a lambda in {@link RabbitConfig}) so it can be @@ -52,6 +62,16 @@ public class MdcRequestIdMessagePostProcessor implements MessagePostProcessor { @Override public Message postProcessMessage(Message message) { String requestId = MDC.get(MDC_REQUEST_ID); + + // If MDC doesn't have requestId, try to get it from OpenTelemetry + if (requestId == null || requestId.isBlank()) { + Span currentSpan = Span.current(); + SpanContext spanContext = currentSpan.getSpanContext(); + if (spanContext.isValid()) { + requestId = spanContext.getTraceId(); + } + } + if (requestId != null && !requestId.isBlank()) { message.getMessageProperties().setHeader(HEADER_REQUEST_ID, requestId); } diff --git a/dc3-common/dc3-common-web/pom.xml b/dc3-common/dc3-common-web/pom.xml index 75b7faf60..fa06685e5 100644 --- a/dc3-common/dc3-common-web/pom.xml +++ b/dc3-common/dc3-common-web/pom.xml @@ -87,9 +87,35 @@ io.micrometer micrometer-registry-prometheus + + io.micrometer - micrometer-tracing-bridge-brave + micrometer-tracing-bridge-otel + + + io.opentelemetry + opentelemetry-exporter-otlp + + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-sdk-trace + + + io.opentelemetry + opentelemetry-extension-trace-propagators + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-context diff --git a/dc3-common/dc3-common-web/src/main/java/io/github/pnoker/common/filter/RequestIdWebFilter.java b/dc3-common/dc3-common-web/src/main/java/io/github/pnoker/common/filter/RequestIdWebFilter.java index 95e4c82fb..029c74d03 100644 --- a/dc3-common/dc3-common-web/src/main/java/io/github/pnoker/common/filter/RequestIdWebFilter.java +++ b/dc3-common/dc3-common-web/src/main/java/io/github/pnoker/common/filter/RequestIdWebFilter.java @@ -17,6 +17,8 @@ package io.github.pnoker.common.filter; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; @@ -26,18 +28,30 @@ import org.springframework.web.server.WebFilter; import org.springframework.web.server.WebFilterChain; import reactor.core.publisher.Mono; import reactor.util.context.Context; -import reactor.util.context.ContextView; import java.util.UUID; /** * Propagates a per-request id so every log line within a request carries the same * {@code requestId} (rendered by the {@code [%X{requestId:-}]} slot in the logback pattern). + *

+ * Production-grade OpenTelemetry Integration: This filter now integrates with + * OpenTelemetry. The requestId is set to the OpenTelemetry Trace ID when a trace exists, + * falling back to a UUID when no trace is present. This ensures: + *

* - *

The id is taken from the inbound {@code X-Request-Id} header when present (allowing an - * upstream gateway or caller to supply one), otherwise a fresh UUID is generated. The same id is - * echoed back on the response via {@code X-Request-Id}, so callers can correlate a failing - * request with server logs. + *

The id is taken from (in order of priority): + *

    + *
  1. The inbound {@code X-Request-Id} header (backward compatibility)
    2. + *
  2. The current OpenTelemetry Trace ID
    3. + *
  3. A fresh UUID as a last resort
    4. + *
+ * The same id is echoed back on the response via {@code X-Request-Id}, so callers can + * correlate a failing request with server logs and distributed traces. * *

Why Reactor Context, not MDC: This is a WebFlux application. Controllers run their * blocking suppliers on {@code Schedulers.boundedElastic()} (see {@code BaseController.async()}), @@ -76,11 +90,24 @@ public class RequestIdWebFilter implements WebFilter { public Mono filter(ServerWebExchange exchange, WebFilterChain chain) { HttpHeaders headers = exchange.getRequest().getHeaders(); String requestId = headers.getFirst(HEADER_REQUEST_ID); + + // Priority 1: Use X-Request-Id from header (backward compatibility) + // Priority 2: Use OpenTelemetry Trace ID if available + if (requestId == null || requestId.isBlank()) { + Span currentSpan = Span.current(); + SpanContext spanContext = currentSpan.getSpanContext(); + if (spanContext.isValid()) { + requestId = spanContext.getTraceId(); + } + } + + // Priority 3: Fall back to UUID if no trace or header if (requestId == null || requestId.isBlank()) { requestId = UUID.randomUUID().toString(); } + String finalRequestId = requestId; - // Echo back so callers can correlate a failing request with server-side logs. + // Echo back so callers can correlate a failing request with server-side logs and traces. exchange.getResponse().getHeaders().add(HEADER_REQUEST_ID, finalRequestId); // Publish the id through the Reactor Context (not just MDC): the Context propagates // along the reactive call chain regardless of thread hops, so BaseController.async() diff --git a/dc3-common/dc3-common-web/src/main/resources/application-web.yml b/dc3-common/dc3-common-web/src/main/resources/application-web.yml index c34a5199c..7022602b8 100644 --- a/dc3-common/dc3-common-web/src/main/resources/application-web.yml +++ b/dc3-common/dc3-common-web/src/main/resources/application-web.yml @@ -46,8 +46,32 @@ management: endpoints: web: exposure: - # Expose health and Prometheus endpoints for readiness and metrics scraping. - include: health,prometheus + # Expose health, Prometheus, and additional observability endpoints. + include: health,prometheus,info,metrics + metrics: + export: + prometheus: + enabled: true + tags: + application: ${spring.application.name} + # OpenTelemetry Tracing Configuration (Production-grade) + tracing: + enabled: ${OTEL_TRACING_ENABLED:true} + sampling: + probability: ${OTEL_TRACING_SAMPLING_PROBABILITY:1.0} # 1.0 for development, reduce to 0.1-0.3 in production + propagation: + type: B3,W3C # B3 for legacy systems, W3C for standard trace context + baggage: + enabled: true + export: + otlp: + enabled: ${OTEL_OTLP_EXPORTER_ENABLED:true} # Enable OTLP exporter by default + # OTLP tracing export endpoint (Spring Boot: management.otlp.tracing.*, NOT management.tracing.otlp.*) + otlp: + tracing: + endpoint: ${OTEL_EXPORTER_OTLP_ENDPOINT:http://dc3-jaeger:4318/v1/traces} + timeout: ${OTEL_EXPORTER_OTLP_TIMEOUT:10s} + compression: gzip # OpenAPI / Swagger UI baseline, shared by every web service (auth, manager, # data, agentic, single) through the auto-activated "web" profile. Document diff --git a/dc3-driver/dc3-driver-bacnet-ip/src/main/resources/application.yml b/dc3-driver/dc3-driver-bacnet-ip/src/main/resources/application.yml index a918dc6cf..eef481c91 100644 --- a/dc3-driver/dc3-driver-bacnet-ip/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-bacnet-ip/src/main/resources/application.yml @@ -105,6 +105,10 @@ dc3: default-value: PRESENT_VALUE remark: Property identifier + buffer: + enable: true + db-path: dc3/data/driver/bacnet-ip/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-ble/src/main/resources/application.yml b/dc3-driver/dc3-driver-ble/src/main/resources/application.yml index 7b2478f3a..2ad9af888 100644 --- a/dc3-driver/dc3-driver-ble/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-ble/src/main/resources/application.yml @@ -85,6 +85,10 @@ dc3: default-value: '' remark: GATT Characteristic UUID for writing + buffer: + enable: true + db-path: dc3/data/driver/ble/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-can/src/main/resources/application.yml b/dc3-driver/dc3-driver-can/src/main/resources/application.yml index 787e1419e..7986470c0 100644 --- a/dc3-driver/dc3-driver-can/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-can/src/main/resources/application.yml @@ -89,6 +89,10 @@ dc3: attribute-type-flag: STRING default-value: '${value}' + buffer: + enable: true + db-path: dc3/data/driver/can/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-coap/src/main/resources/application.yml b/dc3-driver/dc3-driver-coap/src/main/resources/application.yml index 4d41c03b4..a6224fd85 100644 --- a/dc3-driver/dc3-driver-coap/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-coap/src/main/resources/application.yml @@ -64,6 +64,10 @@ dc3: remark: Content format: json, text, cbor, octet-stream + buffer: + enable: true + db-path: dc3/data/driver/coap/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-dlms/src/main/resources/application.yml b/dc3-driver/dc3-driver-dlms/src/main/resources/application.yml index b021ec300..9fd5e0f06 100644 --- a/dc3-driver/dc3-driver-dlms/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-dlms/src/main/resources/application.yml @@ -100,6 +100,10 @@ dc3: remark: Attribute ID (2=Present Value) command-attribute: [ ] + buffer: + enable: true + db-path: dc3/data/driver/dlms/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-ethernet-ip/src/main/resources/application.yml b/dc3-driver/dc3-driver-ethernet-ip/src/main/resources/application.yml index dec38a9cf..612ae77de 100644 --- a/dc3-driver/dc3-driver-ethernet-ip/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-ethernet-ip/src/main/resources/application.yml @@ -71,6 +71,10 @@ dc3: attribute-type-flag: STRING default-value: '${value}' + buffer: + enable: true + db-path: dc3/data/driver/ethernet-ip/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-fins/src/main/resources/application.yml b/dc3-driver/dc3-driver-fins/src/main/resources/application.yml index 3583d7c62..f11224159 100644 --- a/dc3-driver/dc3-driver-fins/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-fins/src/main/resources/application.yml @@ -102,6 +102,10 @@ dc3: attribute-type-flag: STRING default-value: UINT16 + buffer: + enable: true + db-path: dc3/data/driver/fins/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-http/src/main/resources/application.yml b/dc3-driver/dc3-driver-http/src/main/resources/application.yml index c7c580883..a5874589e 100644 --- a/dc3-driver/dc3-driver-http/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-http/src/main/resources/application.yml @@ -90,6 +90,10 @@ dc3: default-value: POST remark: HTTP method for command + buffer: + enable: true + db-path: dc3/data/driver/http/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-iec104/src/main/resources/application.yml b/dc3-driver/dc3-driver-iec104/src/main/resources/application.yml index 3aa7fa170..011932883 100644 --- a/dc3-driver/dc3-driver-iec104/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-iec104/src/main/resources/application.yml @@ -79,6 +79,10 @@ dc3: attribute-type-flag: STRING default-value: '${value}' + buffer: + enable: true + db-path: dc3/data/driver/iec104/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-listening-virtual/src/main/resources/application.yml b/dc3-driver/dc3-driver-listening-virtual/src/main/resources/application.yml index 5cb30ce36..ae091af45 100644 --- a/dc3-driver/dc3-driver-listening-virtual/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-listening-virtual/src/main/resources/application.yml @@ -63,6 +63,10 @@ dc3: default-value: string remark: Parse type, short, int, long, float, double, boolean, string + buffer: + enable: true + db-path: dc3/data/driver/listening-virtual/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-lwm2m/src/main/resources/application.yml b/dc3-driver/dc3-driver-lwm2m/src/main/resources/application.yml index c1c7c7e1c..836f444a9 100644 --- a/dc3-driver/dc3-driver-lwm2m/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-lwm2m/src/main/resources/application.yml @@ -100,6 +100,10 @@ dc3: secure-port: 5684 security-mode: NOSEC + buffer: + enable: true + db-path: dc3/data/driver/lwm2m/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-melsec/src/main/resources/application.yml b/dc3-driver/dc3-driver-melsec/src/main/resources/application.yml index b6ec00fb3..d0713615f 100644 --- a/dc3-driver/dc3-driver-melsec/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-melsec/src/main/resources/application.yml @@ -64,6 +64,10 @@ dc3: default-value: 0 remark: String read length (0 for non-string types) + buffer: + enable: true + db-path: dc3/data/driver/melsec/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-modbus-rtu/src/main/resources/application.yml b/dc3-driver/dc3-driver-modbus-rtu/src/main/resources/application.yml index a32f774e5..1ee3cc295 100644 --- a/dc3-driver/dc3-driver-modbus-rtu/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-modbus-rtu/src/main/resources/application.yml @@ -100,6 +100,10 @@ dc3: default-value: '${value}' remark: Value template rendered with command params + buffer: + enable: true + db-path: dc3/data/driver/modbus-rtu/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-modbus-tcp/src/main/resources/application.yml b/dc3-driver/dc3-driver-modbus-tcp/src/main/resources/application.yml index fb3452590..02911ec61 100644 --- a/dc3-driver/dc3-driver-modbus-tcp/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-modbus-tcp/src/main/resources/application.yml @@ -85,6 +85,10 @@ dc3: default-value: '${value}' remark: Value template rendered with command params + buffer: + enable: true + db-path: dc3/data/driver/modbus-tcp/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-mqtt/src/main/resources/application.yml b/dc3-driver/dc3-driver-mqtt/src/main/resources/application.yml index 05611fd15..4e8507c75 100644 --- a/dc3-driver/dc3-driver-mqtt/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-mqtt/src/main/resources/application.yml @@ -80,6 +80,10 @@ dc3: default-value: $.payload remark: JSON path used to resolve event payload + buffer: + enable: true + db-path: dc3/data/driver/mqtt/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-mysql/src/main/resources/application.yml b/dc3-driver/dc3-driver-mysql/src/main/resources/application.yml index 444a2b95a..f921790e6 100644 --- a/dc3-driver/dc3-driver-mysql/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-mysql/src/main/resources/application.yml @@ -85,6 +85,10 @@ dc3: default-value: remark: SQL query to execute for command + buffer: + enable: true + db-path: dc3/data/driver/mysql/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-opc-da/src/main/resources/application.yml b/dc3-driver/dc3-driver-opc-da/src/main/resources/application.yml index 51dfe44f0..6a944e062 100644 --- a/dc3-driver/dc3-driver-opc-da/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-opc-da/src/main/resources/application.yml @@ -69,6 +69,10 @@ dc3: default-value: TAG remark: OPC DA item tag name + buffer: + enable: true + db-path: dc3/data/driver/opc-da/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-opc-ua/src/main/resources/application.yml b/dc3-driver/dc3-driver-opc-ua/src/main/resources/application.yml index 0f64272f3..c7983d79b 100644 --- a/dc3-driver/dc3-driver-opc-ua/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-opc-ua/src/main/resources/application.yml @@ -64,6 +64,10 @@ dc3: default-value: TAG remark: OPC UA node tag name + buffer: + enable: true + db-path: dc3/data/driver/opc-ua/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-oracle/src/main/resources/application.yml b/dc3-driver/dc3-driver-oracle/src/main/resources/application.yml index 72da7a385..a9476607b 100644 --- a/dc3-driver/dc3-driver-oracle/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-oracle/src/main/resources/application.yml @@ -100,6 +100,10 @@ dc3: default-value: remark: SQL query to execute for command + buffer: + enable: true + db-path: dc3/data/driver/oracle/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-plcs7/src/main/resources/application.yml b/dc3-driver/dc3-driver-plcs7/src/main/resources/application.yml index ad0cdaa3c..b4547e23b 100644 --- a/dc3-driver/dc3-driver-plcs7/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-plcs7/src/main/resources/application.yml @@ -69,6 +69,10 @@ dc3: default-value: 0 remark: Bit offset (only used for boolean type) + buffer: + enable: true + db-path: dc3/data/driver/plcs7/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-postgresql/src/main/resources/application.yml b/dc3-driver/dc3-driver-postgresql/src/main/resources/application.yml index 3c40b8140..990f54a1d 100644 --- a/dc3-driver/dc3-driver-postgresql/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-postgresql/src/main/resources/application.yml @@ -85,6 +85,10 @@ dc3: default-value: remark: SQL query to execute for command + buffer: + enable: true + db-path: dc3/data/driver/postgresql/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-serial/src/main/resources/application.yml b/dc3-driver/dc3-driver-serial/src/main/resources/application.yml index fb4cf3848..5bac30ef0 100644 --- a/dc3-driver/dc3-driver-serial/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-serial/src/main/resources/application.yml @@ -128,6 +128,10 @@ dc3: remark: Byte order for encoding value: BIG, LITTLE + buffer: + enable: true + db-path: dc3/data/driver/serial/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-sl651/src/main/resources/application.yml b/dc3-driver/dc3-driver-sl651/src/main/resources/application.yml index d0c8e2a51..ee6f5a3c5 100644 --- a/dc3-driver/dc3-driver-sl651/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-sl651/src/main/resources/application.yml @@ -52,6 +52,10 @@ dc3: default-value: 0 remark: Zero-based index into the telemetry body element list + buffer: + enable: true + db-path: dc3/data/driver/sl651/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-snmp/src/main/resources/application.yml b/dc3-driver/dc3-driver-snmp/src/main/resources/application.yml index 1b6366e1f..b2840868f 100644 --- a/dc3-driver/dc3-driver-snmp/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-snmp/src/main/resources/application.yml @@ -97,6 +97,10 @@ dc3: attribute-type-flag: STRING default-value: OCTET_STRING + buffer: + enable: true + db-path: dc3/data/driver/snmp/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-sqlserver/src/main/resources/application.yml b/dc3-driver/dc3-driver-sqlserver/src/main/resources/application.yml index e1120834e..9fefa8d35 100644 --- a/dc3-driver/dc3-driver-sqlserver/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-sqlserver/src/main/resources/application.yml @@ -95,6 +95,10 @@ dc3: default-value: remark: SQL query to execute for command + buffer: + enable: true + db-path: dc3/data/driver/sqlserver/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-tcp-udp/src/main/resources/application.yml b/dc3-driver/dc3-driver-tcp-udp/src/main/resources/application.yml index 865072f54..8f4ab0fc6 100644 --- a/dc3-driver/dc3-driver-tcp-udp/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-tcp-udp/src/main/resources/application.yml @@ -103,6 +103,10 @@ dc3: attribute-type-flag: STRING default-value: '${value}' + buffer: + enable: true + db-path: dc3/data/driver/tcp-udp/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-virtual/src/main/resources/application.yml b/dc3-driver/dc3-driver-virtual/src/main/resources/application.yml index 06da245a1..f4b1f05ee 100644 --- a/dc3-driver/dc3-driver-virtual/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-virtual/src/main/resources/application.yml @@ -76,6 +76,10 @@ dc3: default-value: $.payload remark: JSON path used to resolve event payload + buffer: + enable: true + db-path: dc3/data/driver/virtual/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3-driver/dc3-driver-zigbee/src/main/resources/application.yml b/dc3-driver/dc3-driver-zigbee/src/main/resources/application.yml index a070ad8f0..f830a7c0e 100644 --- a/dc3-driver/dc3-driver-zigbee/src/main/resources/application.yml +++ b/dc3-driver/dc3-driver-zigbee/src/main/resources/application.yml @@ -105,6 +105,10 @@ dc3: default-value: 0 remark: Attribute ID for writing + buffer: + enable: true + db-path: dc3/data/driver/zigbee/buffer.db + spring: application: name: @project.artifactId@ diff --git a/dc3/docker-compose-dev.yml b/dc3/docker-compose-dev.yml index 64d19a723..92782513f 100644 --- a/dc3/docker-compose-dev.yml +++ b/dc3/docker-compose-dev.yml @@ -237,6 +237,7 @@ services: hostname: dc3-driver-listening-virtual volumes: - logs:/dc3-driver/dc3-driver-listening-virtual/dc3/logs + - driver_data:/dc3-driver/dc3-driver-listening-virtual/dc3/data logging: *default-logging networks: dc3net: @@ -259,6 +260,7 @@ services: hostname: dc3-driver-modbus-tcp volumes: - logs:/dc3-driver/dc3-driver-modbus-tcp/dc3/logs + - driver_data:/dc3-driver/dc3-driver-modbus-tcp/dc3/data logging: *default-logging networks: dc3net: @@ -281,6 +283,7 @@ services: hostname: dc3-driver-modbus-rtu volumes: - logs:/dc3-driver/dc3-driver-modbus-rtu/dc3/logs + - driver_data:/dc3-driver/dc3-driver-modbus-rtu/dc3/data logging: *default-logging networks: dc3net: @@ -318,6 +321,7 @@ services: hostname: dc3-driver-bacnet-ip volumes: - logs:/dc3-driver/dc3-driver-bacnet-ip/dc3/logs + - driver_data:/dc3-driver/dc3-driver-bacnet-ip/dc3/data logging: *default-logging networks: dc3net: @@ -340,6 +344,7 @@ services: hostname: dc3-driver-ble volumes: - logs:/dc3-driver/dc3-driver-ble/dc3/logs + - driver_data:/dc3-driver/dc3-driver-ble/dc3/data logging: *default-logging networks: dc3net: @@ -362,6 +367,7 @@ services: hostname: dc3-driver-can volumes: - logs:/dc3-driver/dc3-driver-can/dc3/logs + - driver_data:/dc3-driver/dc3-driver-can/dc3/data logging: *default-logging networks: dc3net: @@ -384,6 +390,7 @@ services: hostname: dc3-driver-coap volumes: - logs:/dc3-driver/dc3-driver-coap/dc3/logs + - driver_data:/dc3-driver/dc3-driver-coap/dc3/data logging: *default-logging networks: dc3net: @@ -406,6 +413,7 @@ services: hostname: dc3-driver-dlms volumes: - logs:/dc3-driver/dc3-driver-dlms/dc3/logs + - driver_data:/dc3-driver/dc3-driver-dlms/dc3/data logging: *default-logging networks: dc3net: @@ -428,6 +436,7 @@ services: hostname: dc3-driver-ethernet-ip volumes: - logs:/dc3-driver/dc3-driver-ethernet-ip/dc3/logs + - driver_data:/dc3-driver/dc3-driver-ethernet-ip/dc3/data logging: *default-logging networks: dc3net: @@ -450,6 +459,7 @@ services: hostname: dc3-driver-fins volumes: - logs:/dc3-driver/dc3-driver-fins/dc3/logs + - driver_data:/dc3-driver/dc3-driver-fins/dc3/data logging: *default-logging networks: dc3net: @@ -472,6 +482,7 @@ services: hostname: dc3-driver-http volumes: - logs:/dc3-driver/dc3-driver-http/dc3/logs + - driver_data:/dc3-driver/dc3-driver-http/dc3/data logging: *default-logging networks: dc3net: @@ -494,6 +505,7 @@ services: hostname: dc3-driver-iec104 volumes: - logs:/dc3-driver/dc3-driver-iec104/dc3/logs + - driver_data:/dc3-driver/dc3-driver-iec104/dc3/data logging: *default-logging networks: dc3net: @@ -516,6 +528,7 @@ services: hostname: dc3-driver-lwm2m volumes: - logs:/dc3-driver/dc3-driver-lwm2m/dc3/logs + - driver_data:/dc3-driver/dc3-driver-lwm2m/dc3/data logging: *default-logging networks: dc3net: @@ -538,6 +551,7 @@ services: hostname: dc3-driver-melsec volumes: - logs:/dc3-driver/dc3-driver-melsec/dc3/logs + - driver_data:/dc3-driver/dc3-driver-melsec/dc3/data logging: *default-logging networks: dc3net: @@ -560,6 +574,7 @@ services: hostname: dc3-driver-mysql volumes: - logs:/dc3-driver/dc3-driver-mysql/dc3/logs + - driver_data:/dc3-driver/dc3-driver-mysql/dc3/data logging: *default-logging networks: dc3net: @@ -582,6 +597,7 @@ services: hostname: dc3-driver-oracle volumes: - logs:/dc3-driver/dc3-driver-oracle/dc3/logs + - driver_data:/dc3-driver/dc3-driver-oracle/dc3/data logging: *default-logging networks: dc3net: @@ -604,6 +620,7 @@ services: hostname: dc3-driver-postgresql volumes: - logs:/dc3-driver/dc3-driver-postgresql/dc3/logs + - driver_data:/dc3-driver/dc3-driver-postgresql/dc3/data logging: *default-logging networks: dc3net: @@ -626,6 +643,7 @@ services: hostname: dc3-driver-serial volumes: - logs:/dc3-driver/dc3-driver-serial/dc3/logs + - driver_data:/dc3-driver/dc3-driver-serial/dc3/data logging: *default-logging networks: dc3net: @@ -648,6 +666,7 @@ services: hostname: dc3-driver-sl651 volumes: - logs:/dc3-driver/dc3-driver-sl651/dc3/logs + - driver_data:/dc3-driver/dc3-driver-sl651/dc3/data logging: *default-logging networks: dc3net: @@ -670,6 +689,7 @@ services: hostname: dc3-driver-snmp volumes: - logs:/dc3-driver/dc3-driver-snmp/dc3/logs + - driver_data:/dc3-driver/dc3-driver-snmp/dc3/data logging: *default-logging networks: dc3net: @@ -692,6 +712,7 @@ services: hostname: dc3-driver-sqlserver volumes: - logs:/dc3-driver/dc3-driver-sqlserver/dc3/logs + - driver_data:/dc3-driver/dc3-driver-sqlserver/dc3/data logging: *default-logging networks: dc3net: @@ -714,6 +735,7 @@ services: hostname: dc3-driver-tcp-udp volumes: - logs:/dc3-driver/dc3-driver-tcp-udp/dc3/logs + - driver_data:/dc3-driver/dc3-driver-tcp-udp/dc3/data logging: *default-logging networks: dc3net: @@ -736,6 +758,7 @@ services: hostname: dc3-driver-zigbee volumes: - logs:/dc3-driver/dc3-driver-zigbee/dc3/logs + - driver_data:/dc3-driver/dc3-driver-zigbee/dc3/data logging: *default-logging networks: dc3net: @@ -744,6 +767,7 @@ services: volumes: - logs:/dc3-driver/dc3-driver-mqtt/dc3/logs + - driver_data:/dc3-driver/dc3-driver-mqtt/dc3/data logging: *default-logging networks: dc3net: @@ -766,6 +790,7 @@ services: hostname: dc3-driver-opc-da volumes: - logs:/dc3-driver/dc3-driver-opc-da/dc3/logs + - driver_data:/dc3-driver/dc3-driver-opc-da/dc3/data logging: *default-logging networks: dc3net: @@ -788,6 +813,7 @@ services: hostname: dc3-driver-opc-ua volumes: - logs:/dc3-driver/dc3-driver-opc-ua/dc3/logs + - driver_data:/dc3-driver/dc3-driver-opc-ua/dc3/data logging: *default-logging networks: dc3net: @@ -810,6 +836,7 @@ services: hostname: dc3-driver-plcs7 volumes: - logs:/dc3-driver/dc3-driver-plcs7/dc3/logs + - driver_data:/dc3-driver/dc3-driver-plcs7/dc3/data logging: *default-logging networks: dc3net: @@ -832,6 +859,7 @@ services: hostname: dc3-driver-virtual volumes: - logs:/dc3-driver/dc3-driver-virtual/dc3/logs + - driver_data:/dc3-driver/dc3-driver-virtual/dc3/data logging: *default-logging networks: dc3net: @@ -840,7 +868,7 @@ services: volumes: logs: - + driver_data: networks: dc3net: driver: bridge diff --git a/dc3/docker-compose-optional.yml b/dc3/docker-compose-optional.yml index e2f5706f6..e379a14c7 100644 --- a/dc3/docker-compose-optional.yml +++ b/dc3/docker-compose-optional.yml @@ -127,6 +127,25 @@ services: aliases: - dc3-apm + # ---------- Distributed Tracing (Jaeger) ---------- + jaeger: + image: jaegertracing/all-in-one:latest + restart: always + container_name: dc3-jaeger + hostname: dc3-jaeger + ports: + - "${DC3_BIND_HOST:-127.0.0.1}:${DC3_JAEGER_UI_PORT:-16686}:16686" # Jaeger UI + - "${DC3_BIND_HOST:-127.0.0.1}:${DC3_JAEGER_OTLP_GRPC_PORT:-4317}:4317" # OTLP gRPC + - "${DC3_BIND_HOST:-127.0.0.1}:${DC3_JAEGER_OTLP_HTTP_PORT:-4318}:4318" # OTLP HTTP + environment: + - COLLECTOR_OTLP_ENABLED=true + - LOG_LEVEL=info + logging: *default-logging + networks: + dc3net: + aliases: + - dc3-jaeger + # ---------- Monitoring (Prometheus + exporters + Grafana) ---------- prometheus: image: ${DC3_IMAGE_REGISTRY:-pnoker}/dc3-prometheus:${DC3_IMAGE_TAG:-2026.6} diff --git a/dc3/docker-compose.yml b/dc3/docker-compose.yml index c00ecabb2..7f6ecb72a 100644 --- a/dc3/docker-compose.yml +++ b/dc3/docker-compose.yml @@ -237,6 +237,7 @@ services: hostname: dc3-driver-listening-virtual volumes: - logs:/dc3-driver/dc3-driver-listening-virtual/dc3/logs + - driver_data:/dc3-driver/dc3-driver-listening-virtual/dc3/data logging: *default-logging networks: dc3net: @@ -255,6 +256,7 @@ services: hostname: dc3-driver-modbus-tcp volumes: - logs:/dc3-driver/dc3-driver-modbus-tcp/dc3/logs + - driver_data:/dc3-driver/dc3-driver-modbus-tcp/dc3/data logging: *default-logging networks: dc3net: @@ -273,6 +275,7 @@ services: hostname: dc3-driver-modbus-rtu volumes: - logs:/dc3-driver/dc3-driver-modbus-rtu/dc3/logs + - driver_data:/dc3-driver/dc3-driver-modbus-rtu/dc3/data logging: *default-logging networks: dc3net: @@ -291,6 +294,7 @@ services: hostname: dc3-driver-mqtt volumes: - logs:/dc3-driver/dc3-driver-mqtt/dc3/logs + - driver_data:/dc3-driver/dc3-driver-mqtt/dc3/data logging: *default-logging networks: dc3net: @@ -309,6 +313,7 @@ services: hostname: dc3-driver-opc-da volumes: - logs:/dc3-driver/dc3-driver-opc-da/dc3/logs + - driver_data:/dc3-driver/dc3-driver-opc-da/dc3/data logging: *default-logging networks: dc3net: @@ -327,6 +332,7 @@ services: hostname: dc3-driver-opc-ua volumes: - logs:/dc3-driver/dc3-driver-opc-ua/dc3/logs + - driver_data:/dc3-driver/dc3-driver-opc-ua/dc3/data logging: *default-logging networks: dc3net: @@ -345,6 +351,7 @@ services: hostname: dc3-driver-plcs7 volumes: - logs:/dc3-driver/dc3-driver-plcs7/dc3/logs + - driver_data:/dc3-driver/dc3-driver-plcs7/dc3/data logging: *default-logging networks: dc3net: @@ -363,6 +370,7 @@ services: hostname: dc3-driver-virtual volumes: - logs:/dc3-driver/dc3-driver-virtual/dc3/logs + - driver_data:/dc3-driver/dc3-driver-virtual/dc3/data logging: *default-logging networks: dc3net: @@ -381,6 +389,7 @@ services: hostname: dc3-driver-bacnet-ip volumes: - logs:/dc3-driver/dc3-driver-bacnet-ip/dc3/logs + - driver_data:/dc3-driver/dc3-driver-bacnet-ip/dc3/data logging: *default-logging networks: dc3net: @@ -399,6 +408,7 @@ services: hostname: dc3-driver-fins volumes: - logs:/dc3-driver/dc3-driver-fins/dc3/logs + - driver_data:/dc3-driver/dc3-driver-fins/dc3/data logging: *default-logging networks: dc3net: @@ -417,6 +427,7 @@ services: hostname: dc3-driver-melsec volumes: - logs:/dc3-driver/dc3-driver-melsec/dc3/logs + - driver_data:/dc3-driver/dc3-driver-melsec/dc3/data logging: *default-logging networks: dc3net: @@ -435,6 +446,7 @@ services: hostname: dc3-driver-ethernet-ip volumes: - logs:/dc3-driver/dc3-driver-ethernet-ip/dc3/logs + - driver_data:/dc3-driver/dc3-driver-ethernet-ip/dc3/data logging: *default-logging networks: dc3net: @@ -453,6 +465,7 @@ services: hostname: dc3-driver-iec104 volumes: - logs:/dc3-driver/dc3-driver-iec104/dc3/logs + - driver_data:/dc3-driver/dc3-driver-iec104/dc3/data logging: *default-logging networks: dc3net: @@ -471,6 +484,7 @@ services: hostname: dc3-driver-sl651 volumes: - logs:/dc3-driver/dc3-driver-sl651/dc3/logs + - driver_data:/dc3-driver/dc3-driver-sl651/dc3/data logging: *default-logging networks: dc3net: @@ -489,6 +503,7 @@ services: hostname: dc3-driver-snmp volumes: - logs:/dc3-driver/dc3-driver-snmp/dc3/logs + - driver_data:/dc3-driver/dc3-driver-snmp/dc3/data logging: *default-logging networks: dc3net: @@ -507,6 +522,7 @@ services: hostname: dc3-driver-dlms volumes: - logs:/dc3-driver/dc3-driver-dlms/dc3/logs + - driver_data:/dc3-driver/dc3-driver-dlms/dc3/data logging: *default-logging networks: dc3net: @@ -525,6 +541,7 @@ services: hostname: dc3-driver-coap volumes: - logs:/dc3-driver/dc3-driver-coap/dc3/logs + - driver_data:/dc3-driver/dc3-driver-coap/dc3/data logging: *default-logging networks: dc3net: @@ -543,6 +560,7 @@ services: hostname: dc3-driver-lwm2m volumes: - logs:/dc3-driver/dc3-driver-lwm2m/dc3/logs + - driver_data:/dc3-driver/dc3-driver-lwm2m/dc3/data logging: *default-logging networks: dc3net: @@ -561,6 +579,7 @@ services: hostname: dc3-driver-http volumes: - logs:/dc3-driver/dc3-driver-http/dc3/logs + - driver_data:/dc3-driver/dc3-driver-http/dc3/data logging: *default-logging networks: dc3net: @@ -579,6 +598,7 @@ services: hostname: dc3-driver-serial volumes: - logs:/dc3-driver/dc3-driver-serial/dc3/logs + - driver_data:/dc3-driver/dc3-driver-serial/dc3/data logging: *default-logging networks: dc3net: @@ -597,6 +617,7 @@ services: hostname: dc3-driver-tcp-udp volumes: - logs:/dc3-driver/dc3-driver-tcp-udp/dc3/logs + - driver_data:/dc3-driver/dc3-driver-tcp-udp/dc3/data logging: *default-logging networks: dc3net: @@ -615,6 +636,7 @@ services: hostname: dc3-driver-mysql volumes: - logs:/dc3-driver/dc3-driver-mysql/dc3/logs + - driver_data:/dc3-driver/dc3-driver-mysql/dc3/data logging: *default-logging networks: dc3net: @@ -633,6 +655,7 @@ services: hostname: dc3-driver-postgresql volumes: - logs:/dc3-driver/dc3-driver-postgresql/dc3/logs + - driver_data:/dc3-driver/dc3-driver-postgresql/dc3/data logging: *default-logging networks: dc3net: @@ -651,6 +674,7 @@ services: hostname: dc3-driver-oracle volumes: - logs:/dc3-driver/dc3-driver-oracle/dc3/logs + - driver_data:/dc3-driver/dc3-driver-oracle/dc3/data logging: *default-logging networks: dc3net: @@ -669,6 +693,7 @@ services: hostname: dc3-driver-sqlserver volumes: - logs:/dc3-driver/dc3-driver-sqlserver/dc3/logs + - driver_data:/dc3-driver/dc3-driver-sqlserver/dc3/data logging: *default-logging networks: dc3net: @@ -677,6 +702,7 @@ services: volumes: logs: + driver_data: nginx: networks: diff --git a/docs/.nvmrc b/docs/.nvmrc new file mode 100644 index 000000000..2bd5a0a98 --- /dev/null +++ b/docs/.nvmrc @@ -0,0 +1 @@ +22 diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index c54409315..c2ad9ef73 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -18,6 +18,9 @@ import {defineConfig} from 'vitepress' import {Lang, t} from './i18n' import {transformHead} from './seo.mts' +import {resolveVersion} from './version.mts' + +const versionInfo = resolveVersion() // ── i18n via locales/{lang}.json ── // All user-facing strings are resolved through t(lang, key). @@ -33,17 +36,15 @@ type Pillar = { groups: ReadonlyArray } -// Pillar order: 架构 → 驱动 → AI → 基础 → 开发 (after nav 首页) +// Pillar order: 总览 → 架构 → 驱动 → AI → 基础 → 开发 → 运维 (after nav 首页) const PILLARS: ReadonlyArray = [ - { // ① - navKey: 'pillar.architecture', landing: 'architecture', - paths: ['architecture', 'modules', 'introduction'], - activeMatch: '^/(zh|en)/(architecture|modules|introduction)/', + { // ① 总览 + navKey: 'pillar.overview', landing: 'introduction', + paths: ['introduction', 'quickstart'], activeMatch: '^/(zh|en)/(introduction|quickstart)/', groups: [ - {key: '', items: [['architecture']]}, { - key: 'group.project-overview', - items: [['introduction'], ['introduction/concepts'], ['introduction/paths'], ['introduction/concepts/tenant']] + key: '', + items: [['introduction'], ['introduction/concepts'], ['introduction/paths']] }, { key: 'group.objects-data', @@ -51,18 +52,26 @@ const PILLARS: ReadonlyArray = [ }, { key: 'group.capabilities-boundaries', - items: [['introduction/concepts/command'], ['introduction/concepts/event'], ['introduction/concepts/attribute-config']] + items: [['introduction/concepts/command'], ['introduction/concepts/event'], ['introduction/concepts/attribute-config'], ['introduction/concepts/tenant']] }, + {key: 'group.quickstart', items: [['quickstart'], ['quickstart/environment'], ['quickstart/first-device']]}, + {key: 'group.appendix', items: [['introduction/glossary'], ['introduction/license']]} + ] + }, + { // ② 架构 + navKey: 'pillar.architecture', landing: 'architecture', + paths: ['architecture', 'modules'], activeMatch: '^/(zh|en)/(architecture|modules)/', + groups: [ + {key: '', items: [['architecture']]}, {key: 'group.services-collab', items: [['architecture/services'], ['architecture/facade-modes']]}, { key: 'group.pipelines-model', items: [['architecture/data-plane'], ['architecture/command-plane'], ['architecture/auth-rbac'], ['architecture/domain-model']] }, - {key: 'group.modules', items: [['architecture/modules'], ['modules']]}, - {key: 'group.appendix', items: [['introduction/glossary'], ['introduction/license']]} + {key: 'group.modules', items: [['architecture/modules'], ['modules']]} ] }, - { // ② + { // ③ 驱动 navKey: 'pillar.drivers', landing: 'drivers', paths: ['drivers', 'operation/device-onboarding'], activeMatch: '^/(zh|en)/(drivers/|operation/device-onboarding)', @@ -89,7 +98,7 @@ const PILLARS: ReadonlyArray = [ {key: 'group.appendix-drivers', items: [['drivers/matrix']]} ] }, - { // ③ + { // ④ AI navKey: 'pillar.ai', landing: 'ai', paths: ['ai'], @@ -99,7 +108,7 @@ const PILLARS: ReadonlyArray = [ {key: 'group.ai-integration', items: [['ai/agentic'], ['ai/mcp'], ['ai/spring-ai-deep-dive']]} ] }, - { // ④ + { // ⑤ 基础 navKey: 'pillar.foundations', landing: 'foundations', paths: ['foundations'], activeMatch: '^/(zh|en)/foundations/', groups: [ @@ -111,24 +120,29 @@ const PILLARS: ReadonlyArray = [ {key: 'group.security', items: [['foundations/security']]} ] }, - { // ⑤ + { // ⑥ 开发 navKey: 'pillar.develop', landing: 'development', - paths: ['development', 'frontend', 'automation', 'quickstart', 'operation', 'guide'], - activeMatch: '^/(zh|en)/(development|frontend|automation|quickstart|operation|guide)/', + paths: ['development', 'frontend', 'automation'], + activeMatch: '^/(zh|en)/(development|frontend|automation)/', groups: [ - {key: 'group.quickstart', items: [['quickstart'], ['quickstart/environment'], ['quickstart/first-device']]}, - { - key: 'group.deploy-ops', - items: [['guide/usage'], ['guide/observability'], ['guide/logging'], ['guide/troubleshooting']] - }, { key: 'group.development', items: [['development'], ['development/driver-authoring'], ['development/api-documentation'], ['development/technology-stack'], ['development/testing'], ['development/changelog']] }, {key: 'group.frontend', items: [['frontend'], ['frontend/test-debugging']]}, - {key: 'group.automation', items: [['automation/cli']]}, - {key: 'group.operations', items: [['operation'], ['operation/data-commands'], ['operation/alarms']]} + {key: 'group.automation', items: [['automation/cli']]} + ] + }, + { // ⑦ 运维 + navKey: 'pillar.operations', landing: 'operation', + paths: ['operation', 'guide'], activeMatch: '^/(zh|en)/(operation|guide)/', + groups: [ + {key: 'group.operations', items: [['operation'], ['operation/data-commands'], ['operation/alarms']]}, + { + key: 'group.deploy-ops', + items: [['guide/usage'], ['guide/observability'], ['guide/logging'], ['guide/troubleshooting']] + } ] } ] @@ -271,6 +285,11 @@ export default defineConfig({ // Preconnect to analytics origins so the first request isn't blocked on a cold DNS/TLS handshake ['link', {rel: 'preconnect', href: 'https://www.googletagmanager.com'}], ['link', {rel: 'preconnect', href: 'https://hm.baidu.com'}], + // Version meta — read at runtime by VersionSwitcher and VersionBanner + ['meta', {name: 'dc3-doc-version', content: versionInfo.version}], + ['meta', {name: 'dc3-doc-full-version', content: versionInfo.fullVersion}], + ['meta', {name: 'dc3-doc-is-latest', content: String(versionInfo.isLatest)}], + ['meta', {name: 'dc3-doc-latest-url', content: 'https://docs.dc3.site'}], // Google Analytics (GA4) ['script', {async: '', src: 'https://www.googletagmanager.com/gtag/js?id=G-0S37KX68Y3'}], ['script', {}, `window.dataLayer = window.dataLayer || []; diff --git a/docs/.vitepress/seo.mts b/docs/.vitepress/seo.mts index ef47cb464..5ed664cd4 100644 --- a/docs/.vitepress/seo.mts +++ b/docs/.vitepress/seo.mts @@ -20,6 +20,9 @@ import {execSync} from 'node:child_process' import {fileURLToPath} from 'node:url' import {resolve} from 'node:path' import type {HeadConfig, TransformContext} from 'vitepress' +import {resolveVersion} from './version.mts' + +const versionInfo = resolveVersion() const SITE_URL = 'https://docs.dc3.site' const DOCS_ROOT = fileURLToPath(new URL('..', import.meta.url)) @@ -392,7 +395,7 @@ export function transformHead(context: TransformContext): HeadConfig[] { return [ ['meta', {name: 'description', content: description}], - ['meta', {name: 'robots', content: 'index,follow,max-image-preview:large,max-snippet:-1,max-video-preview:-1'}], + ['meta', {name: 'robots', content: versionInfo.isLatest ? 'index,follow,max-image-preview:large,max-snippet:-1,max-video-preview:-1' : 'noindex,follow'}], ['meta', {name: 'author', content: 'IoT DC3 Contributors'}], ['link', {rel: 'canonical', href: canonicalUrl}], ['link', {rel: 'alternate', type: 'text/plain', href: `${SITE_URL}/llms.txt`, title: 'AI-readable site summary'}], diff --git a/docs/.vitepress/theme/components/VersionBanner.vue b/docs/.vitepress/theme/components/VersionBanner.vue new file mode 100644 index 000000000..0d5517e58 --- /dev/null +++ b/docs/.vitepress/theme/components/VersionBanner.vue @@ -0,0 +1,95 @@ + + + + + + + diff --git a/docs/.vitepress/theme/components/VersionSwitcher.vue b/docs/.vitepress/theme/components/VersionSwitcher.vue new file mode 100644 index 000000000..87706ee51 --- /dev/null +++ b/docs/.vitepress/theme/components/VersionSwitcher.vue @@ -0,0 +1,286 @@ + + + + + + + diff --git a/docs/.vitepress/theme/index.ts b/docs/.vitepress/theme/index.ts index 24dde53ee..e30833b90 100644 --- a/docs/.vitepress/theme/index.ts +++ b/docs/.vitepress/theme/index.ts @@ -30,6 +30,8 @@ import FourLayersDiagram from './components/FourLayersDiagram.vue' import HeroLogo from './components/HeroLogo.vue' import HeroParticles from './components/HeroParticles.vue' import HeroWaves from './components/HeroWaves.vue' +import VersionSwitcher from './components/VersionSwitcher.vue' +import VersionBanner from './components/VersionBanner.vue' // The language preference is written back to localStorage after internal navigation (including the top language switcher), so the head inline script can keep it on refresh. function persistLang(path: string) { @@ -46,8 +48,10 @@ const theme: Theme = { // Home hero: two background layers (HeroWaves wave dot-matrix at the bottom + full-width converging particles HeroParticles above it, home-hero-before), and the image area is the logo particle animation (home-hero-image) Layout() { return h(DefaultTheme.Layout, null, { + 'layout-top': () => h(VersionBanner), 'home-hero-before': () => [h(HeroWaves), h(HeroParticles)], - 'home-hero-image': () => h(HeroLogo) + 'home-hero-image': () => h(HeroLogo), + 'nav-bar-content-after': () => h(VersionSwitcher) }) }, diff --git a/docs/.vitepress/version.mts b/docs/.vitepress/version.mts new file mode 100644 index 000000000..6f472c320 --- /dev/null +++ b/docs/.vitepress/version.mts @@ -0,0 +1,73 @@ +/* + * Copyright 2016-present the IoT DC3 original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import {readFileSync} from 'node:fs' +import {fileURLToPath} from 'node:url' +import {resolve} from 'node:path' + +export interface VersionInfo { + /** Display version, e.g. "2026.5" (trimmed from POM patch version) */ + version: string + /** Full version including patch, e.g. "2026.5.22" */ + fullVersion: string + /** Whether this build is the latest (main/release branch). Archived builds set DC3_DOCS_IS_LATEST=false */ + isLatest: boolean +} + +const POM_PATH = resolve(fileURLToPath(new URL('../../pom.xml', import.meta.url))) + +/** + * Resolve the documentation version at build time. + * + * Resolution order: + * 1. DC3_DOCS_VERSION env var (set by CI for versioned/archived builds) + * 2. Parse from the parent pom.xml (e.g. "2026.5.22") + * 3. Fallback to "dev" + * + * The display version is trimmed to YYYY.M (patch releases share the same docs). + */ +export function resolveVersion(): VersionInfo { + const envVersion = process.env.DC3_DOCS_VERSION + + if (envVersion) { + const parts = envVersion.split('.') + const display = parts.length >= 2 ? parts.slice(0, 2).join('.') : envVersion + return { + version: display, + fullVersion: envVersion, + isLatest: process.env.DC3_DOCS_IS_LATEST !== 'false' + } + } + + try { + const pom = readFileSync(POM_PATH, 'utf8') + const match = pom.match(/(\d{4}\.\d+\.\d+)<\/version>/) + if (match) { + const full = match[1] + const parts = full.split('.') + return { + version: `${parts[0]}.${parts[1]}`, + fullVersion: full, + isLatest: true + } + } + } catch (_) { + // POM not found — standalone docs deployment + } + + return {version: 'dev', fullVersion: 'dev', isLatest: true} +} diff --git a/docs/package.json b/docs/package.json index c9487c660..8cd797d0a 100644 --- a/docs/package.json +++ b/docs/package.json @@ -19,5 +19,5 @@ "medium-zoom": "^1.1.0", "vitepress": "^1.6.4" }, - "packageManager": "pnpm@11.15.0" + "packageManager": "pnpm@11.17.0" } diff --git a/docs/public/versions.json b/docs/public/versions.json new file mode 100644 index 000000000..2819cc11d --- /dev/null +++ b/docs/public/versions.json @@ -0,0 +1,10 @@ +{ + "latest": "2026.5", + "versions": [ + { + "version": "2026.5", + "url": "https://docs.dc3.site", + "latest": true + } + ] +} diff --git a/pom.xml b/pom.xml index 9ce63f20f..3ea64c9ea 100644 --- a/pom.xml +++ b/pom.xml @@ -91,6 +91,9 @@ 3.5.16 4.5.0 + + + 3.53.2.0 3.1.4 @@ -121,6 +124,7 @@ 0.13.0 1.84 1.16.5 + 1.45.0 2.0.4 @@ -285,6 +289,19 @@ pom import + + + io.opentelemetry + opentelemetry-bom + ${opentelemetry.version} + pom + import + + + io.opentelemetry + opentelemetry-semconv + ${opentelemetry.version}-alpha + com.squareup.okhttp3 okhttp-bom @@ -293,6 +310,13 @@ import + + + org.xerial + sqlite-jdbc + ${sqlite-jdbc.version} + + org.assertj