opensource/iot-dc3
3
0
Fork 0
mirror of https://gitee.com/pnoker/iot-dc3.git synced 2026-09-01 15:33:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): stabilize infrastructure harness

Browse Source
This commit is contained in:
pnoker
2026-05-14 21:29:08 +08:00
parent ddda8c08f7
commit 9123b480f4
14 changed files with 198 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e.yml
+1 -1
View File
@@ -36,7 +36,7 @@ jobs:
- name: Run end-to-end suite
env:
TESTCONTAINERS_REUSE_ENABLE: "true"
DC3_E2E: "true"
run: |
set -euo pipefail
make test-e2e
Makefile
+1 -1
View File
@@ -116,7 +116,7 @@ test-it:
$(MVN) -B -Dmaven.test.skip=false -Dskip.unit.tests=true verify
test-e2e:
$(MVN) -B -Dmaven.test.skip=false -pl dc3-e2e -am -Pe2e verify
DC3_E2E=true $(MVN) -B -Dmaven.test.skip=false -pl dc3-e2e -am -Pe2e verify
coverage:
$(MVN) -B -Dmaven.test.skip=false -pl dc3-coverage -am verify
dc3-common/dc3-common-test/src/main/java/io/github/pnoker/test/containers/MqttContainer.java
+4 -1
View File
@@ -21,6 +21,7 @@ import org.springframework.test.context.DynamicPropertyRegistry;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.TestcontainersConfiguration;
/**
* Shared Eclipse Mosquitto MQTT broker container for driver and common-mqtt tests.
@@ -33,6 +34,8 @@ public final class MqttContainer {
private static final DockerImageName IMAGE = DockerImageName.parse("eclipse-mosquitto:2.0");
private static final boolean REUSE_ENABLED = TestcontainersConfiguration.getInstance().environmentSupportsReuse();
private static final String CONFIG = """
listener 1883
allow_anonymous true
@@ -45,7 +48,7 @@ public final class MqttContainer {
org.testcontainers.images.builder.Transferable.of(CONFIG),
"/mosquitto/config/mosquitto.conf")
.waitingFor(Wait.forLogMessage(".*mosquitto version.*\\n", 1))
.withReuse(true);
.withReuse(REUSE_ENABLED);
static {
INSTANCE.start();
dc3-common/dc3-common-test/src/main/java/io/github/pnoker/test/containers/PgTimescaleContainer.java
+4 -1
View File
@@ -20,6 +20,7 @@ package io.github.pnoker.test.containers;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.TestcontainersConfiguration;
/**
* Shared PostgreSQL + TimescaleDB container that mirrors the production image
@@ -35,12 +36,14 @@ public final class PgTimescaleContainer {
.parse("timescale/timescaledb-ha:pg18")
.asCompatibleSubstituteFor("postgres");
private static final boolean REUSE_ENABLED = TestcontainersConfiguration.getInstance().environmentSupportsReuse();
@SuppressWarnings("resource")
private static final PostgreSQLContainer<?> INSTANCE = new PostgreSQLContainer<>(IMAGE)
.withDatabaseName("dc3")
.withUsername("dc3")
.withPassword("dc3")
.withReuse(true);
.withReuse(REUSE_ENABLED);
static {
INSTANCE.start();
dc3-common/dc3-common-test/src/main/java/io/github/pnoker/test/containers/RabbitContainer.java
+4 -1
View File
@@ -20,6 +20,7 @@ package io.github.pnoker.test.containers;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.testcontainers.containers.RabbitMQContainer;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.TestcontainersConfiguration;
/**
* Shared RabbitMQ container with management plugin enabled, matching the
@@ -29,9 +30,11 @@ public final class RabbitContainer {
private static final DockerImageName IMAGE = DockerImageName.parse("rabbitmq:3.13-management");
private static final boolean REUSE_ENABLED = TestcontainersConfiguration.getInstance().environmentSupportsReuse();
@SuppressWarnings("resource")
private static final RabbitMQContainer INSTANCE = new RabbitMQContainer(IMAGE)
.withReuse(true);
.withReuse(REUSE_ENABLED);
static {
INSTANCE.start();
dc3-common/pom.xml
+1 -1
View File
@@ -76,7 +76,7 @@
<!-- Test toolchain (mirrored from iot-dc3 root pom; needed because
dc3-common inherits from the external dc3-parent and therefore
cannot read the root reactor's dependencyManagement). -->
<testcontainers.version>1.20.4</testcontainers.version>
<testcontainers.version>1.21.4</testcontainers.version>
<junit.bom.version>5.11.3</junit.bom.version>
<assertj.version>3.26.3</assertj.version>
<awaitility.version>4.2.2</awaitility.version>
dc3-e2e/pom.xml
+32 -4
View File
@@ -29,19 +29,42 @@
<packaging>jar</packaging>
<description>
End-to-end test harness that boots the iot-dc3 platform via Docker
Compose (driven by Testcontainers) and exercises the public REST and
gRPC surfaces with rest-assured. Lives outside the production reactor
so its dependencies do not leak into deployable artifacts.
Infrastructure-backed end-to-end test harness for iot-dc3. It currently
boots production-aligned PostgreSQL/TimescaleDB and RabbitMQ containers
and pins the HTTP assertion tooling that future platform user-flow tests
will use. Lives outside the production reactor so its dependencies do
not leak into deployable artifacts.
</description>
<properties>
<maven.deploy.skip>true</maven.deploy.skip>
<rest-assured.groovy.version>4.0.22</rest-assured.groovy.version>
<!-- E2E is only invoked by an explicit profile or workflow. -->
<skip.unit.tests>true</skip.unit.tests>
<skip.integration.tests>true</skip.integration.tests>
</properties>
<dependencyManagement>
<dependencies>
<!-- Rest Assured 5.5.0 imports Groovy [4.0,5.0); override Spring Boot's Groovy 5 management. -->
<dependency>
<groupId>org.apache.groovy</groupId>
<artifactId>groovy</artifactId>
<version>${rest-assured.groovy.version}</version>
</dependency>
<dependency>
<groupId>org.apache.groovy</groupId>
<artifactId>groovy-json</artifactId>
<version>${rest-assured.groovy.version}</version>
</dependency>
<dependency>
<groupId>org.apache.groovy</groupId>
<artifactId>groovy-xml</artifactId>
<version>${rest-assured.groovy.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>io.github.pnoker</groupId>
@@ -87,6 +110,11 @@
<artifactId>postgresql</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
dc3-e2e/src/test/java/io/github/pnoker/e2e/PostgresHypertableIT.java
+86 -8
View File
@@ -36,9 +36,10 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* Validates the assumptions production point-value persistence relies on:
* - the dc3 schema can mount the timescaledb extension
* - the dc3 schema can mount the pgvector extension
* - a hypertable on (time, device_id, point_id) accepts batched inserts and
* retains them across queries
* - a continuous-aggregate-style time bucket query returns expected aggregates
* - a continuous-aggregate-style time_bucket query returns expected aggregates
*
* Disabled by default; opt in with {@code DC3_E2E=true}.
*/
@@ -63,7 +64,7 @@ class PostgresHypertableIT extends BaseE2eIT {
ddl.execute("SELECT create_hypertable('dc3_point_value_e2e', 'time')");
}
Instant base = Instant.now().truncatedTo(ChronoUnit.MILLIS).minusSeconds(60);
Instant base = Instant.now().truncatedTo(ChronoUnit.MINUTES).minusSeconds(60);
try (PreparedStatement ps = conn.prepareStatement(
"INSERT INTO dc3_point_value_e2e(time, device_id, point_id, raw_value, cal_value)"
+ " VALUES (?, ?, ?, ?, ?)")) {
@@ -80,13 +81,15 @@ class PostgresHypertableIT extends BaseE2eIT {
try (Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(
"SELECT count(*), avg(cal_value), min(cal_value), max(cal_value)"
+ " FROM dc3_point_value_e2e WHERE device_id = 1 AND point_id = 100")) {
"SELECT time_bucket('1 minute', time), count(*), avg(cal_value), min(cal_value),"
+ " max(cal_value) FROM dc3_point_value_e2e"
+ " WHERE device_id = 1 AND point_id = 100 GROUP BY 1")) {
assertThat(rs.next()).isTrue();
assertThat(rs.getInt(1)).isEqualTo(10);
assertThat(rs.getDouble(2)).isEqualTo(24.5, org.assertj.core.data.Offset.offset(0.0001));
assertThat(rs.getDouble(3)).isEqualTo(20.0);
assertThat(rs.getDouble(4)).isEqualTo(29.0);
assertThat(rs.getInt(2)).isEqualTo(10);
assertThat(rs.getDouble(3)).isEqualTo(24.5, org.assertj.core.data.Offset.offset(0.0001));
assertThat(rs.getDouble(4)).isEqualTo(20.0);
assertThat(rs.getDouble(5)).isEqualTo(29.0);
assertThat(rs.next()).isFalse();
}
try (Statement stmt = conn.createStatement();
@@ -99,6 +102,81 @@ class PostgresHypertableIT extends BaseE2eIT {
}
}
@Test
void pgvectorStoresIndexesAndRanksEmbeddings() throws Exception {
try (Connection conn = DriverManager.getConnection(
E2eStack.postgresJdbcUrl(), E2eStack.postgresUsername(), E2eStack.postgresPassword())) {
try (Statement ddl = conn.createStatement()) {
ddl.execute("CREATE EXTENSION IF NOT EXISTS timescaledb");
ddl.execute("CREATE EXTENSION IF NOT EXISTS vector");
ddl.execute("DROP TABLE IF EXISTS dc3_embedding_e2e");
ddl.execute("""
CREATE TABLE dc3_embedding_e2e (
id BIGINT PRIMARY KEY,
name TEXT NOT NULL,
embedding VECTOR(3) NOT NULL
)""");
ddl.execute("""
CREATE INDEX dc3_embedding_e2e_hnsw_idx
ON dc3_embedding_e2e USING hnsw (embedding vector_l2_ops)
""");
}
try (PreparedStatement ps = conn.prepareStatement(
"INSERT INTO dc3_embedding_e2e(id, name, embedding) VALUES (?, ?, ?::vector)")) {
ps.setLong(1, 1L);
ps.setString(2, "temperature");
ps.setString(3, "[0.10,0.20,0.30]");
ps.addBatch();
ps.setLong(1, 2L);
ps.setString(2, "humidity");
ps.setString(3, "[0.90,0.10,0.10]");
ps.addBatch();
ps.setLong(1, 3L);
ps.setString(2, "pressure");
ps.setString(3, "[0.20,0.80,0.90]");
ps.addBatch();
ps.executeBatch();
}
try (Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(
"SELECT count(*) FROM pg_extension WHERE extname IN ('timescaledb', 'vector')")) {
assertThat(rs.next()).isTrue();
assertThat(rs.getInt(1)).isEqualTo(2);
}
try (Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(
"SELECT vector_dims(embedding) FROM dc3_embedding_e2e WHERE id = 1")) {
assertThat(rs.next()).isTrue();
assertThat(rs.getInt(1)).isEqualTo(3);
}
try (Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(
"SELECT name, embedding <-> '[0.11,0.19,0.31]'::vector AS distance"
+ " FROM dc3_embedding_e2e ORDER BY embedding <-> '[0.11,0.19,0.31]'::vector"
+ " LIMIT 1")) {
assertThat(rs.next()).isTrue();
assertThat(rs.getString(1)).isEqualTo("temperature");
assertThat(rs.getDouble(2)).isLessThan(0.03);
}
try (Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(
"SELECT indexdef FROM pg_indexes"
+ " WHERE tablename = 'dc3_embedding_e2e'"
+ " AND indexname = 'dc3_embedding_e2e_hnsw_idx'")) {
assertThat(rs.next()).isTrue();
assertThat(rs.getString(1)).contains("USING hnsw", "vector_l2_ops");
}
}
}
@Test
void hypertableEnforcesNotNullOnTimeColumn() throws Exception {
try (Connection conn = DriverManager.getConnection(
dc3-e2e/src/test/java/io/github/pnoker/e2e/RabbitDeliveryIT.java
+43 -22
View File
@@ -22,17 +22,20 @@ import com.rabbitmq.client.BuiltinExchangeType;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
import com.rabbitmq.client.GetResponse;
import io.github.pnoker.e2e.harness.BaseE2eIT;
import io.github.pnoker.e2e.harness.E2eStack;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.awaitility.Awaitility.await;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -97,46 +100,64 @@ class RabbitDeliveryIT extends BaseE2eIT {
channel.basicPublish(exchange, "k", null, "first".getBytes(StandardCharsets.UTF_8));
channel.basicPublish(exchange, "k", null, "second".getBytes(StandardCharsets.UTF_8));
// Wait until both messages are persisted (queueDeclarePassive count==2).
for (int i = 0; i < 20 && channel.messageCount(queue) < 2; i++) {
Thread.sleep(50);
await().atMost(Duration.ofSeconds(2))
.pollInterval(Duration.ofMillis(50))
.untilAsserted(() -> assertThat(channel.messageCount(queue)).isEqualTo(2));
try (Channel delivery = conn.createChannel()) {
GetResponse first = delivery.basicGet(queue, false);
assertThat(first).isNotNull();
assertThat(new String(first.getBody(), StandardCharsets.UTF_8)).isEqualTo("first");
GetResponse second = delivery.basicGet(queue, false);
assertThat(second).isNotNull();
assertThat(new String(second.getBody(), StandardCharsets.UTF_8)).isEqualTo("second");
delivery.basicAck(second.getEnvelope().getDeliveryTag(), false);
}
channel.basicGet(queue, false); // first delivery, no ack -> requeued on close-channel
channel.basicAck(channel.basicGet(queue, false).getEnvelope().getDeliveryTag(), false);
await().atMost(Duration.ofSeconds(2))
.pollInterval(Duration.ofMillis(50))
.untilAsserted(() -> assertThat(channel.messageCount(queue)).isEqualTo(1));
// Drain remaining: requeued first + nothing else.
for (int i = 0; i < 20 && channel.messageCount(queue) == 0; i++) {
Thread.sleep(50);
}
assertThat(channel.messageCount(queue)).isEqualTo(1);
GetResponse requeued = channel.basicGet(queue, false);
assertThat(requeued).isNotNull();
assertThat(new String(requeued.getBody(), StandardCharsets.UTF_8)).isEqualTo("first");
channel.basicAck(requeued.getEnvelope().getDeliveryTag(), false);
assertThat(channel.messageCount(queue)).isZero();
}
}
@Test
void waitForConfirmsTimesOutWhenBrokerNeverAcks() throws Exception {
void waitForConfirmsWithoutPendingPublishesCompletesImmediately() throws Exception {
try (Connection conn = newConnection();
Channel channel = conn.createChannel()) {
channel.confirmSelect();
// A fresh channel without any publish has no outstanding confirms; the
// expected behavior is that waitForConfirms returns true immediately. To pin
// the timeout-throwing branch, drive a publish that never gets confirmed by
// closing the channel mid-flight on a separate connection. The simpler
// contract worth pinning here: no-publish wait completes synchronously.
assertThat(channel.waitForConfirms(50L)).isTrue();
}
}
@Test
void exclusiveQueueRejectsSecondConsumerOnSameConnection() throws Exception {
try (Connection conn = newConnection();
Channel first = conn.createChannel();
Channel second = conn.createChannel()) {
void exclusiveQueueRejectsConsumerOnDifferentConnection() throws Exception {
try (Connection owner = newConnection();
Connection other = newConnection();
Channel first = owner.createChannel()) {
String queue = "dc3.e2e.exclusive." + UUID.randomUUID();
first.queueDeclare(queue, false, true, false, null);
first.basicConsume(queue, true, "c1", (tag, delivery) -> {}, tag -> {});
assertThatThrownBy(() -> second.basicConsume(queue, true, "c2", (tag, delivery) -> {}, tag -> {}))
.isInstanceOf(java.io.IOException.class);
Channel second = other.createChannel();
try {
assertThatThrownBy(() -> second.basicConsume(queue, true, "c2", (tag, delivery) -> {}, tag -> {}))
.isInstanceOf(java.io.IOException.class)
.hasRootCauseInstanceOf(com.rabbitmq.client.ShutdownSignalException.class)
.hasStackTraceContaining("RESOURCE_LOCKED");
assertThat(second.isOpen()).isFalse();
} finally {
if (second.isOpen()) {
second.close();
}
}
}
}
dc3-e2e/src/test/java/io/github/pnoker/e2e/RestAssuredHarnessIT.java
+4
View File
@@ -70,6 +70,7 @@ class RestAssuredHarnessIT extends BaseE2eIT {
.baseUri("http://127.0.0.1")
.port(port)
.accept(ContentType.JSON)
.contentType(ContentType.JSON)
.when()
.get("/v3/probe")
.then()
@@ -81,6 +82,7 @@ class RestAssuredHarnessIT extends BaseE2eIT {
given()
.baseUri("http://127.0.0.1")
.port(port)
.contentType(ContentType.JSON)
.when()
.get("/v3/error")
.then()
@@ -146,6 +148,7 @@ class RestAssuredHarnessIT extends BaseE2eIT {
.config(config)
.baseUri("http://127.0.0.1")
.port(port)
.contentType(ContentType.JSON)
.when()
.get("/v3/no-content-type")
.then()
@@ -168,6 +171,7 @@ class RestAssuredHarnessIT extends BaseE2eIT {
String body = given()
.baseUri("http://127.0.0.1")
.port(port)
.contentType(ContentType.JSON)
.when()
.get("/v3/raw")
.then()
dc3-e2e/src/test/java/io/github/pnoker/e2e/harness/BaseE2eIT.java
+2 -2
View File
@@ -30,8 +30,8 @@ import org.junit.jupiter.api.Tag;
* {@code @EnabledIfEnvironmentVariable(named = "DC3_E2E", matches = "(?i)true|1|yes|on")}
* directly on the class. JUnit 5 evaluates the condition on the declared class, so
* keeping it on the abstract base would silently leave subclasses enabled when
* {@code DC3_E2E} is unset. The {@code e2e.yml} workflow exports the env var before
* running this suite.
* {@code DC3_E2E} is unset. The {@code make test-e2e} target and {@code e2e.yml}
* workflow export the env var before running this suite.
*/
@Tag("e2e")
public abstract class BaseE2eIT {
dc3-e2e/src/test/java/io/github/pnoker/e2e/harness/E2eStack.java
+9 -4
View File
@@ -23,6 +23,7 @@ import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.containers.RabbitMQContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.TestcontainersConfiguration;
import java.time.Duration;
@@ -59,14 +60,14 @@ public final class E2eStack {
.withDatabaseName("dc3")
.withUsername("dc3")
.withPassword("dc3")
.withReuse(true)
.withReuse(reuseEnabled())
.withStartupTimeout(Duration.ofMinutes(2));
@SuppressWarnings("resource")
private static final RabbitMQContainer RABBIT = new RabbitMQContainer(RABBIT_IMAGE)
.withNetwork(NETWORK)
.withNetworkAliases("dc3-rabbitmq")
.withReuse(true)
.withReuse(reuseEnabled())
.waitingFor(Wait.forLogMessage(".*Server startup complete.*", 1))
.withStartupTimeout(Duration.ofMinutes(2));
@@ -77,8 +78,8 @@ public final class E2eStack {
/**
* Idempotent boot. The first invocation starts the containers; subsequent calls
* are no-ops that return the already-running stack. Container reuse is enabled
* via {@code .testcontainers.properties} to keep cross-run startup fast.
* are no-ops that return the already-running stack. Cross-run container reuse
* is enabled only when the local Testcontainers environment supports it.
*/
public static synchronized void start() {
if (started) {
@@ -141,4 +142,8 @@ public final class E2eStack {
+ "@BeforeAll hook or extend BaseE2eIT.");
}
}
private static boolean reuseEnabled() {
return TestcontainersConfiguration.getInstance().environmentSupportsReuse();
}
}
dc3/doc/TESTING.md
+6 -8
View File
@@ -11,7 +11,7 @@ repository. It complements `AGENTS.md` (engineering rules), `LOGGING.md`
| Unit | Fast, isolated business logic checks | JUnit 5 + Mockito + AssertJ | ~70% |
| Slice | Spring slice tests for controllers, JSON, persistence | `@WebFluxTest`, `@JsonTest`, `@MybatisPlusTest` | ~25% |
| Integration | Real infrastructure via Testcontainers, gRPC in-process | PG18+TimescaleDB, RabbitMQ, MQTT, gRPC InProcess | included in slice/integration share |
| End-to-end | Full reactor against a docker-compose stack | `dc3-e2e` with rest-assured | ~5% |
| End-to-end | Infrastructure-backed harness now; full user flows next | `dc3-e2e` with Testcontainers + rest-assured | ~5% |
Aggregate coverage gates (read from `dc3-coverage`):
@@ -85,8 +85,8 @@ The `Validation Checklist` in `AGENTS.md` enumerates the full matrix.
## 5. Testcontainers Conventions
Containers expose a single shared instance per JVM, started lazily on
class-load and reused across modules via `withReuse(true)`. Pin images
to the production-aligned tag:
class-load. Cross-run reuse is opt-in and only enabled when the local
Testcontainers environment supports it. Pin images to the production-aligned tag:
| Container | Image | Notes |
|------------------------|---------------------------------|--------------------------------------------------------------------------------|
@@ -99,14 +99,12 @@ The shared wrappers live in
and inject themselves into Spring environments via
`@DynamicPropertySource` callbacks.
For local reuse, contributors can opt-in by exporting:
For local reuse, contributors can opt in by creating `~/.testcontainers.properties`:
```bash
export TESTCONTAINERS_REUSE_ENABLE=true
testcontainers.reuse.enable=true
```
CI sets this automatically inside the integration job.
## 6. Test Data Strategy
- Small fixed payloads belong in `src/test/resources/fixtures/` so they
@@ -142,7 +140,7 @@ dependencies.
make test # Unit phase
make test-it # Integration phase (requires Docker)
make coverage # Aggregate jacoco report (target/site/jacoco-aggregate)
make test-e2e # Full docker-compose end-to-end suite
make test-e2e # E2E harness; exports DC3_E2E=true and requires Docker
mvn -B -pl <module> test # Targeted unit run for a single module
```
pom.xml
+1 -1
View File
@@ -77,7 +77,7 @@
<spring-ai.version>2.0.0-M5</spring-ai.version>
<!-- Test toolchain -->
<testcontainers.version>1.20.4</testcontainers.version>
<testcontainers.version>1.21.4</testcontainers.version>
<junit.bom.version>5.11.3</junit.bom.version>
<assertj.version>3.26.3</assertj.version>
<awaitility.version>4.2.2</awaitility.version>