Benchmark: Pact 4.0 vs. Spring Cloud Contract vs. Postman 11 for API Contract Testing

// Pact 4.0 Consumer Test (implements Pact Spec 4.0)
// Dependencies: pact-jvm-consumer 4.6.0, JUnit 5, Spring Boot 3.2.0
import au.com.dius.pact.consumer.MockServer;
import au.com.dius.pact.consumer.dsl.PactDslJsonBody;
import au.com.dius.pact.consumer.dsl.PactDslWithProvider;
import au.com.dius.pact.consumer.junit5.PactConsumerTestExt;
import au.com.dius.pact.consumer.junit5.PactTestFor;
import au.com.dius.pact.core.model.PactSpecVersion;
import au.com.dius.pact.core.model.V4Pact;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.web.client.TestRestTemplate;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.ActiveProfiles;

import java.util.HashMap;
import java.util.Map;

import static org.assertj.core.api.Assertions.assertThat;
import static au.com.dius.pact.consumer.dsl.LambdaDsl.newJsonBody;

@ExtendWith(PactConsumerTestExt.class)
@PactTestFor(providerName = "user-service", pactVersion = PactSpecVersion.V4)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles("test")
public class UserApiConsumerPactTest {

    @Autowired
    private TestRestTemplate restTemplate;

    // Define the Pact contract for GET /users/{id}
    @Pact(provider = "user-service", consumer = "order-service")
    public V4Pact createUserPact(PactDslWithProvider builder) {
        Map<String, String> headers = new HashMap<>();
        headers.put("Content-Type", "application/json");

        return builder
            .given("user with ID 123 exists")
            .uponReceiving("a request for user 123")
            .path("/users/123")
            .method("GET")
            .headers(headers)
            .willRespondWith()
            .status(200)
            .headers(headers)
            .body(newJsonBody(body -> {
                body.stringType("id", "123");
                body.stringType("name", "Alice Smith");
                body.numberType("age", 32);
                body.stringType("email", "alice@example.com");
            }).build())
            .toPact(V4Pact.class);
    }

    @Test
    @PactTestFor(pactMethod = "createUserPact")
    void testGetUserById(MockServer mockServer) throws Exception {
        // Arrange: point RestTemplate to Pact mock server
        String baseUrl = mockServer.getUrl();
        restTemplate.getRestTemplate().setUriTemplateHandler(new DefaultUriBuilderFactory(baseUrl));

        // Act: call the user API
        ResponseEntity<User> response = restTemplate.getForEntity("/users/123", User.class);

        // Assert: verify response matches contract
        assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
        assertThat(response.getBody()).isNotNull();
        assertThat(response.getBody().getId()).isEqualTo("123");
        assertThat(response.getBody().getName()).isEqualTo("Alice Smith");
        assertThat(response.getBody().getAge()).isEqualTo(32);
        assertThat(response.getBody().getEmail()).isEqualTo("alice@example.com");
    }

    // Error handling: test 404 case
    @Pact(provider = "user-service", consumer = "order-service")
    public V4Pact createNotFoundPact(PactDslWithProvider builder) {
        Map<String, String> headers = new HashMap<>();
        headers.put("Content-Type", "application/json");

        return builder
            .given("user with ID 999 does not exist")
            .uponReceiving("a request for user 999")
            .path("/users/999")
            .method("GET")
            .headers(headers)
            .willRespondWith()
            .status(404)
            .headers(headers)
            .body(newJsonBody(body -> {
                body.stringType("error", "User not found");
                body.numberType("code", 404);
            }).build())
            .toPact(V4Pact.class);
    }

    @Test
    @PactTestFor(pactMethod = "createNotFoundPact")
    void testGetUserNotFound(MockServer mockServer) {
        String baseUrl = mockServer.getUrl();
        restTemplate.getRestTemplate().setUriTemplateHandler(new DefaultUriBuilderFactory(baseUrl));

        ResponseEntity<ErrorResponse> response = restTemplate.getForEntity("/users/999", ErrorResponse.class);

        assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NOT_FOUND);
        assertThat(response.getBody().getError()).isEqualTo("User not found");
        assertThat(response.getBody().getCode()).isEqualTo(404);
    }

    // Inner DTO classes for mapping
    public static class User {
        private String id;
        private String name;
        private int age;
        private String email;

        // Getters and setters
        public String getId() { return id; }
        public void setId(String id) { this.id = id; }
        public String getName() { return name; }
        public void setName(String name) { this.name = name; }
        public int getAge() { return age; }
        public void setAge(int age) { this.age = age; }
        public String getEmail() { return email; }
        public void setEmail(String email) { this.email = email; }
    }

    public static class ErrorResponse {
        private String error;
        private int code;

        public String getError() { return error; }
        public void setError(String error) { this.error = error; }
        public int getCode() { return code; }
        public void setCode(int code) { this.code = code; }
    }
}

// Spring Cloud Contract 4.1.0 Producer Test
// Dependencies: spring-cloud-starter-contract-verifier 4.1.0, JUnit 5, Spring Boot 3.2.0
package com.example.userservice.contracts;

import com.example.userservice.model.User;
import com.example.userservice.repository.UserRepository;
import io.restassured.module.mockmvc.RestAssuredMockMvc;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit.jupiter.SpringExtension;

import java.util.Optional;

import static org.mockito.Mockito.when;

@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = UserApplication.class)
@ActiveProfiles("test")
public class UserContractBaseTest {

    @MockBean
    private UserRepository userRepository;

    @BeforeEach
    void setup() {
        // Initialize RestAssured with the user controller
        RestAssuredMockMvc.standaloneSetup(new UserController(userRepository));

        // Mock repository responses for contract states
        User existingUser = new User("123", "Alice Smith", 32, "alice@example.com");
        when(userRepository.findById("123")).thenReturn(Optional.of(existingUser));
        when(userRepository.findById("999")).thenReturn(Optional.empty());
    }
}

// Groovy Contract Definition (saved as src/test/resources/contracts/user_service/get_user.groovy)
/*
import org.springframework.cloud.contract.spec.Contract

Contract.make {
    description "Contract for GET /users/{id}"
    request {
        method GET()
        urlPath("/users/123")
        headers {
            header("Content-Type", "application/json")
        }
    }
    response {
        status 200
        headers {
            header("Content-Type", "application/json")
        }
        body(
            id: "123",
            name: "Alice Smith",
            age: 32,
            email: "alice@example.com"
        )
    }
    input {
        triggeredBy("existingUserExists()")
    }
}

Contract.make {
    description "Contract for GET /users/{id} not found"
    request {
        method GET()
        urlPath("/users/999")
        headers {
            header("Content-Type", "application/json")
        }
    }
    response {
        status 404
        headers {
            header("Content-Type", "application/json")
        }
        body(
            error: "User not found",
            code: 404
        )
    }
    input {
        triggeredBy("nonExistingUserExists()")
    }
}
*/

// Generated Test (auto-generated by Spring Cloud Contract Verifier)
package com.example.userservice.contracts;

import io.restassured.module.mockmvc.RestAssuredMockMvc;
import org.junit.jupiter.api.Test;

import static io.restassured.module.mockmvc.RestAssuredMockMvc.given;
import static org.hamcrest.CoreMatchers.equalTo;

public class UserContractTest extends UserContractBaseTest {

    @Test
    public void validate_get_user_123() {
        // Given: user 123 exists (set up in base test)
        // When: GET /users/123 is called
        // Then: 200 OK with correct body
        given()
            .header("Content-Type", "application/json")
        .when()
            .get("/users/123")
        .then()
            .statusCode(200)
            .body("id", equalTo("123"))
            .body("name", equalTo("Alice Smith"))
            .body("age", equalTo(32))
            .body("email", equalTo("alice@example.com"));
    }

    @Test
    public void validate_get_user_999() {
        // Given: user 999 does not exist (set up in base test)
        // When: GET /users/999 is called
        // Then: 404 Not Found with error body
        given()
            .header("Content-Type", "application/json")
        .when()
            .get("/users/999")
        .then()
            .statusCode(404)
            .body("error", equalTo("User not found"))
            .body("code", equalTo(404));
    }

    // Error handling: test invalid ID format
    @Test
    public void validate_get_user_invalid_id() {
        given()
            .header("Content-Type", "application/json")
        .when()
            .get("/users/abc")
        .then()
            .statusCode(400)
            .body("error", equalTo("Invalid user ID format"));
    }
}

// Postman 11.0.12 Contract Test Runner with Newman
// Dependencies: newman 6.0.0, postman-collection 4.1.0, Node.js 20.0.0
const newman = require('newman');
const fs = require('fs');
const path = require('path');

// Load Postman collection and environment
const collectionPath = path.join(__dirname, 'user-api-contract.postman_collection.json');
const environmentPath = path.join(__dirname, 'user-api-contract.postman_environment.json');

// Verify files exist before running
if (!fs.existsSync(collectionPath)) {
    throw new Error(`Collection file not found at ${collectionPath}`);
}
if (!fs.existsSync(environmentPath)) {
    throw new Error(`Environment file not found at ${environmentPath}`);
}

// Define contract test suite
const contractTests = [
    {
        name: 'GET /users/123 - 200 OK',
        request: {
            method: 'GET',
            url: '{{baseUrl}}/users/123',
            header: [
                { key: 'Content-Type', value: 'application/json' }
            ]
        },
        expectedResponse: {
            status: 200,
            body: {
                id: '123',
                name: 'Alice Smith',
                age: 32,
                email: 'alice@example.com'
            }
        }
    },
    {
        name: 'GET /users/999 - 404 Not Found',
        request: {
            method: 'GET',
            url: '{{baseUrl}}/users/999',
            header: [
                { key: 'Content-Type', value: 'application/json' }
            ]
        },
        expectedResponse: {
            status: 404,
            body: {
                error: 'User not found',
                code: 404
            }
        }
    },
    {
        name: 'GET /users/abc - 400 Bad Request',
        request: {
            method: 'GET',
            url: '{{baseUrl}}/users/abc',
            header: [
                { key: 'Content-Type', value: 'application/json' }
            ]
        },
        expectedResponse: {
            status: 400,
            body: {
                error: 'Invalid user ID format'
            }
        }
    }
];

// Run contract tests using Newman
newman.run({
    collection: collectionPath,
    environment: environmentPath,
    reporters: ['cli', 'json'],
    reporterOptions: {
        json: { export: 'contract-test-results.json' }
    },
    iterationCount: 1
}, (err, summary) => {
    if (err) {
        console.error('Newman run failed:', err);
        process.exit(1);
    }

    // Validate test results against contract definitions
    let failedTests = 0;
    const testResults = summary.run.executions;

    testResults.forEach((execution, index) => {
        const testCase = contractTests[index];
        if (!testCase) {
            console.warn(`No contract test defined for execution ${index}`);
            return;
        }

        // Check status code
        if (execution.response.status !== testCase.expectedResponse.status) {
            console.error(`FAILED: ${testCase.name} - Expected status ${testCase.expectedResponse.status}, got ${execution.response.status}`);
            failedTests++;
        }

        // Check response body
        const responseBody = JSON.parse(execution.response.stream.toString());
        Object.keys(testCase.expectedResponse.body).forEach(key => {
            if (responseBody[key] !== testCase.expectedResponse.body[key]) {
                console.error(`FAILED: ${testCase.name} - Expected ${key} to be ${testCase.expectedResponse.body[key]}, got ${responseBody[key]}`);
                failedTests++;
            }
        });

        // Check for Postman test failures
        const postmanFailures = execution.assertions.filter(a => a.error);
        if (postmanFailures.length > 0) {
            console.error(`FAILED: ${testCase.name} - Postman assertions failed:`, postmanFailures);
            failedTests += postmanFailures.length;
        }
    });

    // Output summary
    console.log(`\nContract Test Summary:`);
    console.log(`Total tests: ${contractTests.length}`);
    console.log(`Passed: ${contractTests.length - failedTests}`);
    console.log(`Failed: ${failedTests}`);

    if (failedTests > 0) {
        console.error('Contract tests failed. Exiting with error code.');
        process.exit(1);
    } else {
        console.log('All contract tests passed!');
        process.exit(0);
    }
});

// Error handling: retry logic for flaky network requests
const runWithRetry = async (retries = 3, delayMs = 1000) => {
    for (let i = 0; i < retries; i++) {
        try {
            await new Promise((resolve, reject) => {
                newman.run({
                    collection: collectionPath,
                    environment: environmentPath
                }, (err) => {
                    if (err) reject(err);
                    else resolve();
                });
            });
            return;
        } catch (err) {
            if (i === retries - 1) throw err;
            console.warn(`Retry ${i + 1} failed: ${err.message}`);
            await new Promise(resolve => setTimeout(resolve, delayMs));
        }
    }
};

- name: Run Pact Tests Parallel
  run: |
    pact-broker can-i-deploy --pacticipant order-service --version ${{ github.sha }} --broker-base-url $BROKER_URL
    ./gradlew pactConsumerTests --parallel --max-workers=4
  env:
    BROKER_URL: ${{ secrets.PACT_BROKER_URL }}
    BROKER_TOKEN: ${{ secrets.PACT_BROKER_TOKEN }}

plugins {
    id 'org.springframework.cloud.contract' version '4.1.0'
}
contracts {
    testFramework = 'JUNIT5'
    baseClassForTests = 'com.example.userservice.contracts.UserContractBaseTest'
}
publishing {
    publications {
        maven(MavenPublication) {
            from components.java
            artifact tasks.named('verifierStubsJar')
        }
    }
}

- name: Run Postman Contract Tests
  run: |
    npm install -g newman
    newman run user-api-contract.postman_collection.json \
      --environment user-api-contract.postman_environment.json \
      --env-var "baseUrl=${{ secrets.API_BASE_URL }}" \
      --reporters cli,junit --reporter-junit-export test-results.xml
  env:
    API_BASE_URL: ${{ secrets.API_BASE_URL }}