We Replaced Jenkins 2.440 with GitLab CI 16 and TeamCity 2026: Build Queue Time Fell 70%

// Jenkins 2.440 Pipeline for spring-boot-order-service v2.1.4
// Demonstrates legacy queue contention: shared runner pool, no priority scaling
pipeline {
    agent none // Use shared agent pool, cause of queue delays
    options {
        timeout(time: 45, unit: 'MINUTES') // Long timeout due to queue waits
        disableConcurrentBuilds() // Prevent further queue bloat
        buildDiscarder(logRotator(numToKeepStr: '20'))
    }
    triggers {
        pollSCM('H/5 * * * *') // Poll every 5 mins, adds to queue
    }
    stages {
        stage('Checkout') {
            agent { label 'java-17' } // Shared label, high contention
            steps {
                checkout scm
                sh 'java -version' // Verify Java version
            }
            post {
                failure {
                    emailext(
                        subject: \"Checkout failed: ${env.JOB_NAME} #${env.BUILD_NUMBER}\",
                        body: \"See ${env.BUILD_URL}console\",
                        to: 'eng-alerts@company.com'
                    )
                }
            }
        }
        stage('Parallel Build & Test') {
            parallel {
                stage('Compile') {
                    agent { label 'java-17' }
                    steps {
                        sh 'chmod +x mvnw'
                        sh './mvnw clean compile -DskipTests'
                    }
                    post {
                        failure {
                            archiveArtifacts artifacts: 'target/compile-logs/**', fingerprint: true
                            emailext(subject: \"Compile failed\", body: \"Logs: ${env.BUILD_URL}artifact/target/compile-logs/\", to: 'eng-alerts@company.com')
                        }
                    }
                }
                stage('Unit Test') {
                    agent { label 'java-17' }
                    steps {
                        sh './mvnw test -Dtest=Unit*'
                        junit 'target/surefire-reports/**/*.xml'
                    }
                    post {
                        failure {
                            junit 'target/surefire-reports/**/*.xml'
                            emailext(subject: \"Unit tests failed\", body: \"See ${env.BUILD_URL}testReport/\", to: 'eng-alerts@company.com')
                        }
                    }
                }
            }
        }
        stage('Integration Test') {
            agent { label 'java-17' }
            steps {
                sh './mvnw verify -Dtest=Integration*'
                junit 'target/failsafe-reports/**/*.xml'
            }
            post {
                failure {
                    archiveArtifacts artifacts: 'target/failsafe-reports/**', fingerprint: true
                    emailext(subject: \"Integration tests failed\", body: \"See ${env.BUILD_URL}testReport/\", to: 'eng-alerts@company.com')
                }
            }
        }
        stage('Build Docker Image') {
            agent { label 'docker' } // Another shared label
            steps {
                sh 'docker build -t company/order-svc:${env.BUILD_NUMBER} .'
                sh 'docker push company/order-svc:${env.BUILD_NUMBER}'
            }
            post {
                failure {
                    emailext(subject: \"Docker build failed\", body: \"See ${env.BUILD_URL}console\", to: 'eng-alerts@company.com')
                }
            }
        }
    }
    post {
        always {
            cleanWs() // Clean workspace to free runner space
        }
        success {
            slackSend(channel: '#eng-releases', message: \"Build ${env.BUILD_NUMBER} succeeded: ${env.BUILD_URL}\")
        }
        failure {
            slackSend(channel: '#eng-alerts', message: \"Build ${env.BUILD_NUMBER} failed: ${env.BUILD_URL}\")
        }
    }
}

# GitLab CI 16.4.1 Pipeline for spring-boot-order-service v2.1.4
# Uses auto-scaling runner pool, priority queues for PR vs scheduled builds
image: maven:3.9.6-eclipse-temurin-17

variables:
  MAVEN_CLI_OPTS: "-s .m2/settings.xml --batch-mode"
  MAVEN_OPTS: "-Xmx1024m"
  DOCKER_REGISTRY: "registry.company.com"
  IMAGE_NAME: "company/order-svc"

# Reusable templates to reduce duplication
.test_template: &test_template
  stage: test
  script:
    - ./mvnw $MAVEN_CLI_OPTS test -Dtest=$TEST_PATTERN
  artifacts:
    when: always
    reports:
      junit: target/surefire-reports/**/*.xml
    paths:
      - target/surefire-reports/
    expire_in: 7 days
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      priority: 10 # Higher priority for MR builds
    - if: $CI_PIPELINE_SOURCE == "schedule"
      priority: 5 # Lower priority for scheduled builds

stages:
  - checkout
  - build
  - test
  - package
  - deploy

checkout_code:
  stage: checkout
  script:
    - git checkout $CI_COMMIT_SHA
    - java -version
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_PIPELINE_SOURCE == "schedule"

compile:
  stage: build
  script:
    - ./mvnw $MAVEN_CLI_OPTS clean compile -DskipTests
  artifacts:
    paths:
      - target/classes/
    expire_in: 1 hour
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_PIPELINE_SOURCE == "schedule"

unit_test:
  <<: *test_template
  variables:
    TEST_PATTERN: "Unit*"
  dependencies:
    - compile

integration_test:
  stage: test
  script:
    - ./mvnw $MAVEN_CLI_OPTS verify -Dtest=Integration*
  artifacts:
    when: always
    reports:
      junit: target/failsafe-reports/**/*.xml
    paths:
      - target/failsafe-reports/
    expire_in: 7 days
  dependencies:
    - compile
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      priority: 10
    - if: $CI_PIPELINE_SOURCE == "schedule"
      priority: 5

build_docker:
  stage: package
  image: docker:24.0.5
  services:
    - docker:24.0.5-dind
  before_script:
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $DOCKER_REGISTRY
  script:
    - docker build -t $DOCKER_REGISTRY/$IMAGE_NAME:$CI_COMMIT_SHA .
    - docker push $DOCKER_REGISTRY/$IMAGE_NAME:$CI_COMMIT_SHA
  after_script:
    - docker logout $DOCKER_REGISTRY
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_PIPELINE_SOURCE == "schedule"

deploy_staging:
  stage: deploy
  script:
    - kubectl set image deployment/order-svc order-svc=$DOCKER_REGISTRY/$IMAGE_NAME:$CI_COMMIT_SHA -n staging
  environment:
    name: staging
    url: https://order-staging.company.com
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
  when: manual

# Error handling: notify on failure
on_failure:
  script:
    - curl -X POST -H \"Content-Type: application/json\" -d \"{\\\"text\\\":\\\"Pipeline $CI_PIPELINE_ID failed: $CI_PIPELINE_URL\\\"}\" $SLACK_WEBHOOK_URL
  when: on_failure

// TeamCity 2026.1 Kotlin DSL Configuration for Nightly Release Pipeline
// Manages scheduled nightly builds, integrates with GitLab CI for artifact promotion
package _self

import jetbrains.buildServer.configs.kotlin.*
import jetbrains.buildServer.configs.kotlin.buildSteps.*
import jetbrains.buildServer.configs.kotlin.triggers.*
import jetbrains.buildServer.configs.kotlin.failureStrategies.*
import jetbrains.buildServer.configs.kotlin.vcs.*

version = "2026.1"

project {
    id("NightlyReleasePipeline")
    name = "Nightly Release Pipeline"
    description = "Scheduled nightly build and deploy to production for order-svc"

    // VCS Root: GitLab repository
    val vcsRoot = gitVcsRoot {
        id("GitLab_OrderSvc")
        name = "GitLab Order Service"
        url = "https://gitlab.company.com/backend/order-svc.git"
        branch = "refs/heads/main"
        authMethod = personalToken {
            tokenId = "gitlab-personal-token" // Stored in TeamCity credentials
        }
    }

    // Nightly build configuration
    buildType {
        id("Nightly_Build")
        name = "Nightly Build"
        description = "Compile, test, and package order-svc nightly"

        vcs {
            root(vcsRoot)
        }

        triggers {
            schedule {
                schedulingPolicy = daily {
                    hour = 2 // Run at 2 AM UTC
                    minute = 0
                }
                triggerBuild = always()
                withPendingChanges = false
            }
        }

        steps {
            // Checkout step (implicit, but explicit for clarity)
            step("checkout") {
                name = "Checkout Code"
                type = "jetbrains.buildServer.vcs.checkout"
            }

            // Compile step
            maven {
                name = "Compile Project"
                goals = "clean compile -DskipTests"
                mavenVersion = "3.9.6"
                jdk {
                    name = "JDK 17"
                    jdkHome = "/usr/lib/jvm/java-17-openjdk"
                }
            }

            // Unit tests
            maven {
                name = "Run Unit Tests"
                goals = "test -Dtest=Unit*"
                mavenVersion = "3.9.6"
                failureStrategy = failureStrategy {
                    failOnExitCode = true
                    retry {
                        attempts = 2
                        delay = 30 // Wait 30s between retries
                    }
                }
            }

            // Integration tests
            maven {
                name = "Run Integration Tests"
                goals = "verify -Dtest=Integration*"
                mavenVersion = "3.9.6"
                failureStrategy = failureStrategy {
                    failOnExitCode = true
                }
            }

            // Build Docker image
            dockerBuild {
                name = "Build Docker Image"
                dockerfile = "Dockerfile"
                imageName = "registry.company.com/company/order-svc:nightly-${buildNumber}"
                push = true
                registry {
                    url = "registry.company.com"
                    credentials = "docker-registry-creds"
                }
            }

            // Promote artifact to GitLab CI if all tests pass
            step("promote") {
                name = "Promote to GitLab"
                type = "custom-script"
                scriptContent = """
                    #!/bin/bash
                    set -e
                    curl -X POST -H \"PRIVATE-TOKEN: ${env.GITLAB_TOKEN}\" \
                         -d \"tag_name=nightly-${buildNumber}&ref=main\" \
                         https://gitlab.company.com/api/v4/projects/123/repository/tags
                """.trimIndent()
            }
        }

        // Failure strategy: notify on failure, retry twice for flaky tests
        failureStrategies {
            retry {
                attempts = 2
                delay = 60
                applyTo = listOf("Run Unit Tests", "Run Integration Tests")
            }
            notify {
                notifier = slackNotifier {
                    webhookUrl = "https://hooks.slack.com/services/xxx/xxx/xxx"
                    message = "Nightly build ${buildNumber} failed: ${teamcity.build.url}"
                }
                when = onFailure
            }
        }

        // Artifacts to archive
        artifactRules = """
            target/surefire-reports/** => test-reports
            target/failsafe-reports/** => integration-test-reports
        """.trimIndent()
    }
}

// Jenkins Groovy script to export queue stats
import jenkins.model.Jenkins
import hudson.model.Queue

def queue = Jenkins.instance.queue
def items = queue.items

println \"Total queue items: ${items.size()}\"
items.each { item ->
    println \"Job: ${item.task.name}, Wait time: ${System.currentTimeMillis() - item.inQueueSince}ms, Priority: ${item.priority}\"
}

// TeamCity 2026 Kotlin DSL step to trigger GitLab CI pipeline
step("trigger-gitlab") {
    name = "Trigger GitLab CI Promote Pipeline"
    type = "custom-script"
    scriptContent = """
        #!/bin/bash
        curl -X POST \
             -H \"PRIVATE-TOKEN: ${env.GITLAB_TOKEN}\" \
             -d \"variables[SOURCE]=teamcity&variables[BUILD_ID]=${teamcity.build.id}\" \
             https://gitlab.company.com/api/v4/projects/123/trigger/pipeline
    """.trimIndent()
}

# Prometheus query for GitLab CI runner utilization (busy vs idle)
sum(rate(gitlab_runner_jobs_total{status=\"running\"}[5m])) by (runner_id) 
/ 
sum(rate(gitlab_runner_jobs_total[5m])) by (runner_id) * 100