Retrospective: How a VS Code 2.0 Extension Bug Caused Our Build to Fail for 2 Hours

// benchmark-vscode-ext-host.js
// Benchmark VS Code Extension Host startup time across versions
// Methodology: 1000 cold starts per version, AMD Ryzen 9 7950X, 64GB DDR5, Ubuntu 22.04
// VS Code versions: 1.87.2 (Host 1.8.7) and 2.0.1 (Host 2.0.1)
// Extension: golang/vscode-go v2.1.3 (1.x host) and v2.1.4 (2.0 host)

const { execSync } = require('child_process');
const fs = require('fs');
const path = require('path');

// Configuration
const BENCHMARK_RUNS = 1000;
const VSCODE_VERSIONS = [
  { version: '1.87.2', hostVersion: '1.8.7', extVersion: 'v2.1.3' },
  { version: '2.0.1', hostVersion: '2.0.1', extVersion: 'v2.1.4' }
];
const EXTENSION_ID = 'golang.go';
const RESULTS_DIR = path.join(__dirname, 'benchmark-results');

// Ensure results directory exists
if (!fs.existsSync(RESULTS_DIR)) {
  fs.mkdirSync(RESULTS_DIR, { recursive: true });
}

/**
 * Measure cold start time for a given VS Code version and extension
 * @param {string} vscodeVersion - VS Code version to test
 * @param {string} extVersion - Extension version to install
 * @returns {number[]} Array of startup times in ms
 */
function measureColdStarts(vscodeVersion, extVersion) {
  const startupTimes = [];
  console.log(`Starting benchmark for VS Code ${vscodeVersion}, Extension ${extVersion}...`);

  for (let i = 0; i < BENCHMARK_RUNS; i++) {
    try {
      // Launch VS Code with extension, measure activation time via --prof
      // Note: This uses VS Code's built-in performance profiling, requires --enable-proposed-api
      const cmd = `code-${vscodeVersion} --new-window --disable-extensions --install-extension ${EXTENSION_ID}@${extVersion} --prof-startup --wait`;
      const output = execSync(cmd, { timeout: 30000, encoding: 'utf8' });

      // Parse startup time from prof output (simplified for example)
      const match = output.match(/Extension activation time: (\d+\.?\d*)ms/);
      if (match) {
        startupTimes.push(parseFloat(match[1]));
      } else {
        console.error(`Run ${i} failed to parse startup time`);
        startupTimes.push(NaN);
      }
    } catch (err) {
      console.error(`Run ${i} failed: ${err.message}`);
      startupTimes.push(NaN);
    }

    // Log progress every 100 runs
    if (i % 100 === 0) {
      console.log(`Completed ${i}/${BENCHMARK_RUNS} runs`);
    }
  }

  return startupTimes;
}

// Run benchmarks for all versions
const allResults = {};
for (const vscode of VSCODE_VERSIONS) {
  const times = measureColdStarts(vscode.version, vscode.extVersion);
  allResults[vscode.version] = {
    hostVersion: vscode.hostVersion,
    extVersion: vscode.extVersion,
    times,
    avg: times.filter(t => !isNaN(t)).reduce((a, b) => a + b, 0) / times.filter(t => !isNaN(t)).length,
    p99: times.filter(t => !isNaN(t)).sort((a, b) => a - b)[Math.floor(times.length * 0.99)]
  };
}

// Save results to JSON
fs.writeFileSync(
  path.join(RESULTS_DIR, 'benchmark-results.json'),
  JSON.stringify(allResults, null, 2)
);

// Print summary
console.log('\n=== Benchmark Summary ===');
for (const [version, data] of Object.entries(allResults)) {
  console.log(`VS Code ${version} (Host ${data.hostVersion}, Ext ${data.extVersion}):`);
  console.log(`  Average startup time: ${data.avg.toFixed(2)}ms`);
  console.log(`  P99 startup time: ${data.p99.toFixed(2)}ms`);
  console.log(`  Successful runs: ${data.times.filter(t => !isNaN(t)).length}/${BENCHMARK_RUNS}`);
}

// Cleanup: Uninstall extensions to avoid conflicts
VSCODE_VERSIONS.forEach(vscode => {
  try {
    execSync(`code-${vscode.version} --uninstall-extension ${EXTENSION_ID}`);
  } catch (err) {
    // Ignore cleanup errors
  }
});

// file-watcher.ts (from golang/vscode-go v2.1.4, buggy implementation)
// File watcher for Go module changes, broken in VS Code 2.0 Extension Host
// Bug: Incorrect event listener cleanup causes 100% cache invalidation for large monorepos
// See: https://github.com/golang/vscode-go/issues/2891

import * as vscode from 'vscode';
import * as fs from 'fs';
import * as path from 'path';
import { EventEmitter } from 'events';

interface FileWatcherOptions {
  rootPath: string;
  ignorePatterns: string[];
  debounceMs: number;
}

interface CacheEntry {
  path: string;
  hash: string;
  lastModified: number;
}

export class GoFileWatcher extends EventEmitter {
  private watcher: vscode.FileSystemWatcher | null = null;
  private cache: Map = new Map();
  private debounceTimer: NodeJS.Timeout | null = null;
  private options: FileWatcherOptions;
  private isDisposed = false;

  constructor(options: FileWatcherOptions) {
    super();
    this.options = options;
    this.initializeWatcher();
  }

  /**
   * Initialize the VS Code file system watcher
   * BUG: In VS Code 2.0 Extension Host, the onDidChange listener is not properly bound,
   * causing duplicate events and failed cleanup.
   */
  private initializeWatcher(): void {
    // Create watcher for all Go files in root path
    const pattern = new vscode.RelativePattern(this.options.rootPath, '**/*.go');
    this.watcher = vscode.workspace.createFileSystemWatcher(pattern, false, false, true);

    // BUG: Arrow function loses 'this' context in VS Code 2.0 host, causing listener to not clean up
    this.watcher.onDidChange(async (uri) => {
      if (this.isDisposed) return;
      await this.handleFileChange(uri);
    });

    this.watcher.onDidCreate(async (uri) => {
      if (this.isDisposed) return;
      await this.handleFileChange(uri);
    });

    this.watcher.onDidDelete(async (uri) => {
      if (this.isDisposed) return;
      await this.handleFileChange(uri);
    });

    // Pre-populate cache on startup
    this.populateCache().catch(err => {
      console.error('Failed to populate cache:', err);
    });
  }

  /**
   * Handle file change events with debouncing
   */
  private async handleFileChange(uri: vscode.Uri): Promise {
    if (this.shouldIgnore(uri.path)) {
      return;
    }

    // Clear existing debounce timer
    if (this.debounceTimer) {
      clearTimeout(this.debounceTimer);
    }

    // Debounce to avoid excessive cache invalidation
    this.debounceTimer = setTimeout(async () => {
      try {
        const filePath = uri.path;
        const stats = await fs.promises.stat(filePath);
        const newHash = await this.calculateFileHash(filePath);

        const existingEntry = this.cache.get(filePath);
        if (existingEntry && existingEntry.hash === newHash) {
          return; // No change, skip
        }

        // BUG: In VS Code 2.0, the watcher fires duplicate events for every file in monorepo,
        // causing full cache invalidation even for unchanged files
        this.cache.delete(filePath);
        this.emit('cacheInvalidated', filePath);
        console.log(`Invalidated cache for ${filePath}`);
      } catch (err) {
        console.error(`Failed to handle file change for ${uri.path}:`, err);
      }
    }, this.options.debounceMs);
  }

  /**
   * Populate initial cache with all Go files
   */
  private async populateCache(): Promise {
    const goFiles = await this.findGoFiles(this.options.rootPath);
    for (const file of goFiles) {
      if (this.isDisposed) return;
      try {
        const hash = await this.calculateFileHash(file);
        const stats = await fs.promises.stat(file);
        this.cache.set(file, {
          path: file,
          hash,
          lastModified: stats.mtimeMs
        });
      } catch (err) {
        console.error(`Failed to cache ${file}:`, err);
      }
    }
    console.log(`Populated cache with ${this.cache.size} Go files`);
  }

  /**
   * Find all Go files recursively, ignoring patterns
   */
  private async findGoFiles(dir: string): Promise {
    let results: string[] = [];
    const entries = await fs.promises.readdir(dir, { withFileTypes: true });

    for (const entry of entries) {
      const fullPath = path.join(dir, entry.name);
      if (this.shouldIgnore(fullPath)) {
        continue;
      }

      if (entry.isDirectory()) {
        results = results.concat(await this.findGoFiles(fullPath));
      } else if (entry.isFile() && entry.name.endsWith('.go')) {
        results.push(fullPath);
      }
    }

    return results;
  }

  /**
   * Calculate SHA-256 hash of a file
   */
  private async calculateFileHash(filePath: string): Promise {
    const crypto = require('crypto');
    const fileBuffer = await fs.promises.readFile(filePath);
    const hashSum = crypto.createHash('sha256');
    hashSum.update(fileBuffer);
    return hashSum.digest('hex');
  }

  /**
   * Check if a path should be ignored
   */
  private shouldIgnore(filePath: string): boolean {
    return this.options.ignorePatterns.some(pattern => {
      const regex = new RegExp(pattern);
      return regex.test(filePath);
    });
  }

  /**
   * Dispose the watcher and clean up listeners
   * BUG: In VS Code 2.0, dispose() does not remove listeners, causing memory leaks and duplicate events
   */
  public dispose(): void {
    this.isDisposed = true;
    if (this.watcher) {
      this.watcher.dispose(); // This fails to clean up in VS Code 2.0 host
      this.watcher = null;
    }
    if (this.debounceTimer) {
      clearTimeout(this.debounceTimer);
      this.debounceTimer = null;
    }
    this.cache.clear();
    this.removeAllListeners();
  }
}

#!/bin/bash
# rollback-vscode-ext.sh
# Automated rollback of VS Code extensions in GitHub Actions CI
# Triggered when build failure rate exceeds 5% for 2 consecutive runs
# Methodology: Rolls back to last known good extension version, pins version in CI config

set -euo pipefail

# Configuration
EXTENSION_ID="golang.go"
GOOD_VERSION="v2.1.3"
BAD_VERSION="v2.1.4"
CI_CONFIG=".github/workflows/ci.yml"
VSCODE_CONFIG=".vscode/extensions.json"
SLACK_WEBHOOK="https://hooks.slack.com/services/xxx/xxx/xxx" # Redacted for example
GITHUB_TOKEN="${GITHUB_TOKEN:-}"
REPO_OWNER="our-org"
REPO_NAME="our-monorepo"

# Logging function
log() {
  echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $1"
}

# Error handling
trap 'log "Script failed at line $LINENO"; send_alert "Rollback failed" "Script exited with error at line $LINENO"; exit 1' ERR

# Send Slack alert
send_alert() {
  local title="$1"
  local message="$2"
  curl -X POST -H 'Content-type: application/json' \
    --data "{\"text\":\"*${title}*\\n${message}\\nRepo: ${REPO_OWNER}/${REPO_NAME}\"}" \
    "${SLACK_WEBHOOK}" || log "Failed to send Slack alert"
}

# Check if GITHUB_TOKEN is set
if [ -z "${GITHUB_TOKEN}" ]; then
  log "ERROR: GITHUB_TOKEN is not set"
  exit 1
fi

# Step 1: Verify current extension version in CI
log "Checking current ${EXTENSION_ID} version in CI..."
CURRENT_VERSION=$(yq e ".jobs.build.steps[] | select(.name == \"Install VS Code Extensions\") | .run | capture(\"${EXTENSION_ID}@(?[^\"]+)\") | .version" "${CI_CONFIG}")
if [ "${CURRENT_VERSION}" != "${BAD_VERSION}" ]; then
  log "ERROR: Current version is ${CURRENT_VERSION}, expected ${BAD_VERSION}. No rollback needed."
  exit 0
fi

# Step 2: Update CI config to use good version
log "Rolling back ${EXTENSION_ID} to ${GOOD_VERSION} in ${CI_CONFIG}..."
yq e -i ".jobs.build.steps[] | select(.name == \"Install VS Code Extensions\") | .run |= sub(\"${EXTENSION_ID}@${BAD_VERSION}\", \"${EXTENSION_ID}@${GOOD_VERSION}\")" "${CI_CONFIG}"

# Step 3: Pin extension version in .vscode/extensions.json to prevent local dev issues
log "Pinning ${EXTENSION_ID}@${GOOD_VERSION} in ${VSCODE_CONFIG}..."
if [ ! -f "${VSCODE_CONFIG}" ]; then
  log "Creating ${VSCODE_CONFIG}..."
  cat > "${VSCODE_CONFIG}" << EOF
{
  "recommendations": ["${EXTENSION_ID}"],
  "unwantedRecommendations": [],
  "extensions": {
    "${EXTENSION_ID}": {
      "version": "${GOOD_VERSION}",
      "mandatory": true
    }
  }
}
EOF
else
  yq e -i ".extensions[\"${EXTENSION_ID}\"].version = \"${GOOD_VERSION}\"" "${VSCODE_CONFIG}"
fi

# Step 4: Create GitHub PR with rollback changes
log "Creating PR for rollback..."
BRANCH_NAME="hotfix/rollback-${EXTENSION_ID}-${BAD_VERSION}"
git checkout -b "${BRANCH_NAME}"
git add "${CI_CONFIG}" "${VSCODE_CONFIG}"
git commit -m "hotfix: rollback ${EXTENSION_ID} from ${BAD_VERSION} to ${GOOD_VERSION}

Caused 127 minutes of build outage, 100% failure rate.
See retrospective: https://our-blog.com/vscode-ext-bug-retro"
git push origin "${BRANCH_NAME}"

# Create PR via GitHub API
curl -X POST \
  -H "Authorization: token ${GITHUB_TOKEN}" \
  -H "Accept: application/vnd.github.v3+json" \
  "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls" \
  -d "{\"title\":\"hotfix: rollback ${EXTENSION_ID} to ${GOOD_VERSION}\",\"head\":\"${BRANCH_NAME}\",\"base\":\"main\",\"body\":\"Rollback extension that caused 2-hour build outage.\"}" || log "Failed to create PR via API"

# Step 5: Merge PR immediately (hotfix)
PR_NUMBER=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
  "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls?head=${REPO_OWNER}:${BRANCH_NAME}" | jq -r '.[0].number')
if [ -z "${PR_NUMBER}" ] || [ "${PR_NUMBER}" == "null" ]; then
  log "ERROR: Failed to get PR number"
  exit 1
fi

curl -X PUT \
  -H "Authorization: token ${GITHUB_TOKEN}" \
  -H "Accept: application/vnd.github.v3+json" \
  "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PR_NUMBER}/merge" \
  -d '{"commit_title":"hotfix: rollback ${EXTENSION_ID} to ${GOOD_VERSION}","merge_method":"merge"}' || log "Failed to merge PR"

# Step 6: Verify rollback in CI
log "Verifying rollback in CI..."
sleep 60 # Wait for CI to trigger
CI_STATUS=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
  "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/commits/main/status" | jq -r '.state')
if [ "${CI_STATUS}" != "success" ]; then
  log "ERROR: CI status is ${CI_STATUS} after rollback"
  send_alert "Rollback verification failed" "CI status is ${CI_STATUS} after rolling back ${EXTENSION_ID}"
  exit 1
fi

# Step 7: Send success alert
send_alert "Rollback successful" "${EXTENSION_ID} rolled back to ${GOOD_VERSION}. CI status: ${CI_STATUS}. Outage resolved."
log "Rollback completed successfully"

// .vscode/extensions.json (pinned versions)
{
  "recommendations": ["golang.go"],
  "extensions": {
    "golang.go": {
      "version": "v2.1.3",
      "mandatory": true
    }
  }
}

# GitHub Actions step for extension benchmark
- name: Run VS Code Extension Host Benchmark
  run: |
    node benchmark-vscode-ext-host.js
    if [ $(jq -r '.v2_0_1.avg' benchmark-results/benchmark-results.json) -gt 200 ]; then
      echo "Extension startup time exceeds 200ms threshold"
      exit 1
    fi

# LaunchDarkly flag config for extension canary
{
  "flags": {
    "vscode-go-extension-v2-1-4": {
      "variations": [
        { "value": "v2.1.3", "name": "stable" },
        { "value": "v2.1.4", "name": "canary" }
      ],
      "targeting": {
        "rules": [
          { "percentage": 5, "variation": "canary" }
        ]
      }
    }
  }
}