Retrospective: Adopting Platform Engineering with Backstage 1.20: 40% Faster Onboarding

// custom-scaffolder-action-gke-namespace.ts
// Backstage 1.20 Scaffolder Action: Provisions GKE namespace with IAM bindings for new hires
// Requires @backstage/plugin-scaffolder-backend ^1.18.0, @google-cloud/container ^5.0.0

import { createTemplateAction } from '@backstage/plugin-scaffolder-node';
import { ContainerClient } from '@google-cloud/container';
import { logger } from '../logger';
import { Config } from '@backstage/config';

// Input type definitions for the action
interface GkeNamespaceInput {
  clusterName: string;
  zone: string;
  namespace: string;
  serviceAccountEmail: string;
  projectId: string;
}

// Output type for action results
interface GkeNamespaceOutput {
  namespaceUrl: string;
  iamBinding: string;
}

export const createGkeNamespaceAction = (config: Config) => {
  // Initialize GCP Container client with credentials from Backstage config
  const gcpCredentials = config.getConfig('gcp').getOptionalString('serviceAccountKey');
  if (!gcpCredentials) {
    throw new Error('Missing GCP service account key in app-config.yaml under gcp.serviceAccountKey');
  }

  const containerClient = new ContainerClient({
    credentials: JSON.parse(gcpCredentials),
  });

  return createTemplateAction({
    id: 'acme:gke:create-namespace',
    description: 'Provisions a GKE namespace with read-only IAM bindings for a new developer service account',
    schema: {
      input: {
        type: 'object',
        required: ['clusterName', 'zone', 'namespace', 'serviceAccountEmail', 'projectId'],
        properties: {
          clusterName: { type: 'string', description: 'Target GKE cluster name' },
          zone: { type: 'string', description: 'GCP zone of the cluster' },
          namespace: { type: 'string', description: 'Desired namespace name (must follow RFC 1123)' },
          serviceAccountEmail: { type: 'string', description: 'GCP service account email for the new developer' },
          projectId: { type: 'string', description: 'GCP project ID' },
        },
      },
      output: {
        type: 'object',
        properties: {
          namespaceUrl: { type: 'string', description: 'Kubernetes namespace URL' },
          iamBinding: { type: 'string', description: 'IAM binding resource name' },
        },
      },
    },
    async handler(ctx) {
      const { clusterName, zone, namespace, serviceAccountEmail, projectId } = ctx.input;
      const clusterPath = `projects/${projectId}/locations/${zone}/clusters/${clusterName}`;

      try {
        // Step 1: Verify cluster exists and is reachable
        ctx.logger.info(`Checking existence of cluster ${clusterName} in ${zone}`);
        const [cluster] = await containerClient.getCluster({ name: clusterPath });
        if (!cluster) {
          throw new Error(`Cluster ${clusterName} not found in zone ${zone}`);
        }

        // Step 2: Create namespace via Kubernetes API (assumes GKE cluster has RBAC configured)
        // Note: In production, use the GKE API to create namespace instead of direct k8s client for auditability
        const k8sClient = await getK8sClient(cluster, gcpCredentials);
        const namespaceResponse = await k8sClient.namespaces.create({
          body: {
            metadata: {
              name: namespace,
              labels: {
                'acme.io/onboarded': 'true',
                'acme.io/created-by': 'backstage-scaffolder',
              },
            },
          },
        });

        // Step 3: Bind service account to namespace with read-only permissions
        const iamClient = await getIamClient(gcpCredentials);
        const binding = await iamClient.projects.serviceAccounts.setIamPolicy({
          resource: `projects/${projectId}/serviceAccounts/${serviceAccountEmail}`,
          requestBody: {
            policy: {
              bindings: [
                {
                  role: 'roles/container.namespaceViewer',
                  members: [`serviceAccount:${serviceAccountEmail}`],
                  condition: {
                    title: `namespace-access-${namespace}`,
                    expression: `resource.name.startsWith('namespace:${namespace}')`,
                  },
                },
              ],
            },
          },
        });

        ctx.output('namespaceUrl', `https://console.cloud.google.com/kubernetes/namespace/${projectId}/${zone}/${clusterName}/${namespace}`);
        ctx.output('iamBinding', binding.data.bindings?.[0].role || '');
        ctx.logger.info(`Successfully created namespace ${namespace} and bound ${serviceAccountEmail}`);
      } catch (error) {
        ctx.logger.error(`Failed to provision GKE namespace: ${error.message}`, { error });
        throw new Error(`GKE namespace provisioning failed: ${error instanceof Error ? error.message : String(error)}`);
      }
    },
  });
};

// Helper: Initialize Kubernetes client with GCP credentials
async function getK8sClient(cluster: any, credentials: string) {
  // Implementation uses GCP auth to get k8s bearer token
  // Omitted for brevity but included in full repo: https://github.com/acme-corp/backstage-extensions
  throw new Error('Helper implementation omitted for example length');
}

// Helper: Initialize IAM client
async function getIamClient(credentials: string) {
  // Uses @google-cloud/iam to set service account policies
  // Full code at https://github.com/acme-corp/backstage-extensions
  throw new Error('Helper implementation omitted for example length');
}

// confluence-to-backstage-ingester.ts
// Migrates legacy Confluence onboarding docs to Backstage 1.20 Software Catalog Component entities
// Requires @backstage/catalog-client ^1.5.0, confluence.js ^5.0.0, dotenv ^16.0.0

import { CatalogClient } from '@backstage/catalog-client';
import { ConfigReader } from '@backstage/config';
import ConfluenceClient from 'confluence.js';
import dotenv from 'dotenv';
import fs from 'fs/promises';
import path from 'path';

dotenv.config();

// Validate required environment variables
const requiredEnvVars = ['CONFLUENCE_API_KEY', 'CONFLUENCE_DOMAIN', 'BACKSTAGE_API_TOKEN', 'BACKSTAGE_BASE_URL'];
for (const varName of requiredEnvVars) {
  if (!process.env[varName]) {
    throw new Error(`Missing required environment variable: ${varName}`);
  }
}

// Initialize clients
const confluence = new ConfluenceClient({
  host: process.env.CONFLUENCE_DOMAIN!,
  authentication: {
    type: 'token',
    token: process.env.CONFLUENCE_API_KEY!,
  },
});

const catalogClient = new CatalogClient({
  baseUrl: process.env.BACKSTAGE_BASE_URL!,
  token: process.env.BACKSTAGE_API_TOKEN!,
});

// Define Confluence page structure for onboarding docs
interface ConfluenceOnboardingPage {
  id: string;
  title: string;
  spaceKey: string;
  body: {
    storage: {
      value: string;
    };
  };
  labels: { name: string }[];
}

// Map Confluence space keys to Backstage entity kinds
const spaceToKindMap: Record = {
  'ENG': 'Component',
  'DATA': 'Resource',
  'INFRA': 'System',
};

async function ingestConfluencePages(spaceKey: string) {
  let start = 0;
  const limit = 50;
  const ingestedEntities = [];

  try {
    while (true) {
      // Fetch paginated Confluence pages
      const { data: pages } = await confluence.content.getPageBySpace({
        spaceKey,
        start,
        limit,
        expand: ['body.storage', 'labels'],
      });

      if (pages.length === 0) break;

      for (const page of pages) {
        try {
          // Skip non-onboarding pages
          const isOnboarding = page.labels.some(l => l.name.includes('onboarding'));
          if (!isOnboarding) continue;

          // Parse Confluence page to extract service metadata
          const serviceName = page.title.replace(/Onboarding: /, '').replace(/\s+/g, '-').toLowerCase();
          const description = extractDescription(page.body.storage.value);
          const owner = extractOwner(page.body.storage.value);
          const lifecycle = extractLifecycle(page.body.storage.value);

          // Construct Backstage Component entity
          const entity = {
            apiVersion: 'backstage.io/v1alpha1',
            kind: spaceToKindMap[spaceKey] || 'Component',
            metadata: {
              name: serviceName,
              title: page.title,
              description: description || `Onboarding docs for ${serviceName}`,
              labels: {
                'confluence.id': page.id,
                'confluence.space': spaceKey,
                'acme.io/migrated': 'true',
              },
              annotations: {
                'confluence.com/page-id': page.id,
                'backstage.io/techdocs-ref': 'dir:.',
              },
            },
            spec: {
              type: 'service',
              owner: owner || 'team-unspecified',
              lifecycle: lifecycle || 'experimental',
            },
          };

          // Validate entity against Backstage schema before submitting
          const validationResult = await catalogClient.validateEntity(entity);
          if (!validationResult.valid) {
            console.error(`Invalid entity for ${serviceName}:`, validationResult.errors);
            continue;
          }

          // Submit to Backstage Catalog
          await catalogClient.createOrUpdateEntity(entity);
          ingestedEntities.push(entity.metadata.name);
          console.log(`Ingested ${entity.metadata.name} from Confluence page ${page.id}`);
        } catch (pageError) {
          console.error(`Failed to process Confluence page ${page.id}:`, pageError);
          // Continue processing other pages instead of failing entire batch
        }
      }

      start += limit;
    }

    // Write ingestion report
    const reportPath = path.join(__dirname, 'ingestion-report.json');
    await fs.writeFile(reportPath, JSON.stringify({
      spaceKey,
      ingestedCount: ingestedEntities.length,
      entities: ingestedEntities,
      timestamp: new Date().toISOString(),
    }, null, 2));
    console.log(`Ingestion complete. Report saved to ${reportPath}`);
  } catch (error) {
    console.error(`Fatal error ingesting Confluence space ${spaceKey}:`, error);
    throw error;
  }
}

// Helper: Extract description from Confluence HTML body
function extractDescription(html: string): string | undefined {
  const match = html.match(/ {
  console.error('Ingestion failed:', error);
  process.exit(1);
});

// backstage-onboarding-metrics-exporter.ts
// Exports Backstage 1.20 onboarding event metrics to Prometheus for tracking velocity
// Requires @backstage/plugin-observability ^1.0.0, prom-client ^15.0.0, express ^4.18.0

import { createPlugin } from '@backstage/core-plugin-api';
import { createRouter } from '@backstage/plugin-observability';
import express from 'express';
import promClient from 'prom-client';
import { logger } from '../logger';
import { AuditClient } from '@backstage/plugin-audit-logger';

// Initialize Prometheus registry
const register = new promClient.Registry();
promClient.collectDefaultMetrics({ register });

// Define custom metrics for onboarding tracking
const onboardingDurationHistogram = new promClient.Histogram({
  name: 'backstage_onboarding_duration_seconds',
  help: 'Time taken for a new developer to complete onboarding steps',
  labelNames: ['team', 'role', 'step'],
  buckets: [3600, 7200, 14400, 28800, 43200, 86400, 172800, 345600, 604800], // 1h to 7d buckets
});

const onboardingStepCounter = new promClient.Counter({
  name: 'backstage_onboarding_steps_total',
  help: 'Total number of onboarding steps completed',
  labelNames: ['team', 'role', 'step', 'status'],
});

const scaffolderRunCounter = new promClient.Counter({
  name: 'backstage_scaffolder_runs_total',
  help: 'Total number of scaffolder template runs for onboarding',
  labelNames: ['template', 'status'],
});

// Register metrics
register.registerMetric(onboardingDurationHistogram);
register.registerMetric(onboardingStepCounter);
register.registerMetric(scaffolderRunCounter);

export const onboardingMetricsPlugin = createPlugin({
  id: 'acme.onboarding-metrics',
  routes: {
    root: '/onboarding-metrics',
  },
});

export const onboardingMetricsRouter = createRouter({
  path: '/onboarding-metrics',
  router: async () => {
    const router = express.Router();

    // Endpoint to expose Prometheus metrics
    router.get('/metrics', async (req, res) => {
      try {
        res.set('Content-Type', register.contentType);
        res.end(await register.metrics());
      } catch (error) {
        logger.error(`Failed to expose metrics: ${error.message}`);
        res.status(500).end('Error generating metrics');
      }
    });

    // Endpoint to receive onboarding event webhooks from Backstage
    router.post('/events', express.json(), async (req, res) => {
      try {
        const event = req.body;
        if (!event.type || !event.payload) {
          return res.status(400).json({ error: 'Invalid event payload' });
        }

        // Process only onboarding-related events
        if (event.type.startsWith('onboarding.')) {
          const { team, role, step, durationSeconds, status } = event.payload;

          // Validate required fields
          if (!team || !role || !step) {
            return res.status(400).json({ error: 'Missing required fields: team, role, step' });
          }

          // Record histogram for duration if provided
          if (durationSeconds) {
            onboardingDurationHistogram.observe(
              { team, role, step },
              durationSeconds
            );
          }

          // Increment step counter
          onboardingStepCounter.inc(
            { team, role, step, status: status || 'completed' },
            1
          );
        }

        // Process scaffolder events
        if (event.type === 'scaffolder.template.run') {
          const { templateName, status } = event.payload;
          if (templateName && status) {
            scaffolderRunCounter.inc(
              { template: templateName, status },
              1
            );
          }
        }

        res.status(200).json({ success: true });
      } catch (error) {
        logger.error(`Failed to process event: ${error.message}`, { error });
        res.status(500).json({ error: 'Internal server error' });
      }
    });

    // Initialize audit log listener to capture onboarding events automatically
    const auditClient = AuditClient.fromConfig(config);
    auditClient.on('auditEntry', (entry) => {
      if (entry.action.startsWith('onboarding.')) {
        // Forward audit entry to metrics endpoint
        fetch('http://localhost:3000/api/onboarding-metrics/events', {
          method: 'POST',
          headers: { 'Content-Type': 'application/json' },
          body: JSON.stringify({
            type: entry.action,
            payload: entry.metadata,
          }),
        }).catch(err => logger.error(`Failed to forward audit entry: ${err.message}`));
      }
    });

    return router;
  },
});

// package.json (excerpt)
{
  "dependencies": {
    "@backstage/plugin-scaffolder-backend": "1.18.0",
    "@backstage/plugin-catalog-backend": "1.20.0"
  },
  "overrides": {
    "@backstage/*": "1.20.2"
  }
}

# mkdocs.yml (excerpt for TechDocs)
site_name: Acme Engineering Docs
theme:
  name: material
  palette:
    - media: "(prefers-color-scheme: light)"
      scheme: default
    - media: "(prefers-color-scheme: dark)"
      scheme: slate
plugins:
  - search
  - mdx:
      extensions:
        - admonition
        - codehilite
markdown_extensions:
  - admonition
  - toc:
      permalink: true

# Run catalog lint via Backstage CLI
npx backstage-cli catalog lint \
  --config app-config.yaml \
  --project-root . \
  --output stale-entities.json