Code Story: Building a Custom LangChain 0.30 Agent for Jira Ticket Automation

// jira-tools.ts
// LangChain 0.30+ requires strict ESM imports, no CJS support
import { tool } from "@langchain/core/tools";
import { z } from "zod";
import { JiraApi } from "jira-client"; // v8.2.1, pinned for reproducibility
import { env } from "./env.js"; // Loads JIRA_HOST, JIRA_EMAIL, JIRA_API_TOKEN from process.env

// Initialize Jira client with retries and timeout handling
const jiraClient = new JiraApi({
  host: env.JIRA_HOST,
  username: env.JIRA_EMAIL,
  password: env.JIRA_API_TOKEN,
  apiVersion: "3", // Jira Cloud API v3, pinned to avoid breaking changes
  timeout: 10000, // 10s timeout for all requests
  maxRetries: 3, // Retry on 429/5xx errors
  retryDelay: 1000, // 1s base delay between retries
});

// Define a custom Jira ticket creation tool with Zod schema validation (LangChain 0.30 requirement)
export const createJiraTicket = tool(
  async (input) => {
    // Input validation beyond Zod: check project exists, issue type is valid
    try {
      // Verify project exists first to avoid cryptic Jira errors
      const project = await jiraClient.findProject(input.projectKey);
      if (!project) {
        throw new Error(`Project with key ${input.projectKey} not found`);
      }

      // Verify issue type is valid for the project
      const validIssueTypes = project.issueTypes.map((t) => t.name);
      if (!validIssueTypes.includes(input.issueType)) {
        throw new Error(
          `Invalid issue type ${input.issueType}. Valid types: ${validIssueTypes.join(", ")}`
        );
      }

      // Construct ticket payload with strict Jira API v3 fields
      const ticketPayload = {
        fields: {
          project: { key: input.projectKey },
          summary: input.summary,
          description: {
            type: "doc",
            version: 1,
            content: [
              {
                type: "paragraph",
                content: [{ type: "text", text: input.description }],
              },
            ],
          },
          issuetype: { name: input.issueType },
          // Optional fields: only include if provided
          ...(input.assignee && { assignee: { name: input.assignee } }),
          ...(input.labels && { labels: input.labels }),
          ...(input.priority && { priority: { name: input.priority } }),
        },
      };

      // Create ticket with error handling for Jira API errors
      const response = await jiraClient.addNewIssue(ticketPayload);
      return `Successfully created ticket ${response.key}: ${response.self}`;
    } catch (error) {
      // Structured error handling for common Jira errors
      if (error.statusCode === 401) {
        throw new Error("Jira authentication failed: invalid API token or email");
      }
      if (error.statusCode === 403) {
        throw new Error("Jira permission denied: service account lacks create ticket access");
      }
      if (error.statusCode === 429) {
        throw new Error("Jira rate limit exceeded: wait 60s before retrying");
      }
      // Re-throw unexpected errors with context
      throw new Error(`Failed to create Jira ticket: ${error.message}`);
    }
  },
  {
    name: "create_jira_ticket",
    description:
      "Creates a new Jira ticket with required fields. Use this when a user requests a new bug, story, or task. Always confirm project key and issue type with the user if not provided.",
    schema: z.object({
      projectKey: z.string().min(2).max(10).describe("Jira project key, e.g. ENG, OPS"),
      issueType: z.enum(["Bug", "Story", "Task", "Epic"]).describe("Type of Jira issue to create"),
      summary: z.string().min(10).max(255).describe("Short summary of the ticket, max 255 chars"),
      description: z.string().min(20).describe("Detailed description of the ticket, markdown supported"),
      assignee: z.string().optional().describe("Jira username of assignee, if known"),
      labels: z.array(z.string()).optional().describe("Array of labels to apply to the ticket"),
      priority: z.enum(["Highest", "High", "Medium", "Low", "Lowest"]).optional().describe("Ticket priority"),
    }),
  }
);

// custom-jira-agent.ts
import { ChatOpenAI } from "@langchain/openai"; // v0.3.0+, supports LangChain 0.30
import { AgentExecutor, createToolCallingAgent } from "langchain/agents"; // LangChain 0.30 agent module
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { BufferMemory } from "langchain/memory"; // Persistent memory for conversation context
import { createJiraTicket } from "./jira-tools.js";
import { searchJiraTickets } from "./jira-tools.js"; // Assume another tool defined similarly
import { updateJiraTicket } from "./jira-tools.js"; // Assume another tool defined similarly
import { env } from "./env.js";

// Initialize LLM with strict parameters for deterministic agent behavior
// Using Llama 3.1 70B self-hosted for cost/privacy, but works with GPT-4o too
const llm = new ChatOpenAI({
  model: env.LLM_MODEL || "llama3.1-70b", // Supports OpenAI-compatible APIs (Ollama, vLLM, etc.)
  temperature: 0, // Zero temperature for deterministic tool calling, critical for admin tasks
  maxTokens: 4096, // Enough for long ticket descriptions
  timeout: 30000, // 30s LLM timeout
  maxRetries: 2, // Retry LLM calls on transient errors
  // OpenAI-compatible API config for self-hosted Llama
  configuration: {
    baseURL: env.LLM_BASE_URL || "http://localhost:11434/v1", // Ollama default
    apiKey: env.LLM_API_KEY || "ollama", // Ollama requires non-empty key
  },
});

// Define agent prompt with strict instructions to reduce hallucination
const prompt = ChatPromptTemplate.fromMessages([
  [
    "system",
    `You are a Jira admin agent for the engineering team. Your only job is to:
1. Create, update, and search Jira tickets accurately
2. Never make up ticket keys, project keys, or user names
3. Always validate inputs against Jira before taking action
4. Return structured JSON responses for all actions
5. If a user asks for something outside Jira admin, politely decline
6. Log all actions to the audit trail for compliance

Current date: {current_date}
Team projects: {team_projects}
Valid issue types: Bug, Story, Task, Epic`,
  ],
  ["placeholder", "{chat_history}"],
  ["human", "{input}"],
  ["placeholder", "{agent_scratchpad}"],
]);

// Initialize buffer memory to persist conversation context across turns
const memory = new BufferMemory({
  memoryKey: "chat_history",
  returnMessages: true,
  inputKey: "input",
  outputKey: "output",
});

// Create the tool-calling agent with LangChain 0.30's new factory method
const agent = createToolCallingAgent({
  llm,
  tools: [createJiraTicket, searchJiraTickets, updateJiraTicket],
  prompt,
});

// Initialize agent executor with error handling and iteration limits
const agentExecutor = new AgentExecutor({
  agent,
  tools: [createJiraTicket, searchJiraTickets, updateJiraTicket],
  memory,
  maxIterations: 5, // Prevent infinite loops, fail after 5 tool calls
  earlyStoppingMethod: "force", // Return error if max iterations reached
  returnIntermediateSteps: true, // Include tool calls in output for debugging
  handleParsingErrors: (error) => {
    // Custom parsing error handling for malformed LLM responses
    console.error("Agent parsing error:", error);
    return "I encountered an error processing your request. Please rephrase your query with clear Jira details.";
  },
});

// Wrapper function to invoke the agent with audit logging
export async function runJiraAgent(input: string, userId: string) {
  try {
    // Add current date and team context to agent input
    const currentDate = new Date().toISOString().split("T")[0];
    const teamProjects = env.TEAM_PROJECT_KEYS.split(",").join(", "); // e.g. "ENG, OPS, SRE"

    const response = await agentExecutor.invoke({
      input,
      current_date: currentDate,
      team_projects: teamProjects,
    });

    // Audit log: log all agent actions to stdout (pipe to Splunk/Elastic in prod)
    console.log(JSON.stringify({
      timestamp: new Date().toISOString(),
      userId,
      input,
      output: response.output,
      intermediateSteps: response.intermediateSteps?.map((step) => ({
        tool: step.action.tool,
        input: step.action.toolInput,
        output: step.observation,
      })),
    }));

    return response.output;
  } catch (error) {
    // Structured error handling for agent failures
    console.error("Agent execution failed:", error);
    throw new Error(`Jira agent failed to process request: ${error.message}`);
  }
}

// benchmark.ts
import { runJiraAgent } from "./custom-jira-agent.js";
import { Client } from "langsmith"; // v0.2.0+ for LangChain benchmarking
import { faker } from "@faker-js/faker"; // v8.4.1 for generating test tickets
import { env } from "./env.js";

// Initialize LangSmith client for tracing and benchmark logging
const langsmith = new Client({
  apiUrl: env.LANGSMITH_API_URL || "https://api.smith.langchain.com",
  apiKey: env.LANGSMITH_API_KEY,
});

// Define 100 test cases matching real-world Jira ticket requests
const testCases = Array.from({ length: 100 }, (_, i) => ({
  id: `test-${i}`,
  input: faker.helpers.arrayElement([
    `Create a Bug ticket in project ENG with summary "${faker.hacker.phrase()}" and description "${faker.lorem.paragraph()}"`,
    `Search for all open Story tickets in project OPS assigned to ${faker.internet.userName()}`,
    `Update ticket ${faker.helpers.arrayElement(["ENG-123", "OPS-456", "SRE-789"])} to set priority to High`,
    `Create a Task ticket in project SRE with summary "Deploy ${faker.system.networkInterface()}" and assign to ${faker.internet.userName()}`,
  ]),
  expectedTool: faker.helpers.arrayElement(["create_jira_ticket", "search_jira_tickets", "update_jira_ticket"]),
}));

// Benchmark custom LangChain 0.30 agent
async function benchmarkCustomAgent() {
  const results = [];
  const start = Date.now();

  for (const testCase of testCases) {
    const caseStart = Date.now();
    try {
      const output = await runJiraAgent(testCase.input, "benchmark-user");
      const latency = Date.now() - caseStart;

      // Check if the correct tool was called (simplified check)
      const toolCalled = output.includes(testCase.expectedTool.split("_").join(" ")) || output.includes(testCase.expectedTool);
      results.push({
        testCaseId: testCase.id,
        latency,
        success: toolCalled,
        output,
      });
    } catch (error) {
      results.push({
        testCaseId: testCase.id,
        latency: Date.now() - caseStart,
        success: false,
        error: error.message,
      });
    }
  }

  const totalTime = Date.now() - start;
  const successRate = (results.filter((r) => r.success).length / results.length) * 100;
  const p50Latency = calculatePercentile(results.map((r) => r.latency), 50);
  const p99Latency = calculatePercentile(results.map((r) => r.latency), 99);

  return {
    agentType: "Custom LangChain 0.30 Agent",
    totalTime,
    successRate,
    p50Latency,
    p99Latency,
    results,
  };
}

// Helper to calculate percentile latency
function calculatePercentile(latencies: number[], percentile: number) {
  const sorted = [...latencies].sort((a, b) => a - b);
  const index = Math.floor((percentile / 100) * sorted.length);
  return sorted[index];
}

// Run benchmark and log results
async function main() {
  console.log("Starting Jira agent benchmark...");
  const customAgentResults = await benchmarkCustomAgent();

  // Log to LangSmith for persistent tracking
  await langsmith.createRun({
    name: "jira-agent-benchmark",
    inputs: { testCases: testCases.length },
    outputs: customAgentResults,
  });

  console.log("Benchmark Results:");
  console.log(`Agent Type: ${customAgentResults.agentType}`);
  console.log(`Success Rate: ${customAgentResults.successRate.toFixed(2)}%`);
  console.log(`p50 Latency: ${customAgentResults.p50Latency}ms`);
  console.log(`p99 Latency: ${customAgentResults.p99Latency}ms`);
  console.log(`Total Time: ${customAgentResults.totalTime}ms`);
}

main().catch(console.error);

const updateTicketSchema = z.object({
  ticketKey: z.string().regex(/^[A-Z]+-\d+$/, "Invalid Jira ticket key format (e.g. ENG-123)"),
  priority: z.enum(["Highest", "High", "Medium", "Low", "Lowest"]).optional(),
  assignee: z.string().min(3).max(50).optional(),
  status: z.enum(["To Do", "In Progress", "Done", "Blocked"]).optional(),
});

const llm = new ChatOpenAI({
  model: "llama3.1-70b",
  temperature: 0, // Critical: zero temperature for deterministic tool calls
  maxTokens: 4096,
  timeout: 30000,
  maxRetries: 2,
  configuration: {
    baseURL: "http://vllm-jira-agent:8000/v1", // Self-hosted vLLM endpoint
    apiKey: "local-llm", // Non-empty string required for OpenAI client
  },
});

const agentExecutor = new AgentExecutor({
  agent,
  tools,
  memory,
  maxIterations: 5, // Fail after 5 tool calls to prevent infinite loops
  earlyStoppingMethod: "force",
  returnIntermediateSteps: true, // Include tool calls in output for debugging
  handleParsingErrors: (error) => {
    console.error("Parsing error:", error);
    return "I encountered an error processing your request. Please provide clear Jira project and ticket details.";
  },
});