Automate Data Visualization: Our Experience

import os
import logging
import time
from typing import Dict, List, Optional, Any
from dataclasses import dataclass
from sqlalchemy import create_engine, text
from plotly import graph_objects as go
from plotly.subplots import make_subplots
import pandas as pd
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type

# Configure logging for audit trails and debugging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

@dataclass
class ChartRequest:
    """Data class holding all required parameters for a chart generation request"""
    chart_id: str
    chart_type: str  # line, bar, scatter, heatmap
    data_source: str  # SQL query or API endpoint
    filters: Dict[str, Any]
    output_path: str
    retention_days: int = 30

class ChartOrchestrator:
    def __init__(self, db_connection_string: str, plotly_config: Optional[Dict] = None):
        self.engine = create_engine(db_connection_string)
        self.plotly_config = plotly_config or {"displayModeBar": False}
        self.supported_chart_types = {"line", "bar", "scatter", "heatmap"}
        logger.info(f"Initialized ChartOrchestrator with supported types: {self.supported_chart_types}")

    @retry(
        stop=stop_after_attempt(3),
        wait=wait_exponential(multiplier=1, min=4, max=10),
        retry=retry_if_exception_type((pd.errors.DatabaseError, ConnectionError)),
        after=lambda retry_state: logger.warning(f"Retry attempt {retry_state.attempt_number} for data fetch")
    )
    def _fetch_data(self, data_source: str, filters: Dict[str, Any]) -> pd.DataFrame:
        """Fetch data from SQL source with retry logic for transient failures"""
        try:
            # Sanitize filters to prevent SQL injection (basic example, use parameterized queries in prod)
            sanitized_filters = {k: str(v).replace("'", "''") for k, v in filters.items()}
            query = text(data_source).bindparams(**sanitized_filters)
            with self.engine.connect() as conn:
                df = pd.read_sql(query, conn)
            logger.info(f"Fetched {len(df)} rows from data source")
            if df.empty:
                raise ValueError("Query returned empty dataset")
            return df
        except Exception as e:
            logger.error(f"Data fetch failed: {str(e)}")
            raise

    def _validate_chart_request(self, request: ChartRequest) -> None:
        """Validate all chart request parameters before processing"""
        if request.chart_type not in self.supported_chart_types:
            raise ValueError(f"Unsupported chart type: {request.chart_type}. Supported: {self.supported_chart_types}")
        if not os.path.isdir(os.path.dirname(request.output_path)):
            raise FileNotFoundError(f"Output directory does not exist: {os.path.dirname(request.output_path)}")
        logger.info(f"Validated chart request {request.chart_id}")

    def _render_chart(self, df: pd.DataFrame, request: ChartRequest) -> go.Figure:
        """Render the appropriate chart type based on request parameters"""
        if request.chart_type == "line":
            fig = go.Figure(data=[go.Scatter(x=df.iloc[:, 0], y=df.iloc[:, 1], mode="lines+markers")])
        elif request.chart_type == "bar":
            fig = go.Figure(data=[go.Bar(x=df.iloc[:, 0], y=df.iloc[:, 1])])
        elif request.chart_type == "scatter":
            fig = go.Figure(data=[go.Scatter(x=df.iloc[:, 0], y=df.iloc[:, 1], mode="markers")])
        elif request.chart_type == "heatmap":
            fig = go.Figure(data=[go.Heatmap(z=df.values, x=df.columns, y=df.index)])
        else:
            raise ValueError(f"Unimplemented chart type: {request.chart_type}")

        fig.update_layout(
            title=f"Automated Chart: {request.chart_id}",
            xaxis_title=df.columns[0],
            yaxis_title=df.columns[1],
            **self.plotly_config
        )
        return fig

    def generate_chart(self, request: ChartRequest) -> str:
        """Main entry point to generate a chart, returns path to output file"""
        start_time = time.time()
        try:
            self._validate_chart_request(request)
            df = self._fetch_data(request.data_source, request.filters)
            fig = self._render_chart(df, request)
            output_path = f"{request.output_path}/{request.chart_id}.html"
            fig.write_html(output_path)
            # Apply retention policy: delete charts older than retention_days
            self._apply_retention_policy(request.output_path, request.retention_days)
            logger.info(f"Generated chart {request.chart_id} in {time.time() - start_time:.2f}s")
            return output_path
        except Exception as e:
            logger.error(f"Chart generation failed for {request.chart_id}: {str(e)}")
            raise

    def _apply_retention_policy(self, output_dir: str, retention_days: int) -> None:
        """Delete chart files older than retention_days to control storage costs"""
        current_time = time.time()
        for filename in os.listdir(output_dir):
            filepath = os.path.join(output_dir, filename)
            if os.path.isfile(filepath) and filename.endswith(".html"):
                file_mtime = os.path.getmtime(filepath)
                if (current_time - file_mtime) > (retention_days * 86400):
                    os.remove(filepath)
                    logger.info(f"Deleted expired chart: {filepath}")

if __name__ == "__main__":
    # Example usage with error handling
    try:
        orchestrator = ChartOrchestrator(
            db_connection_string="postgresql://user:pass@localhost:5432/analytics",
            plotly_config={"displayModeBar": True, "responsive": True}
        )
        request = ChartRequest(
            chart_id="monthly_revenue_2024_10",
            chart_type="line",
            data_source="SELECT month, revenue FROM monthly_revenue WHERE year = :year",
            filters={"year": 2024},
            output_path="/var/charts/monthly_revenue",
            retention_days=30
        )
        output = orchestrator.generate_chart(request)
        print(f"Chart generated at: {output}")
    except Exception as e:
        print(f"Failed to generate chart: {str(e)}")
        exit(1)

import React, { useState, useEffect, useCallback } from "react";
import axios, { AxiosError } from "axios";
import useSWR, { SWRConfiguration } from "swr";
import { PlotParams, usePlotly } from "react-plotly.js";
import { Spinner, Alert, AlertDescription, AlertTitle, Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui";
import { ChartType } from "@/types/analytics";

// Configure SWR for caching and revalidation
const swrConfig: SWRConfiguration = {
  revalidateOnFocus: false,
  revalidateOnReconnect: true,
  dedupingInterval: 5000,
  errorRetryCount: 3,
};

// Fetcher function with error handling
const fetcher = async (url: string) => {
  try {
    const response = await axios.get(url, {
      timeout: 10000, // 10s timeout for chart data fetches
      headers: { "Content-Type": "application/json" },
    });
    return response.data;
  } catch (error) {
    const axiosError = error as AxiosError;
    throw new Error(`Failed to fetch chart data: ${axiosError.message} (Status: ${axiosError.response?.status || "unknown"})`);
  }
};

interface AutomatedChartProps {
  chartId: string;
  initialChartType?: ChartType;
  refreshInterval?: number; // in seconds
  onError?: (error: Error) => void;
}

type ChartData = {
  chart_id: string;
  chart_type: ChartType;
  data: Array<{ x: number | string; y: number }>;
  layout: PlotParams["layout"];
  last_updated: string;
};

const SUPPORTED_CHART_TYPES: ChartType[] = ["line", "bar", "scatter", "heatmap"];

export const AutomatedChart: React.FC = ({
  chartId,
  initialChartType = "line",
  refreshInterval = 300, // 5 minutes default
  onError,
}) => {
  const [selectedChartType, setSelectedChartType] = useState(initialChartType);
  const { Plot } = usePlotly(); // Lazy load Plotly to reduce initial bundle size
  const [isPlotlyLoaded, setIsPlotlyLoaded] = useState(false);

  // Load Plotly dynamically to avoid bundling large library upfront
  useEffect(() => {
    import("plotly.js-dist-min").then(() => {
      setIsPlotlyLoaded(true);
    }).catch((err) => {
      console.error("Failed to load Plotly:", err);
      onError?.(new Error("Plotly library failed to load"));
    });
  }, []);

  // Fetch chart data with SWR caching
  const { data, error, isLoading, mutate } = useSWR(
    `/api/charts/${chartId}?type=${selectedChartType}`,
    fetcher,
    {
      ...swrConfig,
      refreshInterval: refreshInterval * 1000,
      onError: (err) => {
        console.error(`Chart ${chartId} fetch error:`, err);
        onError?.(err);
      },
    }
  );

  // Handle chart type change with optimistic update
  const handleChartTypeChange = useCallback(
    (newType: ChartType) => {
      setSelectedChartType(newType);
      // Optimistically revalidate with new type
      mutate();
    },
    [mutate]
  );

  // Format last updated time for display
  const formatLastUpdated = useCallback((isoString: string) => {
    try {
      const date = new Date(isoString);
      return date.toLocaleString("en-US", {
        month: "short",
        day: "numeric",
        year: "numeric",
        hour: "2-digit",
        minute: "2-digit",
      });
    } catch {
      return "Unknown";
    }
  }, []);

  if (isLoading || !isPlotlyLoaded) {
    return (


        Loading chart...

    );
  }

  if (error) {
    return (

        Failed to load chart
        {error.message}

    );
  }

  if (!data) {
    return (

        No data available
        No data found for chart ID: {chartId}

    );
  }

  return (


        Automated Chart: {data.chart_id}






              {SUPPORTED_CHART_TYPES.map((type) => (

                  {type.charAt(0).toUpperCase() + type.slice(1)}

              ))}



            Last updated: {formatLastUpdated(data.last_updated)}







  );
};

export default AutomatedChart;

name: Automated Chart Validation Pipeline

on:
  push:
    branches: [ main, release/* ]
    paths:
      - "src/charting/**"
      - "tests/charting/**"
      - ".github/workflows/chart-validation.yml"
  pull_request:
    branches: [ main ]
    paths:
      - "src/charting/**"
      - "tests/charting/**"

env:
  PYTHON_VERSION: "3.11"
  POETRY_VERSION: "1.7.1"
  CHART_TEST_DIR: "tests/charting"
  BENCHMARK_THRESHOLD_MS: 2000 # Fail if chart render time exceeds 2s

jobs:
  validate-charts:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        chart-type: [line, bar, scatter, heatmap]
      fail-fast: false # Run all chart types even if one fails

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch full history for benchmark comparison

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Poetry ${{ env.POETRY_VERSION }}
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: ${{ env.POETRY_VERSION }}

      - name: Install dependencies
        run: |
          poetry install --no-interaction --no-ansi
          # Install additional test dependencies
          poetry run pip install pytest pytest-benchmark plotly pandas sqlalchemy

      - name: Run unit tests for chart orchestrator
        run: |
          poetry run pytest ${{ env.CHART_TEST_DIR }}/unit/ -v --tb=short
        continue-on-error: false

      - name: Run integration tests for ${{ matrix.chart-type }} charts
        run: |
          poetry run pytest ${{ env.CHART_TEST_DIR }}/integration/ -v --tb=short -k "chart_type == '${{ matrix.chart-type }}'"
        env:
          TEST_DB_URL: "sqlite:///test.db" # Use in-memory SQLite for CI tests

      - name: Run performance benchmarks
        id: benchmark
        run: |
          # Run benchmark for current chart type, output JSON results
          poetry run pytest ${{ env.CHART_TEST_DIR }}/benchmark/ -v --benchmark-json=benchmark-results-${{ matrix.chart-type }}.json -k "chart_type == '${{ matrix.chart-type }}'"
          # Parse benchmark results to get mean render time
          MEAN_TIME_MS=$(cat benchmark-results-${{ matrix.chart-type }}.json | jq '.benchmarks[0].stats.mean * 1000 | floor')
          echo "mean_render_time_ms=$MEAN_TIME_MS" >> $GITHUB_OUTPUT
          echo "Chart type: ${{ matrix.chart-type }}, Mean render time: $MEAN_TIME_MS ms"

      - name: Check benchmark regression
        if: github.event_name == 'pull_request'
        run: |
          # Fetch baseline benchmark from main branch
          git checkout origin/main -- benchmark-baseline.json
          BASELINE_MS=$(cat benchmark-baseline.json | jq '.${{ matrix.chart-type }} // 0')
          CURRENT_MS=${{ steps.benchmark.outputs.mean_render_time_ms }}
          echo "Baseline: $BASELINE_MS ms, Current: $CURRENT_MS ms"
          if [ $CURRENT_MS -gt $((BASELINE_MS * 120 / 100)) ]; then
            echo "::error::Performance regression detected for ${{ matrix.chart-type }}: Current ${CURRENT_MS}ms exceeds baseline ${BASELINE_MS}ms by more than 20%"
            exit 1
          fi

      - name: Upload benchmark results
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: benchmark-results-${{ matrix.chart-type }}
          path: benchmark-results-${{ matrix.chart-type }}.json
          retention-days: 7

      - name: Update baseline benchmarks (main branch only)
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        run: |
          # Update baseline with latest benchmark results
          CURRENT_MS=${{ steps.benchmark.outputs.mean_render_time_ms }}
          jq '. + {"${{ matrix.chart-type }}": $CURRENT_MS}' benchmark-baseline.json > tmp.json
          mv tmp.json benchmark-baseline.json
          git config --global user.name "Chart Benchmark Bot"
          git config --global user.email "bot@example.com"
          git add benchmark-baseline.json
          git commit -m "Update baseline benchmark for ${{ matrix.chart-type }} to ${CURRENT_MS}ms"
          git push origin main

{
  "$schema": "https://json-schema.org/draft/2023-09/schema",
  "title": "ChartRequest",
  "version": "1.2.0",
  "type": "object",
  "properties": {
    "chart_id": { "type": "string", "pattern": "^[a-z0-9_-]+$" },
    "chart_type": { "enum": ["line", "bar", "scatter", "heatmap"] },
    "data_source": { "type": "string", "minLength": 10 },
    "filters": { "type": "object" },
    "output_path": { "type": "string", "format": "uri-path" }
  },
  "required": ["chart_id", "chart_type", "data_source", "output_path"]
}

# Good: Lightweight static export with Plotly + Kaleido
import plotly.graph_objects as go
fig = go.Figure(data=[go.Bar(x=["A", "B"], y=[10, 20])])
fig.write_image("chart.png", width=800, height=600)  # Uses Kaleido, no browser

# Avoid unless necessary: Puppeteer for simple charts
# const puppeteer = require("puppeteer");
# const browser = await puppeteer.launch(); // 150MB+ RAM per instance

import plotly.graph_objects as go
from plotly_accessibility import add_accessibility

fig = go.Figure(data=[go.Line(x=[1,2,3], y=[4,5,6])])
fig.update_layout(
    title="Monthly Revenue 2024",
    # Mandatory alt text for screen readers
    meta={"description": "Line chart showing monthly revenue growth from $10k in Jan 2024 to $45k in Oct 2024"}
)
# Add ARIA labels automatically
add_accessibility(fig)
fig.write_html("revenue_chart.html")  # Includes accessibility tags