AI-Driven Automated Network Monitoring & Anomaly Detection — Part 2: Collecting Network Metrics via Bash Scripts and Exporters

global:
  scrape_interval: 15s
  evaluation_interval: 30s

scrape_configs:
  - job_name: 'node'
    static_configs:
      - targets: ['localhost:9100']

  - job_name: 'network_metrics'
    static_configs:
      - targets: ['localhost:9100']
    metrics_path: /metrics
    file_sd_configs:
      - files:
        - /etc/prometheus/file_sd/network_metrics.yml

#!/usr/bin/env bash
#
# network_if_exporter.sh – Export per‑interface stats for Prometheus
# Author: Vijay Vinoth (Lead Programmer Analyst)
# ---------------------------------------------------------

# Directory where Prometheus textfile collector reads files
OUT_DIR="/var/lib/prometheus/node_exporter/textfile_collector"
OUT_FILE="${OUT_DIR}/network_if.prom"

# Ensure the output directory exists
mkdir -p "${OUT_DIR}"

# Helper to emit a metric line
emit_metric() {
  local metric=$1
  local labels=$2
  local value=$3
  echo "${metric}${labels} ${value}"
}

# Header for Prometheus (optional but nice)
{
  echo "# HELP net_if_bytes_total Total bytes transmitted/received on the interface"
  echo "# TYPE net_if_bytes_total counter"
  echo "# HELP net_if_packets_total Total packets transmitted/received on the interface"
  echo "# TYPE net_if_packets_total counter"
  echo "# HELP net_if_errors_total Total error counters on the interface"
  echo "# TYPE net_if_errors_total counter"
} > "${OUT_FILE}"

# Loop over each physical interface (skip lo)
for IFACE in $(ls /sys/class/net | grep -v '^lo$'); do
  # Read counters from sysfs
  RX_BYTES=$(cat /sys/class/net/${IFACE}/statistics/rx_bytes)
  TX_BYTES=$(cat /sys/class/net/${IFACE}/statistics/tx_bytes)
  RX_PACKETS=$(cat /sys/class/net/${IFACE}/statistics/rx_packets)
  TX_PACKETS=$(cat /sys/class/net/${IFACE}/statistics/tx_packets)
  RX_ERRORS=$(cat /sys/class/net/${IFACE}/statistics/rx_errors)
  TX_ERRORS=$(cat /sys/class/net/${IFACE}/statistics/tx_errors)

  # Emit metrics with interface label
  emit_metric "net_if_bytes_total" "{interface=\"${IFACE}\",direction=\"rx\"}" "${RX_BYTES}" >> "${OUT_FILE}"
  emit_metric "net_if_bytes_total" "{interface=\"${IFACE}\",direction=\"tx\"}" "${TX_BYTES}" >> "${OUT_FILE}"
  emit_metric "net_if_packets_total" "{interface=\"${IFACE}\",direction=\"rx\"}" "${RX_PACKETS}" >> "${OUT_FILE}"
  emit_metric "net_if_packets_total" "{interface=\"${IFACE}\",direction=\"tx\"}" "${TX_PACKETS}" >> "${OUT_FILE}"
  emit_metric "net_if_errors_total" "{interface=\"${IFACE}\",direction=\"rx\"}" "${RX_ERRORS}" >> "${OUT_FILE}"
  emit_metric "net_if_errors_total" "{interface=\"${IFACE}\",direction=\"tx\"}" "${TX_ERRORS}" >> "${OUT_FILE}"
done

[Unit]
Description=Network Interface Prometheus Exporter
After=network.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/network_if_exporter.sh

[Install]
WantedBy=multi-user.target

[Unit]
Description=Run network_if_exporter every 15 seconds

[Timer]
OnBootSec=5sec
OnUnitActiveSec=15sec
Unit=network_if_exporter.service

[Install]
WantedBy=timers.target

#!/usr/bin/env bash
#
# snmp_if_exporter.sh – Pull SNMP interface counters and expose to Prometheus
# Requires: net-snmp, jq (for JSON conversion)
# ---------------------------------------------------------

# Configuration – replace with your own device list
DEVICES=(
  "router01|public|10.0.0.1"
  "switch02|public|10.0.0.2"
)

OUT_DIR="/var/lib/prometheus/node_exporter/textfile_collector"
OUT_FILE="${OUT_DIR}/snmp_if.prom"

mkdir -p "${OUT_DIR}"
{
  echo "# HELP snmp_if_bytes_total Total bytes on SNMP‑managed interfaces"
  echo "# TYPE snmp_if_bytes_total counter"
  echo "# HELP snmp_if_oper_status Current operational status (1=up,2=down,3=testing)"
  echo "# TYPE snmp_if_oper_status gauge"
} > "${OUT_FILE}"

# OIDs we care about
OID_IN_OCTETS="IF-MIB::ifHCInOctets"
OID_OUT_OCTETS="IF-MIB::ifHCOutOctets"
OID_OPER_STATUS="IF-MIB::ifOperStatus"

for ENTRY in "${DEVICES[@]}"; do
  IFS='|' read -r NAME COMMUNITY HOST << EOF
$ENTRY
EOF

  # Pull data via SNMP bulk walk
  SNMP_DATA=$(snmpbulkwalk -v2c -c "${COMMUNITY}" "${HOST}" "${OID_IN_OCTETS}" "${OID_OUT_OCTETS}" "${OID_OPER_STATUS}" 2>/dev/null)

  # Parse each line
  while IFS= read -r line; do
    # Example line: IF-MIB::ifHCInOctets.1 = Counter64: 123456789
    if [[ $line =~ ^([A-Za-z0-9-]+)::([A-Za-z0-9]+)\.([0-9]+)\ =\ (.+):\ (.+)$ ]]; then
      MIB="${BASH_REMATCH[1]}"
      METRIC="${BASH_REMATCH[2]}"
      IFINDEX="${BASH_REMATCH[3]}"
      TYPE="${BASH_REMATCH[4]}"
      VALUE="${BASH_REMATCH[5]}"

      case "$METRIC" in
        ifHCInOctets)
          echo "snmp_if_bytes_total{device=\"${NAME}\",interface=\"${IFINDEX}\",direction=\"rx\"} ${VALUE}" >> "${OUT_FILE}"
          ;;
        ifHCOutOctets)
          echo "snmp_if_bytes_total{device=\"${NAME}\",interface=\"${IFINDEX}\",direction=\"tx\"} ${VALUE}" >> "${OUT_FILE}"
          ;;
        ifOperStatus)
          echo "snmp_if_oper_status{device=\"${NAME}\",interface=\"${IFINDEX}\"} ${VALUE}" >> "${OUT_FILE}"
          ;;
      esac
    fi
  done  "${OUT_FILE}"

# nvidia-smi JSON format (requires driver >= 450)
JSON=$(nvidia-smi --query-gpu=index,utilization.gpu,memory.used,temperature.gpu --format=csv,noheader,nounits)

while IFS=',' read -r IDX UTIL MEM TEMP; do
  IDX=$(echo $IDX | xargs)   # trim spaces
  UTIL=$(echo $UTIL | xargs)
  MEM=$(echo $MEM | xargs)
  TEMP=$(echo $TEMP | xargs)

  echo "gpu_utilization_percent{gpu=\"${IDX}\"} ${UTIL}" >> "${OUT_FILE}"
  echo "gpu_memory_used_bytes{gpu=\"${IDX}\"} $((MEM * 1024 * 1024))" >> "${OUT_FILE}"
  echo "gpu_temperature_celsius{gpu=\"${IDX}\"} ${TEMP}" >> "${OUT_FILE}"
done 
- Use `promtool tsdb create-blocks-from` or a `remote_write` endpoint to stream data into a vector database (e.g., Milvus).
- Launch a **Claude 4.6 Opus** agentic workflow that:

    Normalizes the series (z‑score per interface).
    - Feeds sliding windows (e.g., 5‑minute) into a pre‑trained LSTM or a transformer model hosted on a GPU node.
    - Generates an anomaly score (0‑1) and writes it back as `network_anomaly_score` metric.


- Configure Prometheus alerting rules to fire when `network_anomaly_score > 0.8` for more than two consecutive scrapes.

This architecture mirrors the AI‑driven workflow demonstrated by IBM, where “AI‑enabled workflows—many driven by agentic AI—are poised to expand from 3 % in 2024 to 25 % by 2026” [IBM AI Network Monitoring](https://www.ibm.com/think/topics/ai-network-monitoring). By keeping the ingestion layer simple and observable (Bash + Prometheus), you give the AI agents a reliable foundation to reason over.

### 6. Adding Alerting & Incident Automation

Prometheus alone can fire alerts, but modern NOC stacks push those alerts into ticketing or chat tools. The Medium AIOps post outlines integration with ZenDesk, ServiceNow, and Slack. Below is a minimal `alertmanager.yml` snippet that routes high‑severity network anomalies to Slack.

Corresponding Prometheus rule (saved as `network_anomaly.rules.yml`):

When the AI model spikes the `network_anomaly_score`, the alert fires, Slack notifies the team, and a parallel GPT‑5.4 Pro agent can automatically open a ServiceNow ticket, attach the last 10 minutes of metric graphs, and suggest a remediation playbook.

### 7. Testing the Exporters – A Quick Lab

To verify everything works before you push to production, spin up a Docker‑Compose stack that includes Prometheus, node_exporter, and a tiny `bash` container that runs the scripts.

version: '3.8'
services:
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./file_sd:/etc/prometheus/file_sd
      - prom_data:/prometheus
    ports:
      - "9090:9090"
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'

  node_exporter:
    image: prom/node-exporter:latest
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'

  exporter_host:
    image: alpine:3.19
    depends_on:
      - node_exporter
    volumes:
      - ./exporters:/usr/local/bin
      - ./node_exporter_text:/var/lib/prometheus/node_exporter/textfile

---
*Originally published at [https://artificial-inteligence.phptutorial.co.in](https://artificial-inteligence.phptutorial.co.in/ai-driven-automated-network-monitoring-anomaly-detection-part-2-collecting-network-metrics-via-bash-scripts-and-exporters/)*