[Application Layer]  ViciDial disposition codes, agent states, call routing
        |
[Signaling Layer]    SIP INVITE/BYE/CANCEL, 4xx/5xx responses, SDP negotiation
        |
[Media Layer]        RTP streams, codecs, jitter buffers, packet loss
        |
[Network Layer]      Latency, packet loss, MTU, firewall rules, NAT traversal
        |
[Infrastructure]     Server load, disk space, DNS resolution, time sync

-- Search by phone number in the inbound call log
SELECT call_date, phone_number, length_in_sec, status, term_reason,
       uniqueid, user, campaign_id, queue_seconds, closecallid
FROM vicidial_closer_log
WHERE phone_number LIKE '%PHONE_NUMBER%'
ORDER BY call_date DESC
LIMIT 20;

-- Search by phone number in the outbound call log
SELECT call_date, phone_number, length_in_sec, status, term_reason,
       uniqueid, user, campaign_id, list_id
FROM vicidial_log
WHERE phone_number LIKE '%PHONE_NUMBER%'
ORDER BY call_date DESC
LIMIT 20;

-- Find all abnormal disconnects in the last 24 hours
SELECT call_date, phone_number, status, term_reason, uniqueid, user
FROM vicidial_closer_log
WHERE status IN ('DISMX', 'DCMX')
  AND call_date > NOW() - INTERVAL 24 HOUR
ORDER BY call_date DESC;

-- Find calls that dropped within 15 seconds (possible one-way audio)
SELECT call_date, phone_number, length_in_sec, status, term_reason, user
FROM vicidial_closer_log
WHERE length_in_sec < 15
  AND length_in_sec > 0
  AND term_reason = 'CALLER'
  AND call_date > NOW() - INTERVAL 24 HOUR
ORDER BY call_date DESC;

v=0
o=- 12345 12345 IN IP4 203.0.113.50
c=IN IP4 203.0.113.50          <-- Media IP (should be public)
m=audio 18000 RTP/AVP 0 8 18   <-- RTP port and codec offers
a=rtpmap:0 PCMU/8000            <-- G.711 ulaw
a=rtpmap:8 PCMA/8000            <-- G.711 alaw
a=rtpmap:18 G729/8000           <-- G.729

# Live capture of SIP traffic on port 5060 (run during a test call)
tcpdump -i eth0 -n -s 0 port 5060 -w /tmp/sip-capture.pcap

# Then analyze with sngrep (if installed) or download the pcap
sngrep -I /tmp/sip-capture.pcap

# Or use ngrep for real-time SIP message viewing
ngrep -W byline -d eth0 port 5060

# Using the uniqueid from Step 1, search the Asterisk message log
grep 'UNIQUEID_FROM_STEP_1' /var/log/asterisk/messages

# If you have a Call-ID (C-XXXXXXXX format), search by that
grep 'C-XXXXXXXX' /var/log/asterisk/messages

# Search by phone number to find the Call-ID first
grep 'PHONE_NUMBER' /var/log/asterisk/messages | tail -50

-- Executing [s@default:1] Answer()
-- Executing [s@default:2] Dial(SIP/agent_100,,tT)
-- SIP/agent_100-00001234 answered SIP/trunk_provider-00001235
-- Channel SIP/trunk_provider-00001235 joined bridge
-- Channel SIP/agent_100-00001234 joined bridge

chan_iax2.c: Resyncing the jb -- Loss: 0.0024  Delay: 15234  Jit: 0.03

# Count jitter buffer resyncs today
grep "$(date +%Y-%m-%d)" /var/log/asterisk/messages | grep -c 'Resyncing the jb'
# More than 100/day = problem; more than 1000/day = critical

res_rtp_asterisk.c: Strict RTP switching source address to 198.51.100.25:12345

app_dial.c: Called SIP/trunk_provider/441234567890
app_dial.c: SIP/trunk_provider-0001 is circuit-busy

-- Executing [h@default:1] NoOp("SIP/trunk-0001", "Hangup cause: 16")

func_hangupcause.c: Unable to find information for channel
bridge_channel.c: Channel SIP/agent_100-0001 left bridge

-- Check carrier log for the specific call (use uniqueid from Step 1)
SELECT call_date, dialstatus, hangup_cause, sip_hangup_cause,
       sip_hangup_reason, dial_time, answered_time, channel
FROM vicidial_carrier_log
WHERE uniqueid = 'UNIQUEID_FROM_STEP_1';

-- Or search by trunk name for recent failures
SELECT call_date, dialstatus, hangup_cause, sip_hangup_cause,
       sip_hangup_reason, channel
FROM vicidial_carrier_log
WHERE channel LIKE '%TRUNK_NAME%'
ORDER BY call_date DESC
LIMIT 20;

-- Aggregate trunk health: failure rate in the last hour
SELECT
    SUBSTRING_INDEX(channel, '-', 1) AS trunk,
    COUNT(*) AS total_calls,
    SUM(CASE WHEN dialstatus = 'ANSWER' THEN 1 ELSE 0 END) AS answered,
    SUM(CASE WHEN dialstatus = 'BUSY' THEN 1 ELSE 0 END) AS busy,
    SUM(CASE WHEN dialstatus = 'NOANSWER' THEN 1 ELSE 0 END) AS noanswer,
    SUM(CASE WHEN dialstatus = 'CHANUNAVAIL' THEN 1 ELSE 0 END) AS unavail,
    SUM(CASE WHEN dialstatus = 'CONGESTION' THEN 1 ELSE 0 END) AS congestion
FROM vicidial_carrier_log
WHERE call_date > NOW() - INTERVAL 1 HOUR
GROUP BY trunk
ORDER BY total_calls DESC;

# Recordings are stored by date. Find by the call date from Step 1:
find /var/spool/asterisk/monitorDONE/YYYYMMDD/ -name "*UNIQUEID*" -o -name "*PHONE_NUMBER*"

# Or search the ViciDial recordings table:
mysql -u USER -pPASSWORD DATABASE -e "
    SELECT recording_id, filename, location, start_time, length_in_sec
    FROM recording_log
    WHERE lead_id = LEAD_ID
    ORDER BY start_time DESC
    LIMIT 10;
"

# If you have deployed the AI audio analysis service (Tutorial 02):
curl "http://YOUR_MONITORING_SERVER:8084/analyze?recording_url=http://YOUR_VICIDIAL_SERVER/RECORDINGS/path/to/file.wav"

# This returns NISQA MOS scores, silence detection, and AI-generated analysis

# Show RTP channel statistics for all active calls
asterisk -rx 'sip show channelstats'

Peer             Call ID      Duration Recv: Pack  Lost       (     %) Jitter Send: Pack  Lost       (     %) Jitter
203.0.113.50     2cb43240375  00:44:44 0000000134K 0000000008 ( 0.01%) 0.0019 0000000134K 0000000016 ( 0.01%) 0.0004
198.51.100.25    9af21c9f-99  00:01:58 0000005766  0000000000 ( 0.00%) 0.0000 0000005892  0000000001 ( 0.02%) 0.0000

# Ping the trunk provider (check latency and loss)
ping -c 20 TRUNK_PROVIDER_IP

# Traceroute to identify where latency is introduced
traceroute -n TRUNK_PROVIDER_IP

# Check for asymmetric routing (can cause one-way audio)
mtr --report --report-cycles 50 TRUNK_PROVIDER_IP

# Verify RTP port range is open in firewall
iptables -L INPUT -n --line-numbers | grep -E "udp.*10000|udp.*20000"
# You should see an ACCEPT rule for UDP ports 10000:20000 (or your configured range)

# Check RTP configuration
asterisk -rx 'rtp show settings'

# Read the configured RTP port range
grep -E 'rtpstart|rtpend' /etc/asterisk/rtp.conf
# Example output:
#   rtpstart=10000
#   rtpend=20000

# Verify firewall allows that UDP range
iptables -S INPUT | grep -E 'udp.*dport.*(10000|20000)'
# Must see: -A INPUT -p udp --dport 10000:20000 -j ACCEPT

# Show detailed SIP peer information for a specific agent extension
asterisk -rx 'sip show peer AGENT_EXTENSION'

# Show all SIP peers and their qualification status
asterisk -rx 'sip show peers' | grep -E 'LAGGED|UNREACHABLE|UNKNOWN'

-- Check LAGGED agents in ViciDial's live agent table
SELECT user, server_ip, status, callerid, last_update_time,
       TIMESTAMPDIFF(SECOND, last_update_time, NOW()) AS seconds_since_update
FROM vicidial_live_agents
WHERE status = 'LAGGED'
   OR TIMESTAMPDIFF(SECOND, last_update_time, NOW()) > 30;

-- All agents currently logged in with their state
SELECT la.user, vu.full_name, la.status, la.callerid,
       la.campaign_id, la.phone_login, la.server_ip,
       TIMESTAMPDIFF(SECOND, la.last_update_time, NOW()) AS idle_seconds
FROM vicidial_live_agents la
JOIN vicidial_users vu ON la.user = vu.user
ORDER BY la.status, idle_seconds DESC;

# 1. Check the agent's SIP peer for NAT settings
asterisk -rx 'sip show peer AGENT_EXT' | grep -E 'Nat|Addr|Status'

# 2. Check if strict RTP is interfering
grep 'Strict RTP' /var/log/asterisk/messages | tail -10

# 3. Check RTP settings
grep strictrtp /etc/asterisk/rtp.conf

; In sip.conf or sip-vicidial.conf, for the agent's peer definition:
[agent_100]
nat=force_rport,comedia    ; Force symmetric RTP and rport
qualify=yes                ; Keep NAT pinholes open
qualifyfreq=30             ; Qualify every 30 seconds
directmedia=no             ; Force media through Asterisk (never direct)

; Try seqno mode instead of full strict RTP
strictrtp=seqno

; Add to the trunk peer definition
nat=force_rport,comedia

# Check qualify time for the agent
asterisk -rx 'sip show peer AGENT_EXT' | grep -E 'Status|Qualify|Addr'

# Check all LAGGED peers
asterisk -rx 'sip show peers' | grep LAGGED

; Increase qualify tolerance for remote agents:
[agent_100]
qualify=5000         ; Allow up to 5 seconds (default is 2000ms)
qualifyfreq=60       ; Check every 60 seconds instead of 30

-- Find mid-call disconnects (abnormal dispositions)
SELECT call_date, phone_number, length_in_sec, status, term_reason, uniqueid
FROM vicidial_closer_log
WHERE status IN ('DISMX', 'DCMX')
  AND call_date > NOW() - INTERVAL 24 HOUR
ORDER BY call_date DESC;

# Check if there is a pattern in the call duration
# SIP session timers expire at fixed intervals (commonly 1800s = 30 min)
# If calls consistently drop at the same duration, it is a timer issue

# Check Asterisk log for the specific call
grep 'UNIQUEID' /var/log/asterisk/messages | grep -i 'hangup\|destroy\|bye'

# 1. Check trunk status
asterisk -rx 'sip show peers' | grep TRUNK_NAME

# 2. Ping the trunk provider
ping -c 5 TRUNK_PROVIDER_IP

# 3. Check if the IP is whitelisted in the firewall
iptables -S INPUT | grep TRUNK_PROVIDER_IP

# 4. Check if SIP port 5060 is reachable
nc -zvu TRUNK_PROVIDER_IP 5060

# 5. Check DNS resolution (if trunk uses hostname)
dig TRUNK_HOSTNAME

# 6. Manually send a SIP OPTIONS to test connectivity
sipvicious_svmap TRUNK_PROVIDER_IP  # or use sipsak if installed
sipsak -vv -s sip:TRUNK_PROVIDER_IP:5060

# After fixing, reload SIP configuration (does not drop active calls):
asterisk -rx 'sip reload'

# Verify the trunk comes back:
asterisk -rx 'sip show peer TRUNK_NAME' | grep Status

# Check what codecs the trunk supports vs what it is offered
asterisk -rx 'sip show peer TRUNK_NAME' | grep -A5 Codecs

# Check active call codec negotiation
asterisk -rx 'core show channels verbose' | grep -E 'Codec|Format'

# Look for transcoding (uses CPU, degrades quality)
asterisk -rx 'core show translation'

; Ensure trunk and agent peers agree on codecs
; In the trunk peer definition:
[trunk_provider]
disallow=all
allow=alaw          ; G.711 A-law (European standard)
allow=ulaw          ; G.711 Mu-law (North American standard)
allow=g729          ; G.729 (bandwidth-efficient, requires license)

; In the agent peer definition -- match the trunk:
[agent_100]
disallow=all
allow=alaw
allow=ulaw

# 1. Verify the agent is behind NAT (their registered IP will be public, not private)
asterisk -rx 'sip show peer AGENT_EXT' | grep 'Addr->IP'

# 2. Check if the SIP ALG is interfering (common on consumer routers)
# The agent needs to disable SIP ALG on their router
# Signs of SIP ALG: mangled Contact headers, wrong port in Via

# 3. Verify NAT settings in the peer config
grep -A 20 '\[AGENT_EXT\]' /etc/asterisk/sip-vicidial.conf | grep nat

# 4. Check if RTP ports are open
iptables -L INPUT -n | grep -E "udp.*(10000|20000)"

[agent_100]
nat=force_rport,comedia
qualify=yes
qualifyfreq=25
directmedia=no
; directmedia=no forces ALL media through Asterisk.
; Without this, Asterisk may try to send RTP directly between
; the trunk and agent -- which fails if the agent is behind NAT.

; In [general] section of sip.conf:
localnet=10.0.0.0/8          ; Define your local networks
localnet=172.16.0.0/12
localnet=192.168.0.0/16
externaddr=YOUR_PUBLIC_IP     ; Your server's public IP

# Check live RTP stats for all active calls
asterisk -rx 'sip show channelstats'

# Check for IAX2 jitter buffer issues (if using IAX trunks or ConfBridge with IAX2)
grep -c 'Resyncing the jb' /var/log/asterisk/messages

# Check today's jitter buffer resyncs
grep "$(date +%b\ %d)" /var/log/asterisk/messages | grep -c 'Resyncing the jb'

; In iax.conf, disable the jitter buffer for the local loopback:
[general]
jitterbuffer=no
forcejitterbuffer=no

; Enable Asterisk-side jitter buffer for the agent peer
[agent_100]
jbimpl=adaptive
jbmaxsize=200       ; Maximum jitter buffer in milliseconds
jbresyncthreshold=1000
jblog=no

-- Check if DISMX/DCMX correlates with specific agents
SELECT user, COUNT(*) AS abnormal_count
FROM vicidial_closer_log
WHERE status IN ('DISMX', 'DCMX')
  AND call_date > NOW() - INTERVAL 7 DAY
GROUP BY user
ORDER BY abnormal_count DESC
LIMIT 10;
-- If one agent has significantly more than others, it is their connection

-- Check if DISMX/DCMX correlates with time of day
SELECT HOUR(call_date) AS hour, COUNT(*) AS count
FROM vicidial_closer_log
WHERE status IN ('DISMX', 'DCMX')
  AND call_date > NOW() - INTERVAL 7 DAY
GROUP BY hour
ORDER BY hour;
-- Spikes at specific hours suggest network congestion patterns

# Check active conferences
asterisk -rx 'confbridge list'          # For ConfBridge
asterisk -rx 'meetme list'              # For MeetMe

# Check for zombie conferences (conferences with 0 or 1 participant that persist)
asterisk -rx 'confbridge list' | awk '$2 <= 1 {print}'

# Check DAHDI timing (required for MeetMe)
asterisk -rx 'dahdi show status'
# Should show timer device with accuracy close to 100%

# Check for stuck channels in conferences
asterisk -rx 'core show channels' | grep -i conf

# Kick all users from a specific conference
asterisk -rx 'confbridge kick CONF_NUMBER all'

# Or for MeetMe:
asterisk -rx 'meetme kick CONF_NUMBER all'

# Check if DAHDI timer module is loaded
lsmod | grep dahdi

# If not loaded:
modprobe dahdi
dahdi_genconf
dahdi_cfg -vv

# Verify timing accuracy
cat /proc/dahdi/timer

# Check disk space (most common cause)
df -h /var/spool/asterisk/monitorDONE/
# If > 90% full, recordings will fail silently

# Check recording directory permissions
ls -la /var/spool/asterisk/monitor/
# Owner should be asterisk:asterisk with write permission

# Check for 0-byte recording files today
find /var/spool/asterisk/monitorDONE/$(date +%Y%m%d)/ -size 0 -name "*.wav" | wc -l

# Check Asterisk recording settings
asterisk -rx 'core show settings' | grep -i record

# Verify MixMonitor/Monitor is running on active calls
asterisk -rx 'core show channels verbose' | grep -i mix

CALLER REPORTS NO AUDIO
│
├── Is it ONE-WAY or BOTH-WAY silence?
│   │
│   ├── ONE-WAY (caller hears agent but agent can't hear caller, or vice versa)
│   │   │
│   │   ├── Check: Is agent behind NAT?
│   │   │   ├── YES → Verify nat=force_rport,comedia and directmedia=no
│   │   │   │         Also check: Is SIP ALG disabled on agent's router?
│   │   │   └── NO  → Check firewall: Are RTP ports (10000-20000 UDP) open?
│   │   │
│   │   ├── Check: Does carrier SDP show private IP in c= line?
│   │   │   ├── YES → Add nat=force_rport,comedia to trunk peer config
│   │   │   └── NO  → Check codec negotiation in SDP INVITE vs 200 OK
│   │   │
│   │   └── Check: Is strictrtp enabled?
│   │       ├── YES → Try strictrtp=seqno or strictrtp=no temporarily
│   │       └── NO  → Escalate: capture RTP with tcpdump, analyze packet flow
│   │
│   └── BOTH-WAY (complete silence for both parties)
│       │
│       ├── Check: Are there ANY RTP packets flowing?
│       │   ├── NO  → Firewall is blocking ALL RTP. Check iptables UDP rules.
│       │   └── YES → Codec mismatch. Check 'core show translation' for errors.
│       │
│       ├── Check: Is the media IP reachable?
│       │   ├── NO  → Carrier or agent has wrong externaddr/externip
│       │   └── YES → Check if directmedia=yes is causing media bypass issues
│       │
│       └── Check: Did the call actually connect? (200 OK + ACK in Homer)
│           ├── NO  → SIP signaling issue, not media issue
│           └── YES → Media path is broken. Capture and compare SDP from both legs.

CALLS KEEP DROPPING
│
├── Check: Are ALL calls dropping or just some?
│   │
│   ├── ALL CALLS (every call on the system)
│   │   │
│   │   ├── Check: Is Asterisk running?
│   │   │   ├── NO  → Restart: systemctl start asterisk
│   │   │   │         Check /var/log/asterisk/ for crash dumps
│   │   │   └── YES → Check server load: uptime, top, df -h
│   │   │
│   │   ├── Check: Is disk full?
│   │   │   ├── YES → Emergency cleanup of old recordings/logs
│   │   │   └── NO  → Check: Are ALL trunks UNREACHABLE?
│   │   │
│   │   └── Check: Is there a network outage?
│   │       ├── YES → Contact datacenter/ISP
│   │       └── NO  → Check Asterisk error log for repeated errors
│   │
│   ├── CALLS ON ONE TRUNK ONLY
│   │   │
│   │   ├── Check: sip show peers | grep TRUNK_NAME
│   │   │   ├── UNREACHABLE → Trunk is down (see Section 4.4)
│   │   │   └── OK          → Trunk is up but failing
│   │   │
│   │   ├── Check: carrier_log for sip_hangup_cause patterns
│   │   │   ├── 503 consistently → Carrier is overloaded
│   │   │   ├── 403 consistently → Authentication failure
│   │   │   └── Various codes   → Check each code in Section 6
│   │   │
│   │   └── Contact the carrier with Call-IDs and timestamps
│   │
│   └── CALLS FOR ONE AGENT ONLY
│       │
│       ├── Check: sip show peer AGENT_EXT
│       │   ├── LAGGED      → Agent's internet is slow (Section 4.2)
│       │   ├── UNREACHABLE → Agent disconnected
│       │   └── OK (high ms)→ Marginal connection, drops under load
│       │
│       ├── Check: Does the agent's call always drop at the same duration?
│       │   ├── YES (e.g., always at 30 min) → SIP session timer issue
│       │   └── NO (random times)            → NAT mapping timeout
│       │
│       └── Ask agent: Are they on WiFi, VPN, or shared internet?
│           Any of these can cause intermittent drops.

TRUNK IS UNREACHABLE
│
├── Step 1: Can you ping the trunk IP?
│   │
│   ├── NO (100% loss)
│   │   │
│   │   ├── Check: Has the provider's IP changed?
│   │   │   ├── dig PROVIDER_HOSTNAME → Compare with config
│   │   │   └── If changed → Update sip-vicidial.conf, reload SIP
│   │   │
│   │   ├── Check: Is the IP blocked by your firewall?
│   │   │   ├── iptables -S INPUT | grep PROVIDER_IP
│   │   │   └── If missing → Add: iptables -I INPUT -s IP -j ACCEPT
│   │   │
│   │   └── Provider may be down → Check their status page
│   │       └── Route traffic to backup trunk while waiting
│   │
│   └── YES (ping works)
│       │
│       ├── Check: Can you reach SIP port 5060?
│       │   ├── nc -zvu PROVIDER_IP 5060
│       │   ├── If CLOSED → Provider firewall is blocking you
│       │   │   └── Verify your server's IP is whitelisted with provider
│       │   └── If OPEN → SIP is reachable but qualify fails
│       │
│       ├── Check: Is the SIP registration failing?
│       │   ├── asterisk -rx 'sip show registry'
│       │   ├── If "Rejected" → Wrong username/password
│       │   ├── If "Timeout"  → Network issue or wrong port
│       │   └── If "No Registry" → This trunk does not use registration
│       │
│       └── Check: Is there a TLS/SRTP mismatch?
│           └── Try connecting without encryption to test
│
├── Step 2: Do you have a backup trunk?
│   │
│   ├── YES → Reroute traffic via dialplan or ViciDial carrier settings
│   └── NO  → This is a single point of failure. Plan for redundancy.
│
└── Step 3: Document and notify
    ├── Record the time, trunk name, and symptoms
    ├── Open a ticket with the carrier
    └── Set up monitoring to alert on trunk state changes (see Tutorial 01)

# === SIP STATUS ===
asterisk -rx 'sip show peers'                    # All SIP peers with status
asterisk -rx 'sip show peers' | grep LAGGED       # Find LAGGED agents
asterisk -rx 'sip show peers' | grep UNREACHABLE  # Find dead trunks/agents
asterisk -rx 'sip show peer EXTENSION'             # Detailed info for one peer
asterisk -rx 'sip show registry'                   # SIP trunk registrations
asterisk -rx 'sip show channelstats'               # Live RTP stats (loss, jitter)

# === CALLS & CHANNELS ===
asterisk -rx 'core show channels'                  # Active channels summary
asterisk -rx 'core show channels concise'          # Machine-readable channel list
asterisk -rx 'core show channels verbose'          # Detailed channel info
asterisk -rx 'core show channel SIP/peer-id'       # Single channel deep dive

# === CONFERENCES ===
asterisk -rx 'confbridge list'                     # ConfBridge conferences
asterisk -rx 'meetme list'                         # MeetMe conferences
asterisk -rx 'confbridge kick CONF all'            # Kick all from a conference

# === CODEC & MEDIA ===
asterisk -rx 'core show translation'               # Codec translation paths
asterisk -rx 'rtp show settings'                   # RTP configuration
asterisk -rx 'core show codecs'                    # Available codecs

# === DIAGNOSTICS ===
asterisk -rx 'core show uptime'                    # Asterisk uptime
asterisk -rx 'core show version'                   # Asterisk version
asterisk -rx 'core show settings'                  # Global settings
asterisk -rx 'core show sysinfo'                   # System resource info

# === DEBUGGING ===
asterisk -rx 'sip set debug on'                    # Enable SIP debug (VERBOSE)
asterisk -rx 'sip set debug off'                   # Disable SIP debug
asterisk -rx 'rtp set debug on'                    # Enable RTP debug (VERY VERBOSE)
asterisk -rx 'rtp set debug off'                   # Disable RTP debug

# === RELOAD (safe, does not drop calls) ===
asterisk -rx 'sip reload'                          # Reload SIP config
asterisk -rx 'dialplan reload'                     # Reload dialplan

-- Active calls right now
SELECT COUNT(*) AS active_calls FROM vicidial_auto_calls;

-- Agents currently logged in
SELECT user, status, campaign_id, phone_login,
       TIMESTAMPDIFF(SECOND, last_update_time, NOW()) AS idle_sec
FROM vicidial_live_agents
ORDER BY status, idle_sec DESC;

-- Call volume by hour (today)
SELECT HOUR(call_date) AS hr, COUNT(*) AS calls
FROM vicidial_closer_log
WHERE call_date >= CURDATE()
GROUP BY hr ORDER BY hr;

-- Trunk failure rate (last hour)
SELECT SUBSTRING_INDEX(channel, '-', 1) AS trunk,
       COUNT(*) AS total,
       SUM(dialstatus != 'ANSWER') AS failed,
       ROUND(SUM(dialstatus != 'ANSWER') / COUNT(*) * 100, 1) AS fail_pct
FROM vicidial_carrier_log
WHERE call_date > NOW() - INTERVAL 1 HOUR
GROUP BY trunk ORDER BY fail_pct DESC;

-- Average queue time by inbound group (today)
SELECT campaign_id, COUNT(*) AS calls,
       ROUND(AVG(queue_seconds), 0) AS avg_queue_sec,
       MAX(queue_seconds) AS max_queue_sec
FROM vicidial_closer_log
WHERE call_date >= CURDATE()
GROUP BY campaign_id ORDER BY avg_queue_sec DESC;

-- Abnormal disconnect trending (by day, last 7 days)
SELECT DATE(call_date) AS day,
       SUM(status = 'DISMX') AS dismx,
       SUM(status = 'DCMX') AS dcmx,
       COUNT(*) AS total_calls,
       ROUND((SUM(status IN ('DISMX','DCMX')) / COUNT(*)) * 100, 2) AS abnormal_pct
FROM vicidial_closer_log
WHERE call_date > NOW() - INTERVAL 7 DAY
GROUP BY day ORDER BY day;

# === NETWORK ===
ping -c 10 TRUNK_IP                       # Basic latency test
traceroute -n TRUNK_IP                     # Route path
mtr --report -c 50 TRUNK_IP               # Continuous route + loss report
nc -zvu TRUNK_IP 5060                      # Test SIP port reachability
tcpdump -i eth0 -n port 5060 -c 100       # Capture 100 SIP packets
ngrep -W byline -d eth0 port 5060         # Real-time SIP message dump

# === SYSTEM HEALTH ===
uptime                                     # Load average
df -h                                      # Disk space
free -m                                    # Memory usage
top -bn1 | head -20                        # CPU/process overview
iostat -x 1 5                              # Disk I/O stats

# === LOG ANALYSIS ===
# Count Asterisk errors today
grep "$(date +%b\ %d)" /var/log/asterisk/messages | grep -ci error

# Find all WARNING and ERROR lines in the last 100 lines
tail -100 /var/log/asterisk/messages | grep -iE 'WARNING|ERROR'

# Count jitter buffer resyncs (IAX2 audio issues)
grep -c 'Resyncing the jb' /var/log/asterisk/messages

# Find strict RTP source switches (NAT issues)
grep -c 'Strict RTP switching' /var/log/asterisk/messages

# === FIREWALL ===
iptables -L INPUT -n --line-numbers        # Show all INPUT rules with numbers
iptables -S INPUT | grep TRUNK_IP          # Check if trunk IP is whitelisted

#!/bin/bash
# voip-diag -- VoIP Diagnostic Tool
# Usage: voip-diag <command>

case "${1:-help}" in

    # Show SIP peer status overview
    sip)
        echo "=== SIP Peers ==="
        asterisk -rx 'sip show peers' | head -50
        echo ""
        echo "LAGGED:      $(asterisk -rx 'sip show peers' | grep -c LAGGED)"
        echo "UNREACHABLE: $(asterisk -rx 'sip show peers' | grep -c UNREACHABLE)"
        echo "OK:          $(asterisk -rx 'sip show peers' | grep -c 'OK (')"
        ;;

    # Show active calls
    calls)
        echo "=== Active Calls ==="
        asterisk -rx 'core show channels'
        ;;

    # Show logged-in agents
    agents)
        mysql -u USER -pPASSWORD DATABASE -e "
            SELECT user, status, campaign_id,
                   TIMESTAMPDIFF(SECOND, last_update_time, NOW()) AS idle_sec
            FROM vicidial_live_agents
            ORDER BY status, idle_sec DESC;" 2>/dev/null
        ;;

    # Show RTP channel statistics
    rtp)
        echo "=== RTP Channel Stats ==="
        asterisk -rx 'sip show channelstats'
        ;;

    # Check a specific trunk
    trunk)
        if [ -z "$2" ]; then
            echo "Usage: voip-diag trunk TRUNK_NAME"
            exit 1
        fi
        echo "=== Trunk: $2 ==="
        asterisk -rx "sip show peer $2" | grep -E 'Status|Addr|Codecs|Qualify'
        echo ""
        echo "=== Recent Carrier Log ==="
        mysql -u USER -pPASSWORD DATABASE -e "
            SELECT call_date, dialstatus, hangup_cause, sip_hangup_cause
            FROM vicidial_carrier_log
            WHERE channel LIKE '%$2%'
            ORDER BY call_date DESC LIMIT 10;" 2>/dev/null
        ;;

    # Check a specific agent
    agent)
        if [ -z "$2" ]; then
            echo "Usage: voip-diag agent EXTENSION"
            exit 1
        fi
        echo "=== Agent: $2 ==="
        asterisk -rx "sip show peer $2" | grep -E 'Status|Addr|Useragent|Codecs|Nat|Qualify'
        echo ""
        echo "=== ViciDial Status ==="
        mysql -u USER -pPASSWORD DATABASE -e "
            SELECT user, status, campaign_id, phone_login,
                   TIMESTAMPDIFF(SECOND, last_update_time, NOW()) AS idle_sec
            FROM vicidial_live_agents
            WHERE phone_login = '$2';" 2>/dev/null
        ;;

    # Investigate a specific call by phone number
    call)
        if [ -z "$2" ]; then
            echo "Usage: voip-diag call PHONE_NUMBER"
            exit 1
        fi
        echo "=== Inbound Calls for $2 ==="
        mysql -u USER -pPASSWORD DATABASE -e "
            SELECT call_date, length_in_sec, status, term_reason, uniqueid, user
            FROM vicidial_closer_log
            WHERE phone_number LIKE '%$2%'
            ORDER BY call_date DESC LIMIT 10;" 2>/dev/null
        echo ""
        echo "=== Outbound Calls for $2 ==="
        mysql -u USER -pPASSWORD DATABASE -e "
            SELECT call_date, length_in_sec, status, term_reason, uniqueid, user
            FROM vicidial_log
            WHERE phone_number LIKE '%$2%'
            ORDER BY call_date DESC LIMIT 10;" 2>/dev/null
        ;;

    # Show system health
    health)
        echo "=== System Health ==="
        echo "Uptime: $(uptime)"
        echo "Disk:   $(df -h / | tail -1)"
        echo "Memory: $(free -m | grep Mem | awk '{printf "%dMB used / %dMB total (%.0f%%)\n", $3, $2, $3/$2*100}')"
        echo ""
        echo "=== Asterisk ==="
        asterisk -rx 'core show uptime'
        echo "Active channels: $(asterisk -rx 'core show channels concise' | wc -l)"
        echo ""
        echo "=== Errors Today ==="
        echo "Asterisk errors: $(grep "$(date +'%b %d')" /var/log/asterisk/messages 2>/dev/null | grep -ci error)"
        echo "JB resyncs:      $(grep "$(date +'%b %d')" /var/log/asterisk/messages 2>/dev/null | grep -c 'Resyncing the jb')"
        echo "RTP switches:    $(grep "$(date +'%b %d')" /var/log/asterisk/messages 2>/dev/null | grep -c 'Strict RTP')"
        ;;

    help|*)
        cat <<'EOF'
VoIP Diagnostic Tool
====================
Usage: voip-diag <command> [args]

Commands:
  sip              Show SIP peers status summary
  calls            Show active calls
  agents           Show logged-in agents
  rtp              Show live RTP statistics
  trunk NAME       Check a specific trunk
  agent EXT        Check a specific agent extension
  call NUMBER      Look up a phone number in call logs
  health           System health overview
  help             Show this help

Examples:
  voip-diag sip
  voip-diag trunk my_provider
  voip-diag agent 100
  voip-diag call 441234567890
EOF
        ;;
esac

# Save the script
sudo cp voip-diag /usr/local/bin/voip-diag
sudo chmod +x /usr/local/bin/voip-diag

# Update the USER, PASSWORD, and DATABASE variables in the script
# to match your ViciDial MySQL credentials

Date: 2026-03-13
Symptom: Agent reports one-way audio on all calls
Root cause: SIP ALG enabled on agent's new router
Fix: Disabled SIP ALG, added nat=force_rport,comedia to peer config
Time to resolve: 45 minutes