Chapter 30. PPP

# ngctl -f- <<EOF
mkpeer pppoe lb lower pppoe_lb
name pppoe_lb:pppoed
msg pppoe_lb:pppoed setconfig ${ethernet_iface}
EOF

# pppoed -L -w 4 em0

# sysctl net.graph.pppoe_lb.governor.enabled=1

# ngctl msg pppoe_lb:pppoed showworkers

# sysctl net.graph.pppoe_lb.workers

# ngctl msg pppoe_lb:pppoed showworkers
Worker Status:
  Worker 0: ACTIVE (23 sessions, 45m uptime)
  Worker 1: ACTIVE (31 sessions, 45m uptime)
  Worker 2: DRAINING (5 sessions, 10m uptime)
  Worker 3: ACTIVE (28 sessions, 45m uptime)

# ngctl msg pppoe_lb:pppoed showgovernor
Governor Status:
  Enabled: Yes
  Mode: Auto
  Min Workers: 2
  Max Workers: 4
  Current Workers: 3
  Active Workers: 2
  Draining Workers: 1
  Pending Removals: 0
  Last Decision: scale_down (reason: cpu_low)

# sysctl net.graph.pppoe_lb

# sysctl net.graph.pppoe_lb.workers.0
net.graph.pppoe_lb.workers.0.state: 0
net.graph.pppoe_lb.workers.0.sessions: 23
net.graph.pppoe_lb.workers.0.bytes_in: 1234567890
net.graph.pppoe_lb.workers.0.bytes_out: 987654321
net.graph.pppoe_lb.workers.0.uptime: 2700

# ngctl msg pppoe_lb:pppoed addworker

# sysctl net.graph.pppoe_lb.governor.desired_workers=5

# ngctl msg pppoe_lb:pppoed rmworker 2

# sysctl net.graph.pppoe_lb.workers.2.state=1

# sysctl net.graph.pppoe_lb.workers.2.state=0

# Round-robin (default)
# ngctl msg pppoe_lb:pppoed setalgorithm 0

# Hash-based (by source MAC)
# ngctl msg pppoe_lb:pppoed setalgorithm 1

# Least-loaded (fewest sessions)
# ngctl msg pppoe_lb:pppoed setalgorithm 2

# ngctl msg pppoe_lb:pppoed getmap
Session Mappings:
  Session 0x7f3a2b: Worker 2
  Session 0x8c4d1e: Worker 1
  Session 0x9a1f7c: Worker 3

# sysctl net.graph.pppoe_lb.governor.cpu_usage net.graph.pppoe_lb.governor.total_sessions
net.graph.pppoe_lb.governor.cpu_usage: 67
net.graph.pppoe_lb.governor.total_sessions: 847

                            +------------------+
                            |   pppoe_lb       |
    DSLAM <-- Ethernet -->  |   (dispatcher)   |<---> Worker 0
    em0                     |                  |<---> Worker 1
                            +------------------+

# 1. Check that pppoed is running with 2 workers
$ sockstat | grep pppoed
root     pppoed     1234  3  tcp4   *:2000                *:*
root     pppoed     1235  3  tcp4   *:2000                *:*
root     pppoed     1236  3  tcp4   *:2000                *:*

# 2. Verify worker count
$ sysctl net.graph.pppoe_lb.governor.current_workers
net.graph.pppoe_lb.governor.current_workers: 2

# 3. Check session distribution
$ ngctl msg pppoe_lb:pppoed showworkers
Worker Status:
  Worker 0: ACTIVE (0 sessions, 1h uptime)
  Worker 1: ACTIVE (0 sessions, 1h uptime)

# Problem: Only 1 worker showing
$ sysctl net.graph.pppoe_lb.governor.current_workers
net.graph.pppoe_lb.governor.current_workers: 1

# Solution: Check if pppoed restarted; may need manual restart
$ service pppoed restart

    +-----------+
    |  Governor  |    <-- Monitors CPU every 5 seconds
    |  (scaler)   |
    +------+------+    scale up/down signals
           |
           v
    +------------------+     +------------------+
    |   pppoe_lb       |---->| Worker Pool      |
    |   (dispatcher)   |     | (grows: 2 -> N)  |
    +------------------+     | (shrinks: N -> 2)|
                              +------------------+

CPU > 80%           CPU < 30%
    |                    |
    v                    v
+----------+       +----------+
| SCALE UP | ----> | IDLE    |
+----------+       +----------+
                      ^
                      |
               CPU < 30%
               + cooldown
               |
               v
         +----------+
         |SCALE DOWN|
         +----------+

Timeline Example:

Time    CPU     Workers  Event
----    ---     -------  -----
00:00   15%     2        Normal operation
00:15   85%     2        Scale UP triggered!
00:15   85%     3        Worker 2 created
00:30   70%     3        Stabilized
01:00   18%     3        Scale DOWN triggered!
01:00   18%     3        Worker 2 marked DRAINING
01:30   18%     2        Worker removed (drain complete)

Time    CPU     Workers  Event
----    ---     -------  -----
10:00   15%     3        Normal operation
10:10   12%     3        Scale DOWN triggered
10:10   12%     3        Worker 2 marked DRAINING
10:25   85%     3        Traffic spike! (evening peak)
10:25   85%     3        Scale UP needed - CANCEL DRAIN!
10:25   85%     3        Worker 2 returns to ACTIVE
10:25   85%     4        Worker 3 created (scale up)

# 1. Check governor is enabled and running
$ sysctl net.graph.pppoe_lb.governor.enabled net.graph.pppoe_lb.governor.mode
net.graph.pppoe_lb.governor.enabled: 1
net.graph.pppoe_lb.governor.mode: 1

# 2. View governor status
$ ngctl msg pppoe_lb:pppoed showgovernor
Governor Status:
  Enabled: Yes
  Mode: Auto
  Min Workers: 2
  Max Workers: 8
  Current Workers: 3
  Active Workers: 3
  Draining Workers: 0
  Pending Removals: 0
  Last Decision: none

# 3. Monitor for scaling events in logs
$ tail -f /var/log/messages | grep pppoe_lb
May  1 10:15:22 hostname ng_pppoe_lb: governor: scale up (reason: cpu_high)
May  1 10:15:22 hostname ng_pppoe_lb: created worker 2 (now 3 workers)

$ sysctl net.graph.pppoe_lb.governor
net.graph.pppoe_lb.governor.enabled: 1
net.graph.pppoe_lb.governor.mode: 1
net.graph.pppoe_lb.governor.min_workers: 2
net.graph.pppoe_lb.governor.max_workers: 0
net.graph.pppoe_lb.governor.cpu_threshold: 80
net.graph.pppoe_lb.governor.cpu_low_threshold: 30
net.graph.pppoe_lb.governor.scale_up_interval: 10
net.graph.pppoe_lb.governor.scale_down_interval: 60
net.graph.pppoe_lb.governor.current_workers: 3
net.graph.pppoe_lb.governor.active_workers: 3
net.graph.pppoe_lb.governor.draining_workers: 0
net.graph.pppoe_lb.governor.total_sessions: 156
net.graph.pppoe_lb.governor.last_decision: none
net.graph.pppoe_lb.governor.last_reason: none

# Problem: Governor not scaling up
$ sysctl net.graph.pppoe_lb.governor.current_workers
net.graph.pppoe_lb.governor.current_workers: 2

# Check: Is governor enabled?
$ sysctl net.graph.pppoe_lb.governor.enabled
net.graph.pppoe_lb.governor.enabled: 0
# Solution: Enable it
$ sysctl net.graph.pppoe_lb.governor.enabled=1

# Check: Is CPU actually high?
$ sysctl net.graph.pppoe_lb.governor.cpu_usage
net.graph.pppoe_lb.governor.cpu_usage: 45
# CPU must exceed 80% (cpu_threshold) to scale up

# Check: At maximum workers?
$ sysctl net.graph.pppoe_lb.governor.max_workers
net.graph.pppoe_lb.governor.max_workers: 0
# 0 means mp_ncpus, check actual CPU count
$ sysctl mp_ncpus
mp_ncpus: 8
# Currently at 2 workers, max is 8 - should be able to scale

                            +------------------+
                            |   pppoe_lb       |
    DSLAM <-- Ethernet -->  |   (dispatcher)   |<---> Worker 0
    em0                     |                  |<---> Worker 1
                            |                  |<---> ...
                            +------------------+<---> Worker 7

    Expected Capacity: 8 workers x 1000 sessions = 8000 concurrent sessions

Worker Count     Sessions (500/worker)  Sessions (1000/worker)
------------     ---------------------  ----------------------
4 workers       2,000                  4,000
8 workers       4,000                  8,000
16 workers      8,000                  16,000

Note: Actual capacity depends on:
  - Available CPU cores (max_workers = mp_ncpus)
  - Network interface throughput
  - Traffic patterns (session vs. bandwidth)
  - Available memory

# 1. Verify 8 workers started
$ sysctl net.graph.pppoe_lb.governor.current_workers
net.graph.pppoe_lb.governor.current_workers: 8

# 2. Check per-worker session distribution
$ for i in $(seq 0 7); do
    echo -n "Worker $i: "
    sysctl -n net.graph.pppoe_lb.workers.$i.sessions
  done
Worker 0: 134
Worker 1: 142
Worker 2: 128
Worker 3: 156
Worker 4: 139
Worker 5: 147
Worker 6: 131
Worker 7: 125

# 3. Verify balance (all should be within 20% of average)
$ sysctl net.graph.pppoe_lb.governor.avg_sessions_worker
net.graph.pppoe_lb.governor.avg_sessions_worker: 137

# 4. Check for load imbalance warnings
$ sysctl net.graph.pppoe_lb.health
net.graph.pppoe_lb.health.load_imbalance_pct: 12
# Values > 20% may indicate issues

# Monitor CPU usage per worker (top usage)
top -b -n 1 | grep pppoed

# Check total throughput
$ netstat -I em0 -b | head -5

# View per-worker statistics
$ ngctl msg pppoe_lb:pppoed showstats

# Check for errors
$ sysctl net.graph.pppoe_lb.stats.total_errors

# Problem: Uneven session distribution
$ sysctl net.graph.pppoe_lb.workers.0.sessions
net.graph.pppoe_lb.workers.0.sessions: 456
$ sysctl net.graph.pppoe_lb.workers.1.sessions
net.graph.pppoe_lb.workers.1.sessions: 45

# Causes:
# 1. Sessions have session affinity (same MAC -> same worker)
# 2. Some workers are in DRAINING state
# 3. Hash algorithm is distributing unevenly

# Check for draining workers
$ sysctl net.graph.pppoe_lb.governor.draining_workers
net.graph.pppoe_lb.governor.draining_workers: 1
# Worker 1 might be draining!

# Solution: Check worker states
for i in $(seq 0 7); do
  echo "Worker $i: $(sysctl -n net.graph.pppoe_lb.workers.$i.state)"
done
Worker 0: 0 (ACTIVE)
Worker 1: 1 (DRAINING)  <-- This worker won't get new sessions
Worker 2: 0 (ACTIVE)
...

# To cancel draining:
$ sysctl net.graph.pppoe_lb.workers.1.state=0

# 1. Start with Basic (testing)
# 2. Move to Auto-Scaling when ready for production
# 3. Move to High-Load if capacity is exceeded

# To change configuration:
# 1. Edit /etc/sysctl.conf with new settings
# 2. Apply new sysctls: sysctl -f /etc/sysctl.conf
# 3. Adjust worker count: ngctl msg pppoe_lb:pppoed setworkers N
# 4. Or restart pppoed: service pppoed restart

# Build the tool
$ cd /usr/src/usr.sbin/pppoetest
$ make
$ make install

# Verify installation
$ which pppoetest
/usr/sbin/pppoetest

# Basic diagnostics (read-only, safe for production)
$ pppoetest -m diagnose

PPPoE Load Balancer Diagnostic Report
=====================================
Host: pppoe-server.example.com
Time: 2024-05-01 10:15:22 UTC

Workers: 3
  Worker 0: ACTIVE (234 sessions, 45m uptime)
  Worker 1: ACTIVE (256 sessions, 45m uptime)
  Worker 2: DRAINING (12 sessions, 10m uptime)

Sessions: 502 total
  Active: 490
  Draining: 12

Governor: Enabled
  CPU: 52%
  Last Decision: none

Health Score: 87/100 ✓

# Check health score
$ pppoetest -m diagnose --health-score

=== PPPoE Load Balancer Health Score: 87/100 ===

Component Scores:
  ✓ Worker Availability: 100/100
  ⚠ Load Distribution:   72/100 (max deviation: 28%)
  ✓ Error Rate:           100/100
  ✓ Session Affinity:     99/100 (1 drift detected)

Warnings:
  - Load distribution uneven (Worker 0 has 28% more than average)
  - 1 session affinity drift detected

Recommendations:
  - Investigate Worker 0 for potential issues
  - Check for sessions that moved between workers unexpectedly

$ pppoetest -m diagnose --check-balance

=== Load Balance Check ===
┌────────┬───────────┬──────────┬────────────────┐
│ Worker │ Sessions  │ Expected │ Deviation      │
├────────┼───────────┼──────────┼────────────────┤
│ 0      │ 234       │ 200     │ +17% ⚠         │
│ 1      │ 256       │ 200     │ +28% ⚠         │
│ 2      │ 112       │ 200     │ -44% ⚠         │
└────────┴───────────┴──────────┴────────────────┘

Imbalance: 28% (threshold: 20%)
Status: ⚠ WARNING - Load distribution exceeds threshold

Recommendations:
  - Worker 2 may be in DRAINING state
  - Check worker states: sysctl net.graph.pppoe_lb.workers.*.state

$ pppoetest -m diagnose --check-affinity

=== Affinity Check ===
┌────────────┬────────────────┬─────────────────┬────────────┐
│ Session ID │ Expected Worker│ Actual Worker  │ Status     │
├────────────┼────────────────┼─────────────────┼────────────┤
│ 0x7f3a2b  │ 2              │ 2               │ ✓ OK       │
│ 0x8c4d1e  │ 1              │ 1               │ ✓ OK       │
│ 0x9a1f7c  │ 3              │ 2 ⚠             │ ✗ DRIFT    │
└────────────┴────────────────┴─────────────────┴────────────┘

Total: 3 sessions checked
  ✓ Passed: 2
  ✗ Failed: 1 (session drift detected)

Session drift may indicate:
  - Worker was removed while session was active
  - Hash algorithm rebalancing
  - Race condition in session routing

$ pppoetest -m diagnose --show-workers

=== Worker Status ===
┌────────┬─────────┬──────────┬───────────┬───────────┬──────────┐
│ Worker │ State   │ Sessions │ Bytes In  │ Bytes Out │ Uptime   │
├────────┼─────────┼──────────┼───────────┼───────────┼──────────┤
│ 0      │ ACTIVE  │ 234      │ 1.2 GB    │ 456 MB    │ 45m      │
│ 1      │ ACTIVE  │ 256      │ 1.4 GB    │ 523 MB    │ 45m      │
│ 2      │ DRAINING│ 12       │ 89 MB     │ 34 MB     │ 10m      │
└────────┴─────────┴──────────┴───────────┴───────────┴──────────┘

Worker States:
  0 = ACTIVE (accepting new sessions)
  1 = DRAINING (completing existing sessions)
  2 = PENDING_REMOVAL (ready for removal)

# Monitor every 5 seconds
$ pppoetest -m diagnose --monitor --interval 5

Press Ctrl+C to stop

┌──────────────────────────┬────────┬────────┬────────┬────────┐
│ Metric                   │ Now    │ 5s ago │ 30s ago│ 5m ago │
├──────────────────────────┼────────┼────────┼────────┼────────┤
│ Total Sessions           │ 800    │ 798    │ 785    │ 750    │
│ Active Workers           │ 3      │ 3      │ 3      │ 2      │
│ Sessions/Worker (avg)   │ 267    │ 266    │ 262    │ 375    │
│ CPU Usage (avg)        │ 52%    │ 48%    │ 45%    │ 38%    │
└──────────────────────────┴────────┴────────┴────────┴────────┘

Changes detected:
  ↑ Workers: 2 -> 3 (governor scaled up)
  ↑ Sessions: 750 -> 800 (+50 in 5 minutes)

# Start server on test coordinator
$ pppoetest -m server -i em0 -p 9001

Server listening on port 9001
Waiting for client connections...

# Connect to server and run tests
$ pppoetest -m client -h test-coordinator.example.com -p 9001

Connected to test-coordinator.example.com:9001
Running accuracy test...

Both sides will see coordinated test results:

┌─────────────────┐    ┌─────────────────┐
│   CLIENT SIDE   │    │   SERVER SIDE   │
│ Blocks: 800     │    │ Blocks: 800    │
│ Checksum: OK    │    │ Checksum: OK    │
└─────────────────┘    └─────────────────┘
        ↓                      ↓
   VERIFICATION (MATCH/MISMATCH)

✓ Transfer test PASSED
  - 800 blocks transferred
  - Checksums match on both sides

# Console output (default)
$ pppoetest -m diagnose --format console

# JSON output (for parsing)
$ pppoetest -m diagnose --format json
{
  "type": "diagnostic_report",
  "timestamp": "2024-05-01T10:15:22Z",
  "workers": 3,
  "sessions": 502,
  "health_score": 87,
  "status": "warn"
}

# JSONL output (streaming, for log processors)
$ pppoetest -m diagnose --format jsonl
{"type":"worker_status","id":0,"state":"ACTIVE","sessions":234}
{"type":"worker_status","id":1,"state":"ACTIVE","sessions":256}
{"type":"health_score","score":87,"status":"warn"}

# CSV output (for spreadsheets)
$ pppoetest -m diagnose --format csv
worker_id,state,sessions,bytes_in,bytes_out
0,ACTIVE,234,1288490188,478150656
1,ACTIVE,256,1503238553,548453324
2,DRAINING,12,95787918,36372403

# TAP output (for test frameworks)
$ pppoetest -m diagnose --format tap
1..4
ok 1 - Worker 0 is ACTIVE
ok 2 - Worker 1 is ACTIVE
ok 3 - Worker 2 is DRAINING
ok 4 - Health score >= 70