From 5916514375a7de12a785805723da4a9b25b266e3 Mon Sep 17 00:00:00 2001 From: Yudistira Putra <85178972+Yudis-bit@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:07:52 +0700 Subject: [PATCH] feat(sim): add multi-seed fairness benchmark and empirical research notes --- README.md | 1 + simulation/RESEARCH_NOTES.md | 46 +++++++ .../simulations/benchmark_multi_seed.py | 123 ++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 simulation/RESEARCH_NOTES.md create mode 100644 simulation/simulations/benchmark_multi_seed.py diff --git a/README.md b/README.md index 2f9ade7..2f6cb2a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ The Python prototype in this repository has successfully validated the core hypo * **Adaptive Routing:** The Cognitive Router successfully learned to **dynamically avoid a congested network link**, using it less than **0.1%** of the time, compared to the Dumb Router which was stuck in congestion nearly **40%** of the time. * **Performance Gains:** By avoiding these bottlenecks, the Cognitive Router achieved **~22% lower average latency** for successful packet deliveries, proving its ability to optimize for overall network health. * **Known Trade-off:** In the seeded reference run, the Cognitive Router's exploration behaviour delivers only **~24% of packets** (the Dumb Router delivers 100%); the latency figure above is computed over successful deliveries only. These results demonstrate adaptive behaviour, not production readiness — closing the delivery gap is future work (see issue tracker). +* **Multi-Seed Evaluation & Mechanics:** See [`simulation/RESEARCH_NOTES.md`](simulation/RESEARCH_NOTES.md) for a multi-seed analysis (seeds 42, 100, 777, 1337, 2026) explaining the trade-offs between local link-reward exploration and multi-hop loop traps. Run `python simulations/benchmark_multi_seed.py` to reproduce the multi-seed evaluation matrix. * **Full Analysis:** The complete comparative simulation can be run via the `simulations/run_cognitive_sim.py` script; `simulations/run_baseline_sim.py` runs the Dijkstra-only baseline. Smoke tests for both live under `simulation/tests/`. ## Full Project Architecture diff --git a/simulation/RESEARCH_NOTES.md b/simulation/RESEARCH_NOTES.md new file mode 100644 index 0000000..b37048d --- /dev/null +++ b/simulation/RESEARCH_NOTES.md @@ -0,0 +1,46 @@ +# Research Notes: Simulation Dynamics & Fairness Analysis + +## Overview +This document records the empirical analysis of the Cognitive Routing Protocol (CRP) prototype simulator, specifically addressing the interaction between local Multi-Armed Bandit (MAB / UCB1) reinforcement learning and multi-hop network topologies. + +## Empirical Findings + +### 1. Multi-Seed Performance Matrix +When evaluating 1,000 packets per trial across diverse pseudorandom generator seeds, the protocol exhibits significant sensitivity to network edge initialization: + +| Seed | Dijkstra Latency | Dijkstra Congestion | CRP Delivery % | CRP Loss % | CRP Latency (Delivered) | Latency Difference | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| **42** (Ref) | 71.79 ms | 37.3% | 23.80% | 76.20% | 55.94 ms | **+22.08%** | +| **100** | 79.95 ms | 40.1% | 0.20% | 99.80% | 170.02 ms | -112.65% | +| **777** | 89.35 ms | 40.8% | 0.80% | 99.20% | 313.54 ms | -250.91% | +| **1337** | 76.10 ms | 39.4% | 100.00% | 0.00% | 128.27 ms | -68.55% | +| **2026** | 86.50 ms | 38.8% | 2.20% | 97.80% | 299.87 ms | -246.67% | + +### 2. Root Cause Analysis: Local MAB vs Global Topology +The mechanism driving this distribution consists of two factors: + +1. **Immediate 1-Hop Memory vs Multi-Hop Cycles**: + - `CognitiveNode.choose_next_hop` filters only `prev_node_id` (`nid != prev_node_id`). + - While this prevents immediate 2-node bounce (`A -> B -> A`), it permits 3-node or larger cycles (`A -> B -> C -> A`). + - Once trapped in a cyclic subgraph, packets traverse links repeatedly until exhausting `MAX_HOPS = 25`, after which the packet is dropped as failed. + +2. **Greedy Link-Local Rewards vs Destination Progress**: + - Node reward is calculated as `REWARD_FACTOR / link_latency`. + - The reward is granted immediately upon traversal of a low-latency edge, regardless of whether that hop moves the packet closer to `GATEWAY_EAST`. + - In certain topologies (e.g. seeds 100, 777), low-latency internal links form an attractive local reward trap that actively reinforces cyclic behavior. + +### 3. Latency Metric Context +- At seed 42, the reported **~22% latency improvement** applies strictly to the subset of packets that successfully reach the destination gateway without getting caught in cycles (~23.8% of total traffic). +- For packets that exit the cycle trap or find the direct path, avoiding the 10x congestion link (`NODE_2 <-> NODE_3`) does result in faster packet transit. +- However, reporting latency improvements without simultaneously reporting packet delivery rates or multi-seed bounds would constitute cherry-picking. + +## Future Protocol Directions + +To bridge the delivery gap while retaining adaptive congestion avoidance, future iterations should incorporate: +1. **Destination-Aware Reinforcement Learning (Q-Routing)**: + - State should incorporate destination target, maintaining Q-values per `(destination, neighbor)`. +2. **Loop Suppression Mechanisms**: + - Packet header hop-tracing or Bloom filter visited-set tracking. + - Negative rewards (penalties) for packets hitting TTL / hop thresholds. +3. **Directed Acyclic Graph (DAG) Constraints**: + - Combining distance-vector bounds with cognitive bandit exploration so nodes only explore forward-progressing neighbors. diff --git a/simulation/simulations/benchmark_multi_seed.py b/simulation/simulations/benchmark_multi_seed.py new file mode 100644 index 0000000..df0d3f5 --- /dev/null +++ b/simulation/simulations/benchmark_multi_seed.py @@ -0,0 +1,123 @@ +"""Multi-Seed Fair Benchmark for Cognitive Routing Protocol. + +Evaluates delivery rate, packet loss, latency, and congestion across multiple +independent random seeds to avoid cherry-picking single-seed results. +""" +import math +import os +import random +import sys +import time + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from crp.routing.dumb_router import find_path_dijkstra +from simulations import run_cognitive_sim + +SEEDS = [42, 100, 777, 1337, 2026] +NUM_PACKETS = 1000 + + +def evaluate_seed(seed: int, num_packets: int = 1000): + run_cognitive_sim.RANDOM_SEED = seed + random.seed(seed) + + # 1. Dumb Router (Dijkstra) + dumb_net = run_cognitive_sim.build_network(use_cognitive_nodes=False) + static_path, _ = find_path_dijkstra(dumb_net, "GATEWAY_WEST", "GATEWAY_EAST") + dumb_total_lat = 0.0 + dumb_cong = 0 + for _ in range(num_packets): + plat = 0.0 + is_cong = False + for j in range(len(static_path) - 1): + cur = dumb_net.get_node(static_path[j]) + nxt = static_path[j + 1] + lat = run_cognitive_sim.get_current_latency(cur, nxt) + if lat != cur.neighbors[nxt]['latency']: + is_cong = True + plat += lat + dumb_total_lat += plat + if is_cong: + dumb_cong += 1 + dumb_avg_lat = dumb_total_lat / num_packets + dumb_cong_rate = (dumb_cong / num_packets) * 100.0 + + # 2. Cognitive Router (MAB) + cog_net = run_cognitive_sim.build_network(use_cognitive_nodes=True) + cog_total_lat = 0.0 + cog_succ = 0 + cog_cong = 0 + for _ in range(num_packets): + plat = 0.0 + is_cong = False + cur = cog_net.get_node("GATEWAY_WEST") + prev = None + succ = True + hops = 0 + while cur.node_id != "GATEWAY_EAST": + if hops > run_cognitive_sim.MAX_HOPS: + succ = False + break + nxt = cur.choose_next_hop(prev) + if nxt is None: + succ = False + break + lat = run_cognitive_sim.get_current_latency(cur, nxt) + if lat != cur.neighbors[nxt]['latency']: + is_cong = True + plat += lat + cur.update_reward(nxt, run_cognitive_sim.REWARD_FACTOR / lat) + prev = cur.node_id + cur = cog_net.get_node(nxt) + hops += 1 + + if succ: + cog_succ += 1 + cog_total_lat += plat + if is_cong: + cog_cong += 1 + + cog_deliv_rate = (cog_succ / num_packets) * 100.0 + cog_avg_lat = (cog_total_lat / cog_succ) if cog_succ > 0 else float('nan') + cog_cong_rate = (cog_cong / num_packets) * 100.0 + lat_diff = (((dumb_avg_lat - cog_avg_lat) / dumb_avg_lat) * 100.0) if (cog_succ > 0 and dumb_avg_lat > 0) else float('nan') + + return { + 'seed': seed, + 'dumb_avg_lat': dumb_avg_lat, + 'dumb_cong_rate': dumb_cong_rate, + 'cog_deliv_rate': cog_deliv_rate, + 'cog_loss_rate': 100.0 - cog_deliv_rate, + 'cog_avg_lat': cog_avg_lat, + 'cog_cong_rate': cog_cong_rate, + 'lat_diff_pct': lat_diff, + } + + +def main(): + print("=" * 80) + print(" COGNITIVE ROUTING PROTOCOL - MULTI-SEED FAIRNESS EVALUATION") + print("=" * 80) + print(f"Packets per seed: {NUM_PACKETS} | Max hops: {run_cognitive_sim.MAX_HOPS}\n") + print(f"{'Seed':>5} | {'Dijkstra Lat':>12} | {'Dijkstra Cong':>13} | {'CRP Deliv %':>11} | {'CRP Loss %':>10} | {'CRP Lat':>9} | {'Lat Gain %':>10}") + print("-" * 80) + + results = [] + for s in SEEDS: + r = evaluate_seed(s, NUM_PACKETS) + results.append(r) + cog_lat_str = f"{r['cog_avg_lat']:.2f}ms" if not math.isnan(r['cog_avg_lat']) else "N/A" + gain_str = f"{r['lat_diff_pct']:+.2f}%" if not math.isnan(r['lat_diff_pct']) else "N/A" + print(f"{r['seed']:5d} | {r['dumb_avg_lat']:10.2f}ms | {r['dumb_cong_rate']:12.1f}% | {r['cog_deliv_rate']:10.2f}% | {r['cog_loss_rate']:9.2f}% | {cog_lat_str:>9} | {gain_str:>10}") + + print("-" * 80) + print("\nMethodological Notes:") + print("1. Seed 42 produces the reference ~22% latency gain on successful packets, but at ~76.2% packet loss.") + print("2. Multi-seed runs show high sensitivity to topology edge-weight initializations.") + print("3. Single-hop MAB without destination awareness causes packets to cycle in multi-hop loops until MAX_HOPS is reached.") + print("4. Honest research requires reporting delivery rate, loss rate, and latency together.") + + +if __name__ == "__main__": + main()