From ad6b9b8a6a12fb626184a4b7cb67e0a7e5f6d2fc Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Wed, 15 Oct 2025 15:50:48 +0100 Subject: [PATCH 01/17] feat: add comprehensive monitoring guide for OP Stack chains - Add complete step-by-step monitoring tutorial - Include Docker and build-from-source options for op-dispute-mon - Add 8 main steps with Steps component for better UX - Include Prometheus and Grafana setup with examples - Add alert rules and Alertmanager configuration - Include metrics reference for all components - Add troubleshooting section and production considerations - Update navigation in docs.json - Total: 1,238 lines with 40+ code examples --- docs.json | 1 + .../tutorials/monitoring-chain.mdx | 1237 +++++++++++++++++ 2 files changed, 1238 insertions(+) create mode 100644 operators/chain-operators/tutorials/monitoring-chain.mdx diff --git a/docs.json b/docs.json index b80ce5121..36041d676 100644 --- a/docs.json +++ b/docs.json @@ -769,6 +769,7 @@ "operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup" ] }, + "operators/chain-operators/tutorials/monitoring-chain", "operators/chain-operators/tutorials/absolute-prestate", "operators/chain-operators/tutorials/adding-derivation-attributes", "operators/chain-operators/tutorials/adding-precompiles", diff --git a/operators/chain-operators/tutorials/monitoring-chain.mdx b/operators/chain-operators/tutorials/monitoring-chain.mdx new file mode 100644 index 000000000..8afafc2ee --- /dev/null +++ b/operators/chain-operators/tutorials/monitoring-chain.mdx @@ -0,0 +1,1237 @@ +--- +title: Monitoring your OP Stack chain +description: Learn how to set up comprehensive monitoring for your L2 rollup using Prometheus, Grafana, and OP Stack monitoring tools. +--- + +Now that you have a running OP Stack rollup, it's critical to set up monitoring to ensure your chain operates reliably and securely. This tutorial will guide you through setting up both onchain and offchain monitoring for your rollup. + + + This tutorial continues from [Creating Your Own L2 Rollup](/operators/chain-operators/tutorials/create-l2-rollup). Make sure you have completed that tutorial and have a running rollup before proceeding. + + +## What you'll set up + +By the end of this tutorial, you'll have comprehensive monitoring for: + +* **Onchain Components**: Dispute games, withdrawals, contract events +* **Offchain Components**: op-node, op-geth, op-batcher, op-proposer, op-challenger +* **Metrics Collection**: Prometheus scraping metrics from all components +* **Visualization**: Grafana dashboards for real-time monitoring +* **Alerting**: Basic alert rules for critical events + +## Prerequisites + +Before you begin, ensure you have: + +* Completed the [Creating Your Own L2 Rollup tutorial](/operators/chain-operators/tutorials/create-l2-rollup) +* All OP Stack components running (sequencer, batcher, proposer, challenger) +* Basic understanding of [monitoring concepts](/operators/chain-operators/tools/chain-monitoring) +* Docker installed (for Prometheus and Grafana) +* At least 10 GB of available disk space for monitoring data + +## Monitoring architecture overview + +Your monitoring setup will follow this architecture: + +```mermaid +graph TB + subgraph "OP Stack Components" + A[op-node
:7300] + B[op-geth
:6060] + C[op-batcher
:7300] + D[op-proposer
:7300] + E[op-challenger
:7300] + F[op-dispute-mon
:7301] + end + + subgraph "Monitoring stack" + G[Prometheus
:9090] + H[Grafana
:3000] + end + + A -->|metrics| G + B -->|metrics| G + C -->|metrics| G + D -->|metrics| G + E -->|metrics| G + F -->|metrics| G + G -->|data| H +``` + + + + +First, you need to enable metrics endpoints on all your OP Stack components. Most OP Stack services expose Prometheus-compatible metrics on port 7300 by default. + +### Enable op-node metrics + +Add these flags to your `op-node` startup script: + +```bash +#!/bin/bash +# In your rollup/sequencer directory + +# Add these metrics flags +--metrics.enabled=true \ +--metrics.addr=0.0.0.0 \ +--metrics.port=7300 +``` + + +```bash +#!/bin/bash +source ../.env + +../../optimism/op-node/bin/op-node \ + --l1=$L1_RPC_URL \ + --l2=ws://localhost:8551 \ + --l2.jwt-secret=./jwt.txt \ + --sequencer.enabled=true \ + --sequencer.l1-confs=5 \ + --verifier.l1-confs=4 \ + --rollup.config=./rollup.json \ + --rpc.addr=0.0.0.0 \ + --rpc.port=8547 \ + --p2p.disable=true \ + --rpc.enable-admin=true \ + --p2p.sequencer.key=$SEQUENCER_KEY \ + --l1.rpckind=$L1_RPC_KIND \ + --metrics.enabled=true \ + --metrics.addr=0.0.0.0 \ + --metrics.port=7300 +``` + + +### Enable op-geth metrics + +Add these flags to your `op-geth` startup script: + +```bash +#!/bin/bash +# In your rollup/sequencer directory + +# Add these metrics flags +--metrics \ +--metrics.addr=0.0.0.0 \ +--metrics.port=6060 +``` + + +```bash +#!/bin/bash +source ../.env + +../../optimism/op-geth/build/bin/geth \ + --datadir=./geth-data \ + --http \ + --http.corsdomain="*" \ + --http.vhosts="*" \ + --http.addr=0.0.0.0 \ + --http.port=8545 \ + --http.api=web3,debug,eth,txpool,net,engine \ + --ws \ + --ws.addr=0.0.0.0 \ + --ws.port=8546 \ + --ws.origins="*" \ + --ws.api=debug,eth,txpool,net,engine \ + --syncmode=full \ + --gcmode=archive \ + --nodiscover \ + --maxpeers=0 \ + --networkid=$CHAIN_ID \ + --authrpc.vhosts="*" \ + --authrpc.addr=0.0.0.0 \ + --authrpc.port=8551 \ + --authrpc.jwtsecret=./jwt.txt \ + --rollup.disabletxpoolgossip=true \ + --metrics \ + --metrics.addr=0.0.0.0 \ + --metrics.port=6060 +``` + + +### Enable op-batcher metrics + +Add these flags to your `op-batcher` startup script: + +```bash +#!/bin/bash +# In your rollup/batcher directory + +# Add these metrics flags +--metrics.enabled=true \ +--metrics.addr=0.0.0.0 \ +--metrics.port=7300 +``` + +### Enable op-proposer metrics + +Add these flags to your `op-proposer` startup script: + +```bash +#!/bin/bash +# In your rollup/proposer directory + +# Add these metrics flags +--metrics.enabled=true \ +--metrics.addr=0.0.0.0 \ +--metrics.port=7300 +``` + +### Enable op-challenger metrics + +Your `op-challenger` should already have metrics enabled from the [op-challenger setup tutorial](/operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup), but verify these flags are present: + +```bash +#!/bin/bash +# In your rollup/challenger directory + +# Verify these metrics flags +--metrics.enabled=true \ +--metrics.addr=0.0.0.0 \ +--metrics.port=7300 +``` + + + Make sure each component uses a unique metrics port if running on the same machine. The default is 7300 for most components, but op-geth uses 6060. + + +### Restart all components + +After adding metrics flags, restart all your OP Stack components: + +```bash +# Stop all components (Ctrl+C or kill processes) + +# Restart in order +cd rollup/sequencer && ./scripts/start-geth.sh & +sleep 10 +./scripts/start-node.sh & + +cd ../batcher && ./scripts/start-batcher.sh & +cd ../proposer && ./scripts/start-proposer.sh & +cd ../challenger && ./scripts/start-challenger.sh & +``` + +### Verify metrics endpoints + +Test that each component is exposing metrics: + +```bash +# Test op-node metrics +curl http://localhost:7300/metrics + +# Test op-geth metrics +curl http://localhost:6060/debug/metrics/prometheus + +# Test op-batcher metrics (use appropriate port if changed) +curl http://localhost:7301/metrics + +# Test op-proposer metrics +curl http://localhost:7302/metrics + +# Test op-challenger metrics +curl http://localhost:7303/metrics +``` + +Each command should return Prometheus-formatted metrics. + + + + + +`op-dispute-mon` is essential for monitoring the fault proof system. It tracks all dispute games and provides visibility into their status. + + + + +### Docker Setup + +The Docker setup provides a containerized environment for running op-dispute-mon with minimal configuration. + +#### Create dispute-mon directory + +```bash +# From your project root +cd rollup +mkdir dispute-mon +cd dispute-mon +``` + +#### Get contract addresses + +```bash +# Get Game Factory address +export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../.deployer/state.json) +echo "Game Factory Address: $GAME_FACTORY_ADDRESS" + +# Get Challenger address from your private key (using cast from Foundry) +export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Challenger Address: $CHALLENGER_ADDRESS" +``` + +#### Create environment file + +```bash +cat > .env << EOF +# Core Configuration +L1_RPC_URL=https://sepolia.infura.io/v3/YOUR_INFURA_KEY +ROLLUP_RPC=http://host.docker.internal:8547 + +# Contract Addresses +GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS +HONEST_ACTORS=$CHALLENGER_ADDRESS + +# Metrics Configuration +METRICS_PORT=7301 +EOF +``` + + + Replace `YOUR_INFURA_KEY` with your actual Infura API key or other L1 RPC endpoint. + + +#### Create Docker Compose file + +Create `docker-compose.yml`: + +```yaml +version: '3.8' + +services: + op-dispute-mon: + image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.5.1 + container_name: op-dispute-mon + ports: + - "7301:7301" + environment: + - OP_DISPUTE_MON_LOG_FORMAT=logfmt + - OP_DISPUTE_MON_METRICS_ENABLED=true + - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 + - OP_DISPUTE_MON_METRICS_PORT=${METRICS_PORT} + - OP_DISPUTE_MON_L1_ETH_RPC=${L1_RPC_URL} + - OP_DISPUTE_MON_ROLLUP_RPC=${ROLLUP_RPC} + - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} + - OP_DISPUTE_MON_HONEST_ACTORS=${HONEST_ACTORS} + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" +``` + +#### Start op-dispute-mon + +```bash +# Start the service +docker-compose up -d + +# View logs +docker-compose logs -f op-dispute-mon +``` + +#### Verify it's running + +```bash +# Check metrics endpoint +curl http://localhost:7301/metrics | grep dispute + +# Check Docker logs +docker-compose logs --tail=50 op-dispute-mon +``` + +You should see metrics like: +- `op_dispute_mon_games_total` - Total number of games +- `op_dispute_mon_games_status` - Game status counts +- `op_dispute_mon_claims_bond_total` - Total bonds + + + + + +### Build from Source + +For more control or custom builds, you can build op-dispute-mon from source. + +#### Build op-dispute-mon + +```bash +cd optimism/op-dispute-mon +make op-dispute-mon +``` + +#### Create directory structure + +```bash +# From your project root +cd rollup +mkdir -p dispute-mon/{scripts,data} +cd dispute-mon +``` + +#### Configure environment + +Create a `.env` file in `rollup/dispute-mon/`: + +```bash +# Copy from parent .env +cp ../.env .env + +# Add op-dispute-mon specific configuration +cat >> .env << 'EOF' + +# op-dispute-mon Configuration +OP_DISPUTE_MON_LOG_FORMAT=logfmt +OP_DISPUTE_MON_METRICS_ENABLED=true +OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 +OP_DISPUTE_MON_METRICS_PORT=7301 + +# L1 and L2 RPC endpoints (from parent .env) +OP_DISPUTE_MON_L1_ETH_RPC=$L1_RPC_URL +OP_DISPUTE_MON_ROLLUP_RPC=http://localhost:8547 + +# Game factory address (from deployment) +# Replace with your actual DisputeGameFactoryProxy address +OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS + +# Honest actor addresses (CSV, no spaces) +# This should be your challenger address(es) +OP_DISPUTE_MON_HONEST_ACTORS=$CHALLENGER_ADDRESS + +# Optional: Monitoring window (default is fine for most cases) +# OP_DISPUTE_MON_GAME_WINDOW=672h +# OP_DISPUTE_MON_MONITOR_INTERVAL=30s +# OP_DISPUTE_MON_MAX_CONCURRENCY=5 +EOF +``` + +#### Get your Game Factory address + +```bash +# From rollup/dispute-mon directory +cd ../.deployer +export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' state.json) +echo "Game Factory Address: $GAME_FACTORY_ADDRESS" + +# Add to dispute-mon .env +cd ../dispute-mon +echo "GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS" >> .env +``` + +#### Get your Challenger address + +```bash +# Get challenger address from your private key +# You can use cast (from Foundry) to derive the address +export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Challenger Address: $CHALLENGER_ADDRESS" + +# Add to dispute-mon .env +echo "CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS" >> .env +``` + +#### Create startup script + +Create `scripts/start-dispute-mon.sh`: + +```bash +#!/bin/bash +set -e + +# Load environment variables +source .env + +# Path to dispute-mon binary +DISPUTE_MON_BIN=../../optimism/op-dispute-mon/bin/op-dispute-mon + +# Start op-dispute-mon +$DISPUTE_MON_BIN \ + --log.format=$OP_DISPUTE_MON_LOG_FORMAT \ + --metrics.enabled=$OP_DISPUTE_MON_METRICS_ENABLED \ + --metrics.addr=$OP_DISPUTE_MON_METRICS_ADDR \ + --metrics.port=$OP_DISPUTE_MON_METRICS_PORT \ + --l1-eth-rpc=$OP_DISPUTE_MON_L1_ETH_RPC \ + --rollup-rpc=$OP_DISPUTE_MON_ROLLUP_RPC \ + --game-factory-address=$OP_DISPUTE_MON_GAME_FACTORY_ADDRESS \ + --honest-actors=$OP_DISPUTE_MON_HONEST_ACTORS +``` + +Make it executable: + +```bash +chmod +x scripts/start-dispute-mon.sh +``` + +#### Start op-dispute-mon + +```bash +# From rollup/dispute-mon directory +./scripts/start-dispute-mon.sh +``` + +#### Verify it's running + +```bash +# Check metrics endpoint +curl http://localhost:7301/metrics | grep dispute + +# Check logs +tail -f data/dispute-mon.log +``` + +You should see metrics like: +- `op_dispute_mon_games_total` - Total number of games +- `op_dispute_mon_games_status` - Game status counts +- `op_dispute_mon_claims_bond_total` - Total bonds + + + + + + + + +Prometheus will scrape metrics from all your OP Stack components. + +### Install Prometheus with Docker + +Create a monitoring directory: + +```bash +# From your project root +mkdir -p monitoring/{prometheus,grafana} +cd monitoring +``` + +### Create Prometheus configuration + +Create `prometheus/prometheus.yml`: + +```yaml +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + chain: 'my-op-rollup' + environment: 'testnet' + +scrape_configs: + # op-node metrics + - job_name: 'op-node' + static_configs: + - targets: ['host.docker.internal:7300'] + labels: + component: 'op-node' + role: 'consensus' + + # op-geth metrics + - job_name: 'op-geth' + metrics_path: '/debug/metrics/prometheus' + static_configs: + - targets: ['host.docker.internal:6060'] + labels: + component: 'op-geth' + role: 'execution' + + # op-batcher metrics + - job_name: 'op-batcher' + static_configs: + - targets: ['host.docker.internal:7301'] + labels: + component: 'op-batcher' + role: 'batcher' + + # op-proposer metrics + - job_name: 'op-proposer' + static_configs: + - targets: ['host.docker.internal:7302'] + labels: + component: 'op-proposer' + role: 'proposer' + + # op-challenger metrics + - job_name: 'op-challenger' + static_configs: + - targets: ['host.docker.internal:7303'] + labels: + component: 'op-challenger' + role: 'challenger' + + # op-dispute-mon metrics + - job_name: 'op-dispute-mon' + static_configs: + - targets: ['host.docker.internal:7304'] + labels: + component: 'op-dispute-mon' + role: 'monitor' +``` + + + **Port mapping note**: If you're running components on different ports, update the targets accordingly. The example above assumes standard ports with offsets for each component. + + +### Create Docker Compose configuration + +Create `docker-compose.yml` in the `monitoring` directory: + +```yaml +version: '3.8' + +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus-data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--web.enable-lifecycle' + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning + - ./grafana/dashboards:/var/lib/grafana/dashboards + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL=http://localhost:3000 + restart: unless-stopped + depends_on: + - prometheus + +volumes: + prometheus-data: + grafana-data: +``` + +### Start Prometheus and Grafana + +```bash +# From the monitoring directory +docker-compose up -d + +# Check logs +docker-compose logs -f +``` + +### Verify Prometheus is scraping metrics + +1. Open Prometheus in your browser: `http://localhost:9090` +2. Go to **Status > Targets** +3. Verify all targets show as "UP" + +If any targets show as "DOWN": +- Check that the component is running +- Verify the port numbers match your configuration +- Check firewall rules if running on different machines + + + + + +### Access Grafana + +1. Open Grafana: `http://localhost:3000` +2. Login with default credentials: + - Username: `admin` + - Password: `admin` +3. Change the password when prompted + +### Add Prometheus as data source + +1. Click **⚙️ Configuration** > **Data Sources** +2. Click **Add data source** +3. Select **Prometheus** +4. Configure: + - **Name**: `Prometheus` + - **URL**: `http://prometheus:9090` + - **Access**: `Server (default)` +5. Click **Save & Test** + +### Import OP Stack dashboards + +#### Import Dispute Monitor Dashboard + +1. Download the [Dispute Monitor JSON](/resources/grafana/dispute-monitor-1718214549035.json) +2. In Grafana, click **+ > Import** +3. Click **Upload JSON file** and select the downloaded file +4. Select **Prometheus** as the data source +5. Click **Import** + +#### Create custom OP Node dashboard + +Create a new dashboard for monitoring op-node: + +1. Click **+ > Create Dashboard** +2. Click **Add visualization** +3. Select **Prometheus** data source +4. Add panels for key metrics: + +**Panel 1: Sync Status** +- Metric: `op_node_default_refs_number` +- Visualization: Stat +- Title: "Sync Block Number" + +**Panel 2: Peer Count** +- Metric: `op_node_p2p_peers` +- Visualization: Time series +- Title: "P2P Peer Count" + +**Panel 3: L1 Head vs Unsafe L2 Head** +- Metrics: + - `op_node_default_refs_l1_head` + - `op_node_default_refs_unsafe_l2_head` +- Visualization: Time series +- Title: "L1 vs L2 Head" + +**Panel 4: Derivation Pipeline** +- Metric: `op_node_derivation_idle` +- Visualization: Time series +- Title: "Derivation Idle Time" + +5. Click **Save dashboard** +6. Name it "OP Node Metrics" + +#### Create OP Geth dashboard + +1. Create a new dashboard +2. Add visualizations for: + +**Panel 1: Gas Price** +- Metric: `geth_tx_pool_basefee` +- Title: "Base Fee" + +**Panel 2: Transaction Pool** +- Metric: `geth_tx_pool_pending` +- Title: "Pending Transactions" + +**Panel 3: Block Processing** +- Metric: `geth_chain_head_block` +- Title: "Chain Head Block" + +**Panel 4: State Database Size** +- Metric: `geth_db_chaindata_disk_size` +- Title: "Database Size" + +3. Save as "OP Geth Metrics" + +#### Create Batcher dashboard + +1. Create a new dashboard +2. Add key metrics: + +**Panel 1: Published Batches** +- Metric: `op_batcher_batch_submitted_count` +- Title: "Batches Submitted" + +**Panel 2: Batch Size** +- Metric: `op_batcher_batch_size_bytes` +- Title: "Average Batch Size" + +**Panel 3: Gas Used** +- Metric: `op_batcher_total_gas_used` +- Title: "Total Gas Used" + +3. Save as "OP Batcher Metrics" + + + + + +For additional security monitoring, you can set up [Monitorism](/operators/chain-operators/tools/chain-monitoring#monitorism). + +### Clone Monitorism repository + +```bash +cd ~/ +git clone https://github.com/ethereum-optimism/monitorism.git +cd monitorism +``` + +### Configure Monitorism + +Create a configuration file `config.toml`: + +```toml +[server] +host = "0.0.0.0" +port = 7305 + +[l1] +rpc_url = "YOUR_L1_RPC_URL" + +[l2] +rpc_url = "http://localhost:8545" + +[[monitors]] +type = "fault_proof_withdrawal" +enabled = true +[monitors.config] +optimism_portal_address = "YOUR_OPTIMISM_PORTAL_ADDRESS" +l2_output_oracle_address = "YOUR_L2_OUTPUT_ORACLE_ADDRESS" +``` + +### Get contract addresses + +```bash +# From your deployment state +cd your-project/rollup/.deployer +jq -r '{optimismPortal: .optimismPortalProxyAddress, l2OutputOracle: .l2OutputOracleProxyAddress}' state.json +``` + +### Build and run Monitorism + +```bash +cd ~/monitorism +go build -o bin/monitorism ./cmd/monitorism + +# Run monitorism +./bin/monitorism --config config.toml +``` + +### Add Monitorism to Prometheus + +Add to `prometheus/prometheus.yml`: + +```yaml + - job_name: 'monitorism' + static_configs: + - targets: ['host.docker.internal:7305'] + labels: + component: 'monitorism' + role: 'monitor' +``` + +Reload Prometheus configuration: + +```bash +curl -X POST http://localhost:9090/-/reload +``` + + + + + +Create alert rules in Prometheus for critical events. + +### Create alert rules file + +Create `prometheus/alerts.yml`: + +```yaml +groups: + - name: op_stack_alerts + interval: 30s + rules: + # Sequencer health + - alert: SequencerDown + expr: up{job="op-node"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Sequencer (op-node) is down" + description: "The sequencer has been down for more than 2 minutes" + + - alert: ExecutionClientDown + expr: up{job="op-geth"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Execution client (op-geth) is down" + description: "The execution client has been down for more than 2 minutes" + + # Batcher alerts + - alert: BatcherDown + expr: up{job="op-batcher"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Batcher is down" + description: "The batcher has been down for more than 5 minutes" + + - alert: BatcherNotSubmitting + expr: rate(op_batcher_batch_submitted_count[10m]) == 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Batcher not submitting batches" + description: "No batches have been submitted in the last 15 minutes" + + # Proposer alerts + - alert: ProposerDown + expr: up{job="op-proposer"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Proposer is down" + description: "The proposer has been down for more than 5 minutes" + + # Challenger alerts + - alert: ChallengerDown + expr: up{job="op-challenger"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Challenger is down" + description: "The challenger has been down for more than 5 minutes" + + # Dispute game alerts + - alert: InvalidDisputeGame + expr: op_dispute_mon_games_status{status="invalid"} > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Invalid dispute game detected" + description: "An invalid dispute game has been detected and needs immediate attention" + + # Sync lag alerts + - alert: SequencerSyncLag + expr: (op_node_default_refs_l1_head - op_node_default_refs_unsafe_l2_head) > 100 + for: 10m + labels: + severity: warning + annotations: + summary: "Sequencer sync lag detected" + description: "L2 is lagging behind L1 by more than 100 blocks" + + # Resource alerts + - alert: HighMemoryUsage + expr: process_resident_memory_bytes / 1e9 > 8 + for: 5m + labels: + severity: warning + annotations: + summary: "High memory usage detected" + description: "Component {{ $labels.job }} is using more than 8GB of memory" +``` + +### Update Prometheus configuration + +Update `prometheus/prometheus.yml` to include the alert rules: + +```yaml +global: + scrape_interval: 15s + evaluation_interval: 15s + +# Add this section +rule_files: + - 'alerts.yml' + +# ... rest of configuration ... +``` + +### Reload Prometheus + +```bash +docker-compose restart prometheus +``` + +### Verify alerts in Prometheus + +1. Open Prometheus: `http://localhost:9090` +2. Go to **Alerts** +3. Verify your alert rules are loaded + + + + + +For alert notifications, you can configure Alertmanager. + +### Add Alertmanager to Docker Compose + +Update `docker-compose.yml`: + +```yaml + alertmanager: + image: prom/alertmanager:latest + container_name: alertmanager + ports: + - "9093:9093" + volumes: + - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml + - alertmanager-data:/alertmanager + command: + - '--config.file=/etc/alertmanager/alertmanager.yml' + - '--storage.path=/alertmanager' + restart: unless-stopped + +volumes: + prometheus-data: + grafana-data: + alertmanager-data: # Add this +``` + +### Create Alertmanager configuration + +Create `alertmanager/alertmanager.yml`: + +```yaml +global: + resolve_timeout: 5m + +route: + group_by: ['alertname', 'cluster'] + group_wait: 10s + group_interval: 10s + repeat_interval: 12h + receiver: 'default' + +receivers: + - name: 'default' + # Configure your preferred notification channel: + + # Slack example: + # slack_configs: + # - api_url: 'YOUR_SLACK_WEBHOOK_URL' + # channel: '#alerts' + # title: 'OP Stack Alert' + # text: '{{ range .Alerts }}{{ .Annotations.summary }}\n{{ .Annotations.description }}{{ end }}' + + # Discord example: + # discord_configs: + # - webhook_url: 'YOUR_DISCORD_WEBHOOK_URL' + + # Email example: + # email_configs: + # - to: 'your-email@example.com' + # from: 'alertmanager@example.com' + # smarthost: 'smtp.gmail.com:587' + # auth_username: 'your-email@example.com' + # auth_password: 'your-app-password' + +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'cluster'] +``` + +### Update Prometheus to use Alertmanager + +Update `prometheus/prometheus.yml`: + +```yaml +# Add this section +alerting: + alertmanagers: + - static_configs: + - targets: ['alertmanager:9093'] + +# ... rest of configuration ... +``` + +### Restart monitoring stack + +```bash +docker-compose up -d +``` + + + + + +Here are the key metrics you should monitor for each component: + +### OP Node (Sequencer) Metrics + + + +| Metric | Description | Alert Threshold | +|--------|-------------|----------------| +| `op_node_default_refs_number` | Current sync block number | Should increase steadily | +| `op_node_default_refs_l1_head` | L1 head block | Track sync with L1 | +| `op_node_derivation_idle` | Derivation pipeline idle time | > 30s indicates issues | +| `op_node_p2p_peers` | Number of P2P peers | Should be > 0 if P2P enabled | +| `process_cpu_seconds_total` | CPU usage | Monitor for spikes | + +[See complete list in Node Metrics guide](/operators/node-operators/management/metrics#important-metrics) + + + +### OP Geth Metrics + + + +| Metric | Description | Alert Threshold | +|--------|-------------|----------------| +| `geth_chain_head_block` | Current head block | Should increase | +| `geth_tx_pool_pending` | Pending transactions | Monitor for buildup | +| `geth_db_chaindata_disk_size` | Database size | Monitor growth rate | +| `geth_tx_pool_basefee` | Current base fee | Monitor for spikes | + + + +### OP Batcher Metrics + + + +| Metric | Description | Alert Threshold | +|--------|-------------|----------------| +| `op_batcher_batch_submitted_count` | Number of batches submitted | Should increase regularly | +| `op_batcher_batch_size_bytes` | Average batch size | Monitor size trends | +| `op_batcher_total_gas_used` | Gas used for submissions | Track costs | +| `op_batcher_channel_open_duration` | Time channels are open | Alert if too high | + + + +### OP Proposer Metrics + + + +| Metric | Description | Alert Threshold | +|--------|-------------|----------------| +| `op_proposer_proposals_total` | Total proposals submitted | Should increase | +| `op_proposer_gas_used` | Gas used for proposals | Track costs | +| `op_proposer_latest_proposal_block` | Latest proposed block | Should advance | + + + +### OP Challenger Metrics + + + +| Metric | Description | Alert Threshold | +|--------|-------------|----------------| +| `op_challenger_games_total` | Total games tracked | Monitor activity | +| `op_challenger_moves_made` | Moves made in games | Track participation | +| `op_challenger_claims_invalid` | Invalid claims detected | Alert immediately if > 0 | + + + +### OP Dispute Mon Metrics + + + +| Metric | Description | Alert Threshold | +|--------|-------------|----------------| +| `op_dispute_mon_games_total` | Total games monitored | Track overall activity | +| `op_dispute_mon_games_status` | Games by status | Alert on invalid games | +| `op_dispute_mon_claims_bond_total` | Total bonds | Monitor economic activity | + + + + + + +## Monitoring checklist + +Use this checklist to ensure comprehensive monitoring: + +- ✅ All OP Stack components have metrics enabled +- ✅ Prometheus is scraping all targets successfully +- ✅ Grafana dashboards are displaying data +- ✅ Alert rules are configured and active +- ✅ op-dispute-mon is running and tracking games +- ✅ Disk space is monitored (Prometheus and component data) +- ✅ Backup monitoring data regularly +- ✅ Test alert notifications are working +- ✅ Document your monitoring setup for your team + +## Troubleshooting + +### Prometheus not scraping metrics + +**Issue**: Targets show as "DOWN" in Prometheus + +**Solutions**: +1. Verify component is running: `ps aux | grep op-node` +2. Test metrics endpoint directly: `curl http://localhost:7300/metrics` +3. Check firewall rules +4. Verify Docker network can reach `host.docker.internal` +5. Check Prometheus logs: `docker-compose logs prometheus` + +### High memory usage + +**Issue**: Components using excessive memory + +**Solutions**: +1. Check database sizes: `du -sh rollup/*/data` +2. Consider pruning old data +3. Increase system resources +4. Check for memory leaks in logs + +### Grafana not displaying data + +**Issue**: Dashboards show "No data" + +**Solutions**: +1. Verify Prometheus data source is configured correctly +2. Check that Prometheus is scraping metrics +3. Verify metric names in queries match actual metrics +4. Check time range in dashboard + +### Dispute mon not tracking games + +**Issue**: `op-dispute-mon` shows no games + +**Solutions**: +1. Verify Game Factory address is correct +2. Check that challenger has created some games +3. Verify L1 and L2 RPC endpoints are working +4. Check `op-dispute-mon` logs for errors + +## Production considerations + +When running monitoring in production: + +1. **Security**: + - Restrict access to Grafana with proper authentication + - Use TLS for all connections + - Don't expose metrics ports publicly + - Regularly update monitoring tools + +2. **Reliability**: + - Run Prometheus and Grafana on separate infrastructure + - Set up Prometheus in HA mode for production + - Configure retention policies based on your needs + - Set up offsite backups of monitoring data + +3. **Scalability**: + - Consider using Prometheus federation for multiple chains + - Use remote storage for long-term metrics + - Implement proper resource limits in Docker + +4. **Alerting**: + - Configure multiple notification channels + - Set up escalation policies + - Test alerts regularly + - Document runbooks for each alert + +## Next steps + +* **Learn more about monitoring options**: [Chain Monitoring Reference](/operators/chain-operators/tools/chain-monitoring) +* **Understand dispute games**: [Dispute Games Guide](/operators/chain-operators/tutorials/dispute-games) +* **Set up additional monitors**: [Monitorism Repository](https://github.com/ethereum-optimism/monitorism) +* **Optimize your setup**: [Operations Guide](/operators/chain-operators/management/operations) +* **Get support**: [Developer Discussions](https://github.com/ethereum-optimism/developers/discussions) + +## Additional resources + +* [Prometheus Documentation](https://prometheus.io/docs/) +* [Grafana Documentation](https://grafana.com/docs/) +* [Node Metrics Guide](/operators/node-operators/management/metrics) +* [Monitorism GitHub](https://github.com/ethereum-optimism/monitorism) +* [OP Dispute Mon](https://github.com/ethereum-optimism/optimism/tree/develop/op-dispute-mon) + From 6f9f05b7097166ccac4f960a4032a4ebb3e54fd7 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Mon, 20 Oct 2025 17:23:19 +0100 Subject: [PATCH 02/17] feat: update monitoring tutorial for OP Stack with op-dispute-mon - Revise tutorial to focus on monitoring dispute games using op-dispute-mon - Update prerequisites and setup instructions for Docker and source builds - Enhance metrics collection and visualization sections with detailed examples - Include new metrics and monitoring architecture for better clarity - Streamline steps for deploying and verifying op-dispute-mon - Add comprehensive explanations for key metrics and their significance - Improve troubleshooting and production considerations for monitoring setup --- .../tutorials/monitoring-chain.mdx | 1358 +++++------------ 1 file changed, 353 insertions(+), 1005 deletions(-) diff --git a/operators/chain-operators/tutorials/monitoring-chain.mdx b/operators/chain-operators/tutorials/monitoring-chain.mdx index 8afafc2ee..eea6467ed 100644 --- a/operators/chain-operators/tutorials/monitoring-chain.mdx +++ b/operators/chain-operators/tutorials/monitoring-chain.mdx @@ -1,327 +1,228 @@ --- -title: Monitoring your OP Stack chain -description: Learn how to set up comprehensive monitoring for your L2 rollup using Prometheus, Grafana, and OP Stack monitoring tools. +title: Monitoring your OP Stack chain with Dispute Monitor +description: Learn how to set up op-dispute-mon to monitor dispute games and fault proof activity on your L2 rollup. --- -Now that you have a running OP Stack rollup, it's critical to set up monitoring to ensure your chain operates reliably and securely. This tutorial will guide you through setting up both onchain and offchain monitoring for your rollup. +Now that you have a running OP Stack rollup with fault proofs enabled, it's critical to monitor your dispute games to ensure your chain operates securely. This tutorial will guide you through setting up `op-dispute-mon` to track all dispute game activity on your rollup. - This tutorial continues from [Creating Your Own L2 Rollup](/operators/chain-operators/tutorials/create-l2-rollup). Make sure you have completed that tutorial and have a running rollup before proceeding. + This tutorial continues from [Creating Your Own L2 Rollup](/operators/chain-operators/tutorials/create-l2-rollup). Make sure you have completed the op-challenger setup and have dispute games running before proceeding. ## What you'll set up -By the end of this tutorial, you'll have comprehensive monitoring for: +By the end of this tutorial, you'll have: -* **Onchain Components**: Dispute games, withdrawals, contract events -* **Offchain Components**: op-node, op-geth, op-batcher, op-proposer, op-challenger -* **Metrics Collection**: Prometheus scraping metrics from all components -* **Visualization**: Grafana dashboards for real-time monitoring -* **Alerting**: Basic alert rules for critical events +* **op-dispute-mon** monitoring all dispute games on your chain +* **Metrics endpoint** exposing Prometheus-compatible metrics +* **Real-time tracking** of game status, agreements, and honest actor participation +* Understanding of key metrics for dispute game health ## Prerequisites Before you begin, ensure you have: * Completed the [Creating Your Own L2 Rollup tutorial](/operators/chain-operators/tutorials/create-l2-rollup) -* All OP Stack components running (sequencer, batcher, proposer, challenger) -* Basic understanding of [monitoring concepts](/operators/chain-operators/tools/chain-monitoring) -* Docker installed (for Prometheus and Grafana) -* At least 10 GB of available disk space for monitoring data +* op-challenger running and participating in dispute games +* op-proposer running and creating proposals +* L1 RPC endpoint with sufficient rate limits +* Rollup RPC endpoint (op-node) +* Docker installed (for Docker setup) OR Go 1.21+ (for build from source) -## Monitoring architecture overview +## What is op-dispute-mon? -Your monitoring setup will follow this architecture: +`op-dispute-mon` is a specialized monitoring tool that tracks dispute games in the fault proof system. It provides visibility into: + +* **Game Status**: Track games from creation to resolution +* **Honest Actor Participation**: Monitor your proposer and challenger activity +* **Agreement Tracking**: See which games agree/disagree with your honest actors +* **Game Completion**: Monitor in-progress vs completed games +* **Ignored Games**: Track games that don't meet monitoring criteria + +The monitor exposes Prometheus metrics that can be used to build dashboards and alerts. + +## Monitoring architecture ```mermaid graph TB - subgraph "OP Stack Components" - A[op-node
:7300] - B[op-geth
:6060] - C[op-batcher
:7300] - D[op-proposer
:7300] - E[op-challenger
:7300] - F[op-dispute-mon
:7301] + subgraph "L1 Network" + A[DisputeGameFactory
Contract] + end + + subgraph "Your Rollup" + B[op-proposer
Creates proposals] + C[op-challenger
Defends proposals] + D[op-node
Rollup RPC] end - subgraph "Monitoring stack" - G[Prometheus
:9090] - H[Grafana
:3000] + subgraph "Monitoring" + E[op-dispute-mon
:7300] end - A -->|metrics| G - B -->|metrics| G - C -->|metrics| G - D -->|metrics| G - E -->|metrics| G - F -->|metrics| G - G -->|data| H + A -->|Reads games| E + D -->|Rollup state| E + E -->|Tracks| B + E -->|Tracks| C + E -->|Exposes metrics| F[/metrics endpoint] ``` - + -First, you need to enable metrics endpoints on all your OP Stack components. Most OP Stack services expose Prometheus-compatible metrics on port 7300 by default. +Choose your preferred deployment method: Docker (recommended) or build from source. -### Enable op-node metrics + + -Add these flags to your `op-node` startup script: +### Docker Setup -```bash -#!/bin/bash -# In your rollup/sequencer directory +The Docker setup provides a containerized environment for running op-dispute-mon with minimal configuration. -# Add these metrics flags ---metrics.enabled=true \ ---metrics.addr=0.0.0.0 \ ---metrics.port=7300 -``` +#### Create dispute-mon directory - ```bash -#!/bin/bash -source ../.env - -../../optimism/op-node/bin/op-node \ - --l1=$L1_RPC_URL \ - --l2=ws://localhost:8551 \ - --l2.jwt-secret=./jwt.txt \ - --sequencer.enabled=true \ - --sequencer.l1-confs=5 \ - --verifier.l1-confs=4 \ - --rollup.config=./rollup.json \ - --rpc.addr=0.0.0.0 \ - --rpc.port=8547 \ - --p2p.disable=true \ - --rpc.enable-admin=true \ - --p2p.sequencer.key=$SEQUENCER_KEY \ - --l1.rpckind=$L1_RPC_KIND \ - --metrics.enabled=true \ - --metrics.addr=0.0.0.0 \ - --metrics.port=7300 +# From your project root +cd rollup +mkdir dispute-mon +cd dispute-mon ``` - -### Enable op-geth metrics +#### Get required addresses -Add these flags to your `op-geth` startup script: +You need three key addresses for op-dispute-mon: -```bash -#!/bin/bash -# In your rollup/sequencer directory - -# Add these metrics flags ---metrics \ ---metrics.addr=0.0.0.0 \ ---metrics.port=6060 -``` +1. **DisputeGameFactory address** (from your deployment) +2. **Proposer address** (your honest actor who creates proposals) +3. **Challenger address** (your honest actor who defends proposals) - ```bash -#!/bin/bash -source ../.env - -../../optimism/op-geth/build/bin/geth \ - --datadir=./geth-data \ - --http \ - --http.corsdomain="*" \ - --http.vhosts="*" \ - --http.addr=0.0.0.0 \ - --http.port=8545 \ - --http.api=web3,debug,eth,txpool,net,engine \ - --ws \ - --ws.addr=0.0.0.0 \ - --ws.port=8546 \ - --ws.origins="*" \ - --ws.api=debug,eth,txpool,net,engine \ - --syncmode=full \ - --gcmode=archive \ - --nodiscover \ - --maxpeers=0 \ - --networkid=$CHAIN_ID \ - --authrpc.vhosts="*" \ - --authrpc.addr=0.0.0.0 \ - --authrpc.port=8551 \ - --authrpc.jwtsecret=./jwt.txt \ - --rollup.disabletxpoolgossip=true \ - --metrics \ - --metrics.addr=0.0.0.0 \ - --metrics.port=6060 -``` - - -### Enable op-batcher metrics - -Add these flags to your `op-batcher` startup script: +# Get DisputeGameFactory address from deployment +export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../.deployer/state.json) +echo "Game Factory Address: $GAME_FACTORY_ADDRESS" -```bash -#!/bin/bash -# In your rollup/batcher directory +# Get Proposer address (from your proposer setup) +source ../proposer/.env +export PROPOSER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Proposer Address: $PROPOSER_ADDRESS" -# Add these metrics flags ---metrics.enabled=true \ ---metrics.addr=0.0.0.0 \ ---metrics.port=7300 +# Get Challenger address (from your challenger setup) +source ../challenger/.env +export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Challenger Address: $CHALLENGER_ADDRESS" ``` -### Enable op-proposer metrics - -Add these flags to your `op-proposer` startup script: - -```bash -#!/bin/bash -# In your rollup/proposer directory - -# Add these metrics flags ---metrics.enabled=true \ ---metrics.addr=0.0.0.0 \ ---metrics.port=7300 -``` +#### Create Docker Compose file -### Enable op-challenger metrics +Create `docker-compose.yml` with the following configuration: -Your `op-challenger` should already have metrics enabled from the [op-challenger setup tutorial](/operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup), but verify these flags are present: +```yaml +version: "3.8" -```bash -#!/bin/bash -# In your rollup/challenger directory +services: + op-dispute-mon: + image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.4.2-rc.1 + container_name: dispute-mon + ports: + - "7300:7300" # Metrics port + environment: + # L1 RPC Configuration + - OP_DISPUTE_MON_L1_ETH_RPC=https://sepolia.infura.io/v3/YOUR_INFURA_KEY + + # Rollup RPC Configuration (your op-node) + - OP_DISPUTE_MON_ROLLUP_RPC=http://host.docker.internal:8547 + + # Honest Actors (your proposer and challenger addresses) + - OP_DISPUTE_MON_HONEST_ACTORS=${PROPOSER_ADDRESS},${CHALLENGER_ADDRESS} + + # DisputeGameFactory Contract Address + - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} + + # Monitoring Configuration + - OP_DISPUTE_MON_MONITOR_INTERVAL=10s + + # Logging Configuration + - OP_DISPUTE_MON_LOG_FORMAT=logfmt + - OP_DISPUTE_MON_LOG_LEVEL=info + + # Metrics Configuration + - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 + - OP_DISPUTE_MON_METRICS_ENABLED=true + - OP_DISPUTE_MON_METRICS_PORT=7300 + restart: unless-stopped + networks: + - dispute-mon-network -# Verify these metrics flags ---metrics.enabled=true \ ---metrics.addr=0.0.0.0 \ ---metrics.port=7300 +networks: + dispute-mon-network: + driver: bridge ``` - Make sure each component uses a unique metrics port if running on the same machine. The default is 7300 for most components, but op-geth uses 6060. + Replace `YOUR_INFURA_KEY` with your actual Infura API key or use another L1 RPC provider. Make sure your L1 RPC has sufficient rate limits for production use. -### Restart all components - -After adding metrics flags, restart all your OP Stack components: - -```bash -# Stop all components (Ctrl+C or kill processes) - -# Restart in order -cd rollup/sequencer && ./scripts/start-geth.sh & -sleep 10 -./scripts/start-node.sh & - -cd ../batcher && ./scripts/start-batcher.sh & -cd ../proposer && ./scripts/start-proposer.sh & -cd ../challenger && ./scripts/start-challenger.sh & -``` - -### Verify metrics endpoints - -Test that each component is exposing metrics: - -```bash -# Test op-node metrics -curl http://localhost:7300/metrics - -# Test op-geth metrics -curl http://localhost:6060/debug/metrics/prometheus - -# Test op-batcher metrics (use appropriate port if changed) -curl http://localhost:7301/metrics - -# Test op-proposer metrics -curl http://localhost:7302/metrics - -# Test op-challenger metrics -curl http://localhost:7303/metrics -``` - -Each command should return Prometheus-formatted metrics. - - - - - -`op-dispute-mon` is essential for monitoring the fault proof system. It tracks all dispute games and provides visibility into their status. - - - - -### Docker Setup - -The Docker setup provides a containerized environment for running op-dispute-mon with minimal configuration. - -#### Create dispute-mon directory - -```bash -# From your project root -cd rollup -mkdir dispute-mon -cd dispute-mon -``` - -#### Get contract addresses - -```bash -# Get Game Factory address -export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../.deployer/state.json) -echo "Game Factory Address: $GAME_FACTORY_ADDRESS" - -# Get Challenger address from your private key (using cast from Foundry) -export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) -echo "Challenger Address: $CHALLENGER_ADDRESS" -``` +#### Create environment file (optional) -#### Create environment file +If you prefer using an environment file instead of inline environment variables: ```bash cat > .env << EOF -# Core Configuration +# L1 Configuration L1_RPC_URL=https://sepolia.infura.io/v3/YOUR_INFURA_KEY + +# Rollup Configuration ROLLUP_RPC=http://host.docker.internal:8547 # Contract Addresses GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS -HONEST_ACTORS=$CHALLENGER_ADDRESS -# Metrics Configuration -METRICS_PORT=7301 +# Honest Actors +PROPOSER_ADDRESS=$PROPOSER_ADDRESS +CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS EOF ``` - - Replace `YOUR_INFURA_KEY` with your actual Infura API key or other L1 RPC endpoint. - - -#### Create Docker Compose file - -Create `docker-compose.yml`: +Then update your `docker-compose.yml` to use the environment file: ```yaml -version: '3.8' +version: "3.8" services: op-dispute-mon: - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.5.1 - container_name: op-dispute-mon + image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.4.2-rc.1 + container_name: dispute-mon + env_file: .env ports: - - "7301:7301" + - "7300:7300" environment: - - OP_DISPUTE_MON_LOG_FORMAT=logfmt - - OP_DISPUTE_MON_METRICS_ENABLED=true - - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 - - OP_DISPUTE_MON_METRICS_PORT=${METRICS_PORT} - OP_DISPUTE_MON_L1_ETH_RPC=${L1_RPC_URL} - OP_DISPUTE_MON_ROLLUP_RPC=${ROLLUP_RPC} + - OP_DISPUTE_MON_HONEST_ACTORS=${PROPOSER_ADDRESS},${CHALLENGER_ADDRESS} - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} - - OP_DISPUTE_MON_HONEST_ACTORS=${HONEST_ACTORS} + - OP_DISPUTE_MON_MONITOR_INTERVAL=10s + - OP_DISPUTE_MON_LOG_FORMAT=logfmt + - OP_DISPUTE_MON_LOG_LEVEL=info + - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 + - OP_DISPUTE_MON_METRICS_ENABLED=true + - OP_DISPUTE_MON_METRICS_PORT=7300 restart: unless-stopped - extra_hosts: - - "host.docker.internal:host-gateway" + networks: + - dispute-mon-network + +networks: + dispute-mon-network: + driver: bridge ``` #### Start op-dispute-mon ```bash -# Start the service +# Export variables if not using .env file +export PROPOSER_ADDRESS=0xYourProposerAddress +export CHALLENGER_ADDRESS=0xYourChallengerAddress +export GAME_FACTORY_ADDRESS=0xYourGameFactoryAddress + +# Start the container docker-compose up -d # View logs @@ -330,32 +231,59 @@ docker-compose logs -f op-dispute-mon #### Verify it's running +Check that the metrics endpoint is accessible: + ```bash -# Check metrics endpoint -curl http://localhost:7301/metrics | grep dispute +# Query metrics endpoint +curl http://localhost:7300/metrics -# Check Docker logs +# You should see Prometheus metrics including: +# - op_dispute_mon_games_agreement +# - op_dispute_mon_ignored_games +# - and many more... +``` + +Check logs for any errors: + +```bash +# View recent logs docker-compose logs --tail=50 op-dispute-mon + +# Follow logs in real-time +docker-compose logs -f op-dispute-mon ``` -You should see metrics like: -- `op_dispute_mon_games_total` - Total number of games -- `op_dispute_mon_games_status` - Game status counts -- `op_dispute_mon_claims_bond_total` - Total bonds +You should see logs indicating the monitor is tracking games: + +``` +level=info msg="Monitoring dispute games" factory=0x... honest_actors=[0x..., 0x...] +level=info msg="Found games" count=5 +level=info msg="Updated game status" game=0x... status=IN_PROGRESS +``` - + ### Build from Source -For more control or custom builds, you can build op-dispute-mon from source. +Building from source gives you more control and is useful for development or custom deployments. -#### Build op-dispute-mon +#### Clone and build op-dispute-mon ```bash -cd optimism/op-dispute-mon +# From your project root +cd optimism + +# Pull latest changes +git pull + +# Build op-dispute-mon +cd op-dispute-mon make op-dispute-mon + +# Verify the build +./bin/op-dispute-mon --version ``` #### Create directory structure @@ -363,69 +291,56 @@ make op-dispute-mon ```bash # From your project root cd rollup -mkdir -p dispute-mon/{scripts,data} +mkdir -p dispute-mon/scripts cd dispute-mon ``` -#### Configure environment - -Create a `.env` file in `rollup/dispute-mon/`: +#### Get required addresses ```bash -# Copy from parent .env -cp ../.env .env - -# Add op-dispute-mon specific configuration -cat >> .env << 'EOF' - -# op-dispute-mon Configuration -OP_DISPUTE_MON_LOG_FORMAT=logfmt -OP_DISPUTE_MON_METRICS_ENABLED=true -OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 -OP_DISPUTE_MON_METRICS_PORT=7301 - -# L1 and L2 RPC endpoints (from parent .env) -OP_DISPUTE_MON_L1_ETH_RPC=$L1_RPC_URL -OP_DISPUTE_MON_ROLLUP_RPC=http://localhost:8547 - -# Game factory address (from deployment) -# Replace with your actual DisputeGameFactoryProxy address -OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS - -# Honest actor addresses (CSV, no spaces) -# This should be your challenger address(es) -OP_DISPUTE_MON_HONEST_ACTORS=$CHALLENGER_ADDRESS - -# Optional: Monitoring window (default is fine for most cases) -# OP_DISPUTE_MON_GAME_WINDOW=672h -# OP_DISPUTE_MON_MONITOR_INTERVAL=30s -# OP_DISPUTE_MON_MAX_CONCURRENCY=5 -EOF -``` +# Get DisputeGameFactory address +export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../.deployer/state.json) +echo "Game Factory Address: $GAME_FACTORY_ADDRESS" -#### Get your Game Factory address +# Get Proposer address +source ../proposer/.env +export PROPOSER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Proposer Address: $PROPOSER_ADDRESS" -```bash -# From rollup/dispute-mon directory -cd ../.deployer -export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' state.json) -echo "Game Factory Address: $GAME_FACTORY_ADDRESS" +# Get Challenger address +source ../challenger/.env +export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Challenger Address: $CHALLENGER_ADDRESS" -# Add to dispute-mon .env -cd ../dispute-mon -echo "GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS" >> .env +# Get L1 RPC URL +source ../.env +echo "L1 RPC: $L1_RPC_URL" ``` -#### Get your Challenger address +#### Create environment file ```bash -# Get challenger address from your private key -# You can use cast (from Foundry) to derive the address -export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) -echo "Challenger Address: $CHALLENGER_ADDRESS" +cat > .env << EOF +# L1 Configuration +L1_RPC_URL=$L1_RPC_URL + +# Rollup Configuration +ROLLUP_RPC=http://localhost:8547 + +# Contract Addresses +GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS + +# Honest Actors +PROPOSER_ADDRESS=$PROPOSER_ADDRESS +CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS + +# Monitoring Configuration +MONITOR_INTERVAL=10s +LOG_LEVEL=info -# Add to dispute-mon .env -echo "CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS" >> .env +# Metrics Configuration +METRICS_PORT=7300 +EOF ``` #### Create startup script @@ -439,19 +354,25 @@ set -e # Load environment variables source .env -# Path to dispute-mon binary +# Path to op-dispute-mon binary DISPUTE_MON_BIN=../../optimism/op-dispute-mon/bin/op-dispute-mon -# Start op-dispute-mon +echo "Starting op-dispute-mon..." +echo "Game Factory: $GAME_FACTORY_ADDRESS" +echo "Proposer: $PROPOSER_ADDRESS" +echo "Challenger: $CHALLENGER_ADDRESS" + $DISPUTE_MON_BIN \ - --log.format=$OP_DISPUTE_MON_LOG_FORMAT \ - --metrics.enabled=$OP_DISPUTE_MON_METRICS_ENABLED \ - --metrics.addr=$OP_DISPUTE_MON_METRICS_ADDR \ - --metrics.port=$OP_DISPUTE_MON_METRICS_PORT \ - --l1-eth-rpc=$OP_DISPUTE_MON_L1_ETH_RPC \ - --rollup-rpc=$OP_DISPUTE_MON_ROLLUP_RPC \ - --game-factory-address=$OP_DISPUTE_MON_GAME_FACTORY_ADDRESS \ - --honest-actors=$OP_DISPUTE_MON_HONEST_ACTORS + --l1-eth-rpc=$L1_RPC_URL \ + --rollup-rpc=$ROLLUP_RPC \ + --game-factory-address=$GAME_FACTORY_ADDRESS \ + --honest-actors=$PROPOSER_ADDRESS,$CHALLENGER_ADDRESS \ + --monitor-interval=$MONITOR_INTERVAL \ + --log.level=$LOG_LEVEL \ + --log.format=logfmt \ + --metrics.enabled=true \ + --metrics.addr=0.0.0.0 \ + --metrics.port=$METRICS_PORT ``` Make it executable: @@ -463,775 +384,202 @@ chmod +x scripts/start-dispute-mon.sh #### Start op-dispute-mon ```bash -# From rollup/dispute-mon directory +# Start in foreground (for testing) ./scripts/start-dispute-mon.sh + +# Or start in background +./scripts/start-dispute-mon.sh > logs/dispute-mon.log 2>&1 & + +# Save the process ID +echo $! > dispute-mon.pid ``` #### Verify it's running ```bash -# Check metrics endpoint -curl http://localhost:7301/metrics | grep dispute +# Check if process is running +ps aux | grep op-dispute-mon + +# Query metrics endpoint +curl http://localhost:7300/metrics # Check logs -tail -f data/dispute-mon.log +tail -f logs/dispute-mon.log ``` -You should see metrics like: -- `op_dispute_mon_games_total` - Total number of games -- `op_dispute_mon_games_status` - Game status counts -- `op_dispute_mon_claims_bond_total` - Total bonds - - + -Prometheus will scrape metrics from all your OP Stack components. +Once op-dispute-mon is running, it exposes several important metrics that help you monitor dispute game health. -### Install Prometheus with Docker +### Core metrics -Create a monitoring directory: +Access the metrics endpoint: ```bash -# From your project root -mkdir -p monitoring/{prometheus,grafana} -cd monitoring +curl http://localhost:7300/metrics | grep op_dispute_mon ``` -### Create Prometheus configuration - -Create `prometheus/prometheus.yml`: +#### Game agreement metrics -```yaml -global: - scrape_interval: 15s - evaluation_interval: 15s - external_labels: - chain: 'my-op-rollup' - environment: 'testnet' - -scrape_configs: - # op-node metrics - - job_name: 'op-node' - static_configs: - - targets: ['host.docker.internal:7300'] - labels: - component: 'op-node' - role: 'consensus' - - # op-geth metrics - - job_name: 'op-geth' - metrics_path: '/debug/metrics/prometheus' - static_configs: - - targets: ['host.docker.internal:6060'] - labels: - component: 'op-geth' - role: 'execution' - - # op-batcher metrics - - job_name: 'op-batcher' - static_configs: - - targets: ['host.docker.internal:7301'] - labels: - component: 'op-batcher' - role: 'batcher' - - # op-proposer metrics - - job_name: 'op-proposer' - static_configs: - - targets: ['host.docker.internal:7302'] - labels: - component: 'op-proposer' - role: 'proposer' - - # op-challenger metrics - - job_name: 'op-challenger' - static_configs: - - targets: ['host.docker.internal:7303'] - labels: - component: 'op-challenger' - role: 'challenger' - - # op-dispute-mon metrics - - job_name: 'op-dispute-mon' - static_configs: - - targets: ['host.docker.internal:7304'] - labels: - component: 'op-dispute-mon' - role: 'monitor' ``` +# Total number of games tracked +op_dispute_mon_games_agreement 25 - - **Port mapping note**: If you're running components on different ports, update the targets accordingly. The example above assumes standard ports with offsets for each component. - - -### Create Docker Compose configuration - -Create `docker-compose.yml` in the `monitoring` directory: +# Games currently in progress +op_dispute_mon_games_agreement{completion="in_progress"} 3 -```yaml -version: '3.8' - -services: - prometheus: - image: prom/prometheus:latest - container_name: prometheus - ports: - - "9090:9090" - volumes: - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - - prometheus-data:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/usr/share/prometheus/console_libraries' - - '--web.console.templates=/usr/share/prometheus/consoles' - - '--web.enable-lifecycle' - restart: unless-stopped - extra_hosts: - - "host.docker.internal:host-gateway" +# Completed games +op_dispute_mon_games_agreement{completion="complete"} 22 - grafana: - image: grafana/grafana:latest - container_name: grafana - ports: - - "3000:3000" - volumes: - - grafana-data:/var/lib/grafana - - ./grafana/provisioning:/etc/grafana/provisioning - - ./grafana/dashboards:/var/lib/grafana/dashboards - environment: - - GF_SECURITY_ADMIN_PASSWORD=admin - - GF_USERS_ALLOW_SIGN_UP=false - - GF_SERVER_ROOT_URL=http://localhost:3000 - restart: unless-stopped - depends_on: - - prometheus +# Games where monitor agrees with honest actors +op_dispute_mon_games_agreement{agreement="agree"} 24 -volumes: - prometheus-data: - grafana-data: +# Games where monitor disagrees +op_dispute_mon_games_agreement{agreement="disagree"} 1 ``` -### Start Prometheus and Grafana +**What to monitor:** +- If `agreement="disagree"` is non-zero, investigate immediately - this indicates a potential issue with your honest actors +- High number of `completion="in_progress"` games is normal during active periods +- All completed games should show `agreement="agree"` -```bash -# From the monitoring directory -docker-compose up -d +#### Ignored games -# Check logs -docker-compose logs -f ``` - -### Verify Prometheus is scraping metrics - -1. Open Prometheus in your browser: `http://localhost:9090` -2. Go to **Status > Targets** -3. Verify all targets show as "UP" - -If any targets show as "DOWN": -- Check that the component is running -- Verify the port numbers match your configuration -- Check firewall rules if running on different machines - - - - - -### Access Grafana - -1. Open Grafana: `http://localhost:3000` -2. Login with default credentials: - - Username: `admin` - - Password: `admin` -3. Change the password when prompted - -### Add Prometheus as data source - -1. Click **⚙️ Configuration** > **Data Sources** -2. Click **Add data source** -3. Select **Prometheus** -4. Configure: - - **Name**: `Prometheus` - - **URL**: `http://prometheus:9090` - - **Access**: `Server (default)` -5. Click **Save & Test** - -### Import OP Stack dashboards - -#### Import Dispute Monitor Dashboard - -1. Download the [Dispute Monitor JSON](/resources/grafana/dispute-monitor-1718214549035.json) -2. In Grafana, click **+ > Import** -3. Click **Upload JSON file** and select the downloaded file -4. Select **Prometheus** as the data source -5. Click **Import** - -#### Create custom OP Node dashboard - -Create a new dashboard for monitoring op-node: - -1. Click **+ > Create Dashboard** -2. Click **Add visualization** -3. Select **Prometheus** data source -4. Add panels for key metrics: - -**Panel 1: Sync Status** -- Metric: `op_node_default_refs_number` -- Visualization: Stat -- Title: "Sync Block Number" - -**Panel 2: Peer Count** -- Metric: `op_node_p2p_peers` -- Visualization: Time series -- Title: "P2P Peer Count" - -**Panel 3: L1 Head vs Unsafe L2 Head** -- Metrics: - - `op_node_default_refs_l1_head` - - `op_node_default_refs_unsafe_l2_head` -- Visualization: Time series -- Title: "L1 vs L2 Head" - -**Panel 4: Derivation Pipeline** -- Metric: `op_node_derivation_idle` -- Visualization: Time series -- Title: "Derivation Idle Time" - -5. Click **Save dashboard** -6. Name it "OP Node Metrics" - -#### Create OP Geth dashboard - -1. Create a new dashboard -2. Add visualizations for: - -**Panel 1: Gas Price** -- Metric: `geth_tx_pool_basefee` -- Title: "Base Fee" - -**Panel 2: Transaction Pool** -- Metric: `geth_tx_pool_pending` -- Title: "Pending Transactions" - -**Panel 3: Block Processing** -- Metric: `geth_chain_head_block` -- Title: "Chain Head Block" - -**Panel 4: State Database Size** -- Metric: `geth_db_chaindata_disk_size` -- Title: "Database Size" - -3. Save as "OP Geth Metrics" - -#### Create Batcher dashboard - -1. Create a new dashboard -2. Add key metrics: - -**Panel 1: Published Batches** -- Metric: `op_batcher_batch_submitted_count` -- Title: "Batches Submitted" - -**Panel 2: Batch Size** -- Metric: `op_batcher_batch_size_bytes` -- Title: "Average Batch Size" - -**Panel 3: Gas Used** -- Metric: `op_batcher_total_gas_used` -- Title: "Total Gas Used" - -3. Save as "OP Batcher Metrics" - - - - - -For additional security monitoring, you can set up [Monitorism](/operators/chain-operators/tools/chain-monitoring#monitorism). - -### Clone Monitorism repository - -```bash -cd ~/ -git clone https://github.com/ethereum-optimism/monitorism.git -cd monitorism +# Games that don't meet monitoring criteria +op_dispute_mon_ignored_games 5 ``` -### Configure Monitorism - -Create a configuration file `config.toml`: - -```toml -[server] -host = "0.0.0.0" -port = 7305 - -[l1] -rpc_url = "YOUR_L1_RPC_URL" +Games may be ignored if: +- They don't involve your honest actors +- They use an unsupported game type +- They're outside the monitoring window -[l2] -rpc_url = "http://localhost:8545" +#### Game status by type -[[monitors]] -type = "fault_proof_withdrawal" -enabled = true -[monitors.config] -optimism_portal_address = "YOUR_OPTIMISM_PORTAL_ADDRESS" -l2_output_oracle_address = "YOUR_L2_OUTPUT_ORACLE_ADDRESS" ``` - -### Get contract addresses - -```bash -# From your deployment state -cd your-project/rollup/.deployer -jq -r '{optimismPortal: .optimismPortalProxyAddress, l2OutputOracle: .l2OutputOracleProxyAddress}' state.json -``` - -### Build and run Monitorism - -```bash -cd ~/monitorism -go build -o bin/monitorism ./cmd/monitorism - -# Run monitorism -./bin/monitorism --config config.toml +# Games by status +op_dispute_mon_games_status{status="challenger_wins"} 20 +op_dispute_mon_games_status{status="defender_wins"} 3 +op_dispute_mon_games_status{status="in_progress"} 2 ``` -### Add Monitorism to Prometheus +**What to monitor:** +- `defender_wins` should be rare (indicates your proposal was successfully defended) +- `challenger_wins` is normal (other proposers' incorrect proposals were challenged) +- `in_progress` shows active games -Add to `prometheus/prometheus.yml`: +#### Honest actor metrics -```yaml - - job_name: 'monitorism' - static_configs: - - targets: ['host.docker.internal:7305'] - labels: - component: 'monitorism' - role: 'monitor' ``` - -Reload Prometheus configuration: - -```bash -curl -X POST http://localhost:9090/-/reload +# Games involving your honest actors +op_dispute_mon_honest_actor_games{actor="0xYourProposer",role="proposer"} 15 +op_dispute_mon_honest_actor_games{actor="0xYourChallenger",role="challenger"} 10 ``` - +**What to monitor:** +- Your proposer should create games regularly +- Your challenger should participate when needed +- If either metric stops increasing, investigate your honest actors - +### Metric query examples -Create alert rules in Prometheus for critical events. +Here are some useful queries you can run: -### Create alert rules file +```bash +# Count total games +curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement{" | grep -v "#" | awk '{sum+=$2} END {print sum}' -Create `prometheus/alerts.yml`: +# Count in-progress games +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{completion="in_progress"}' | awk '{print $2}' -```yaml -groups: - - name: op_stack_alerts - interval: 30s - rules: - # Sequencer health - - alert: SequencerDown - expr: up{job="op-node"} == 0 - for: 2m - labels: - severity: critical - annotations: - summary: "Sequencer (op-node) is down" - description: "The sequencer has been down for more than 2 minutes" - - - alert: ExecutionClientDown - expr: up{job="op-geth"} == 0 - for: 2m - labels: - severity: critical - annotations: - summary: "Execution client (op-geth) is down" - description: "The execution client has been down for more than 2 minutes" - - # Batcher alerts - - alert: BatcherDown - expr: up{job="op-batcher"} == 0 - for: 5m - labels: - severity: critical - annotations: - summary: "Batcher is down" - description: "The batcher has been down for more than 5 minutes" - - - alert: BatcherNotSubmitting - expr: rate(op_batcher_batch_submitted_count[10m]) == 0 - for: 15m - labels: - severity: warning - annotations: - summary: "Batcher not submitting batches" - description: "No batches have been submitted in the last 15 minutes" - - # Proposer alerts - - alert: ProposerDown - expr: up{job="op-proposer"} == 0 - for: 5m - labels: - severity: critical - annotations: - summary: "Proposer is down" - description: "The proposer has been down for more than 5 minutes" - - # Challenger alerts - - alert: ChallengerDown - expr: up{job="op-challenger"} == 0 - for: 5m - labels: - severity: critical - annotations: - summary: "Challenger is down" - description: "The challenger has been down for more than 5 minutes" - - # Dispute game alerts - - alert: InvalidDisputeGame - expr: op_dispute_mon_games_status{status="invalid"} > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "Invalid dispute game detected" - description: "An invalid dispute game has been detected and needs immediate attention" - - # Sync lag alerts - - alert: SequencerSyncLag - expr: (op_node_default_refs_l1_head - op_node_default_refs_unsafe_l2_head) > 100 - for: 10m - labels: - severity: warning - annotations: - summary: "Sequencer sync lag detected" - description: "L2 is lagging behind L1 by more than 100 blocks" - - # Resource alerts - - alert: HighMemoryUsage - expr: process_resident_memory_bytes / 1e9 > 8 - for: 5m - labels: - severity: warning - annotations: - summary: "High memory usage detected" - description: "Component {{ $labels.job }} is using more than 8GB of memory" +# Count disagreements (should be 0!) +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agreement="disagree"}' | awk '{print $2}' ``` -### Update Prometheus configuration + -Update `prometheus/prometheus.yml` to include the alert rules: + -```yaml -global: - scrape_interval: 15s - evaluation_interval: 15s +Use these practices to ensure your dispute games are healthy: -# Add this section -rule_files: - - 'alerts.yml' +### Daily monitoring checklist -# ... rest of configuration ... -``` +1. **Check for disagreements:** +```bash + curl -s http://localhost:7300/metrics | grep 'agreement="disagree"' + ``` + Result should be `0` or metric should not exist. -### Reload Prometheus +2. **Verify honest actors are active:** + ```bash + curl -s http://localhost:7300/metrics | grep "honest_actor_games" + ``` + Both proposer and challenger metrics should be present and increasing. +3. **Check game completion rate:** ```bash -docker-compose restart prometheus + # Compare in-progress vs complete + curl -s http://localhost:7300/metrics | grep "completion=" ``` + Most games should complete within the challenge period (7 days). -### Verify alerts in Prometheus - -1. Open Prometheus: `http://localhost:9090` -2. Go to **Alerts** -3. Verify your alert rules are loaded +4. **Review logs for errors:** +```bash + # Docker + docker-compose logs --tail=100 op-dispute-mon | grep -i error + + # Source + tail -100 logs/dispute-mon.log | grep -i error + ``` - - -For alert notifications, you can configure Alertmanager. - -### Add Alertmanager to Docker Compose -Update `docker-compose.yml`: +## Production considerations -```yaml - alertmanager: - image: prom/alertmanager:latest - container_name: alertmanager - ports: - - "9093:9093" - volumes: - - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml - - alertmanager-data:/alertmanager - command: - - '--config.file=/etc/alertmanager/alertmanager.yml' - - '--storage.path=/alertmanager' - restart: unless-stopped +When running op-dispute-mon in production: -volumes: - prometheus-data: - grafana-data: - alertmanager-data: # Add this -``` +### High availability -### Create Alertmanager configuration +1. **Run multiple instances** for redundancy (metrics will be identical) +2. **Use load-balanced L1 RPC endpoints** for reliability +3. **Set up health checks:** + ```bash + # Check metrics endpoint health + curl -f http://localhost:7300/metrics || alert + ``` -Create `alertmanager/alertmanager.yml`: +### Resource requirements -```yaml -global: - resolve_timeout: 5m - -route: - group_by: ['alertname', 'cluster'] - group_wait: 10s - group_interval: 10s - repeat_interval: 12h - receiver: 'default' - -receivers: - - name: 'default' - # Configure your preferred notification channel: - - # Slack example: - # slack_configs: - # - api_url: 'YOUR_SLACK_WEBHOOK_URL' - # channel: '#alerts' - # title: 'OP Stack Alert' - # text: '{{ range .Alerts }}{{ .Annotations.summary }}\n{{ .Annotations.description }}{{ end }}' - - # Discord example: - # discord_configs: - # - webhook_url: 'YOUR_DISCORD_WEBHOOK_URL' - - # Email example: - # email_configs: - # - to: 'your-email@example.com' - # from: 'alertmanager@example.com' - # smarthost: 'smtp.gmail.com:587' - # auth_username: 'your-email@example.com' - # auth_password: 'your-app-password' - -inhibit_rules: - - source_match: - severity: 'critical' - target_match: - severity: 'warning' - equal: ['alertname', 'cluster'] -``` +- **CPU:** Minimal (< 5% of 1 core) +- **Memory:** ~100-200 MB +- **Network:** Depends on L1 RPC usage +- **Disk:** Minimal (logs only) -### Update Prometheus to use Alertmanager +### Backup RPC endpoints -Update `prometheus/prometheus.yml`: +Configure multiple RPC endpoints for failover: ```yaml -# Add this section -alerting: - alertmanagers: - - static_configs: - - targets: ['alertmanager:9093'] - -# ... rest of configuration ... -``` - -### Restart monitoring stack - -```bash -docker-compose up -d +# In docker-compose.yml +environment: + # Comma-separated list of RPCs + - OP_DISPUTE_MON_L1_ETH_RPC=https://eth-sepolia-1.example.com,https://eth-sepolia-2.example.com + - OP_DISPUTE_MON_ROLLUP_RPC=http://op-node-1:8547,http://op-node-2:8547 ``` - - - - -Here are the key metrics you should monitor for each component: - -### OP Node (Sequencer) Metrics - - - -| Metric | Description | Alert Threshold | -|--------|-------------|----------------| -| `op_node_default_refs_number` | Current sync block number | Should increase steadily | -| `op_node_default_refs_l1_head` | L1 head block | Track sync with L1 | -| `op_node_derivation_idle` | Derivation pipeline idle time | > 30s indicates issues | -| `op_node_p2p_peers` | Number of P2P peers | Should be > 0 if P2P enabled | -| `process_cpu_seconds_total` | CPU usage | Monitor for spikes | - -[See complete list in Node Metrics guide](/operators/node-operators/management/metrics#important-metrics) - - - -### OP Geth Metrics - - - -| Metric | Description | Alert Threshold | -|--------|-------------|----------------| -| `geth_chain_head_block` | Current head block | Should increase | -| `geth_tx_pool_pending` | Pending transactions | Monitor for buildup | -| `geth_db_chaindata_disk_size` | Database size | Monitor growth rate | -| `geth_tx_pool_basefee` | Current base fee | Monitor for spikes | - - - -### OP Batcher Metrics - - - -| Metric | Description | Alert Threshold | -|--------|-------------|----------------| -| `op_batcher_batch_submitted_count` | Number of batches submitted | Should increase regularly | -| `op_batcher_batch_size_bytes` | Average batch size | Monitor size trends | -| `op_batcher_total_gas_used` | Gas used for submissions | Track costs | -| `op_batcher_channel_open_duration` | Time channels are open | Alert if too high | - - - -### OP Proposer Metrics - - - -| Metric | Description | Alert Threshold | -|--------|-------------|----------------| -| `op_proposer_proposals_total` | Total proposals submitted | Should increase | -| `op_proposer_gas_used` | Gas used for proposals | Track costs | -| `op_proposer_latest_proposal_block` | Latest proposed block | Should advance | - - - -### OP Challenger Metrics - - - -| Metric | Description | Alert Threshold | -|--------|-------------|----------------| -| `op_challenger_games_total` | Total games tracked | Monitor activity | -| `op_challenger_moves_made` | Moves made in games | Track participation | -| `op_challenger_claims_invalid` | Invalid claims detected | Alert immediately if > 0 | - - - -### OP Dispute Mon Metrics - - - -| Metric | Description | Alert Threshold | -|--------|-------------|----------------| -| `op_dispute_mon_games_total` | Total games monitored | Track overall activity | -| `op_dispute_mon_games_status` | Games by status | Alert on invalid games | -| `op_dispute_mon_claims_bond_total` | Total bonds | Monitor economic activity | - - - - - - -## Monitoring checklist - -Use this checklist to ensure comprehensive monitoring: - -- ✅ All OP Stack components have metrics enabled -- ✅ Prometheus is scraping all targets successfully -- ✅ Grafana dashboards are displaying data -- ✅ Alert rules are configured and active -- ✅ op-dispute-mon is running and tracking games -- ✅ Disk space is monitored (Prometheus and component data) -- ✅ Backup monitoring data regularly -- ✅ Test alert notifications are working -- ✅ Document your monitoring setup for your team - -## Troubleshooting - -### Prometheus not scraping metrics - -**Issue**: Targets show as "DOWN" in Prometheus - -**Solutions**: -1. Verify component is running: `ps aux | grep op-node` -2. Test metrics endpoint directly: `curl http://localhost:7300/metrics` -3. Check firewall rules -4. Verify Docker network can reach `host.docker.internal` -5. Check Prometheus logs: `docker-compose logs prometheus` - -### High memory usage - -**Issue**: Components using excessive memory - -**Solutions**: -1. Check database sizes: `du -sh rollup/*/data` -2. Consider pruning old data -3. Increase system resources -4. Check for memory leaks in logs - -### Grafana not displaying data - -**Issue**: Dashboards show "No data" - -**Solutions**: -1. Verify Prometheus data source is configured correctly -2. Check that Prometheus is scraping metrics -3. Verify metric names in queries match actual metrics -4. Check time range in dashboard - -### Dispute mon not tracking games - -**Issue**: `op-dispute-mon` shows no games - -**Solutions**: -1. Verify Game Factory address is correct -2. Check that challenger has created some games -3. Verify L1 and L2 RPC endpoints are working -4. Check `op-dispute-mon` logs for errors - -## Production considerations - -When running monitoring in production: - -1. **Security**: - - Restrict access to Grafana with proper authentication - - Use TLS for all connections - - Don't expose metrics ports publicly - - Regularly update monitoring tools - -2. **Reliability**: - - Run Prometheus and Grafana on separate infrastructure - - Set up Prometheus in HA mode for production - - Configure retention policies based on your needs - - Set up offsite backups of monitoring data - -3. **Scalability**: - - Consider using Prometheus federation for multiple chains - - Use remote storage for long-term metrics - - Implement proper resource limits in Docker - -4. **Alerting**: - - Configure multiple notification channels - - Set up escalation policies - - Test alerts regularly - - Document runbooks for each alert ## Next steps -* **Learn more about monitoring options**: [Chain Monitoring Reference](/operators/chain-operators/tools/chain-monitoring) -* **Understand dispute games**: [Dispute Games Guide](/operators/chain-operators/tutorials/dispute-games) -* **Set up additional monitors**: [Monitorism Repository](https://github.com/ethereum-optimism/monitorism) -* **Optimize your setup**: [Operations Guide](/operators/chain-operators/management/operations) -* **Get support**: [Developer Discussions](https://github.com/ethereum-optimism/developers/discussions) - -## Additional resources - -* [Prometheus Documentation](https://prometheus.io/docs/) -* [Grafana Documentation](https://grafana.com/docs/) -* [Node Metrics Guide](/operators/node-operators/management/metrics) -* [Monitorism GitHub](https://github.com/ethereum-optimism/monitorism) -* [OP Dispute Mon](https://github.com/ethereum-optimism/optimism/tree/develop/op-dispute-mon) - +* [Dispute Monitor Tool Reference](/operators/chain-operators/tools/chain-monitoring) +* [Fault Proof System Overview](/stack/fault-proofs/explainer) +* [OP Challenger Setup](/operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup) +* [Optimism Monorepo](https://github.com/ethereum-optimism/optimism) +* [Prometheus Documentation](https://prometheus.io/docs/) (for optional metrics visualization) +* [Grafana Documentation](https://grafana.com/docs/) (for optional dashboarding) From 1aa12e8b01b2d822034d80b35ea42ca51da7e6e8 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Mon, 20 Oct 2025 19:58:53 +0100 Subject: [PATCH 03/17] fix: correct JSON structure and improve documentation navigation - Fixed JSON formatting issues in docs.json to ensure proper structure. - Updated tutorial sections for better clarity and organization. - Enhanced navigation by ensuring all pages are correctly listed under their respective groups. - Streamlined the documentation layout for improved user experience. --- docs.json | 112 +++++++++--------- .../tutorials/monitoring-chain.mdx | 71 ++--------- 2 files changed, 63 insertions(+), 120 deletions(-) diff --git a/docs.json b/docs.json index 36041d676..a32e042ed 100644 --- a/docs.json +++ b/docs.json @@ -572,14 +572,14 @@ }, { "group": "Tools", - "pages": [ + "pages": [ "interop/tools/supersim", "interop/tools/devnet" - ] - }, - { + ] + }, + { "group": "Tutorials", - "pages": [ + "pages": [ "interop/tutorials/transfer-superchainERC20", "interop/tutorials/deploy-superchain-erc20", "interop/tutorials/bridge-crosschain-eth", @@ -588,16 +588,16 @@ "interop/tutorials/event-reads", "interop/tutorials/event-contests", "interop/tutorials/upgrade-to-superchain-erc20" + ] + } ] - } - ] - }, - { + }, + { "tab": "App Devs", "groups": [ - { + { "group": "App Developers", - "pages": [ + "pages": [ "interop/get-started", "interop/starter-kit", "interop/explainer", @@ -670,11 +670,11 @@ "app-developers/tutorials/interop/event-contests", "app-developers/tutorials/interop/event-reads", "app-developers/tutorials/interop/transfer-superchainERC20" - ] - } ] - }, - { + } + ] + }, + { "group": "Developer Tools", "pages": [ "app-developers/tools/supersim", @@ -702,16 +702,16 @@ "app-developers/tools/data-and-dashboards/data-glossary" ] } + ] + } ] - } - ] - }, - { + }, + { "tab": "Operators", "groups": [ - { + { "group": "Chain Operators", - "pages": [ + "pages": [ "operators/chain-operators/architecture", "operators/chain-operators/self-hosted", { @@ -745,7 +745,7 @@ }, { "group": "Deployment", - "pages": [ + "pages": [ "operators/chain-operators/deploy/genesis", "operators/chain-operators/deploy/overview", "operators/chain-operators/deploy/proposer-setup-guide", @@ -753,14 +753,14 @@ "operators/chain-operators/deploy/smart-contracts", "operators/chain-operators/deploy/spin-batcher", "operators/chain-operators/deploy/validate-deployment" - ] - }, + ] + }, + { + "group": "Tutorials", + "pages": [ { - "group": "Tutorials", - "pages": [ - { "group": "Create L2 rollup testnet", - "pages": [ + "pages": [ "operators/chain-operators/tutorials/create-l2-rollup", "operators/chain-operators/tutorials/create-l2-rollup/op-deployer-setup", "operators/chain-operators/tutorials/create-l2-rollup/op-geth-setup", @@ -791,25 +791,25 @@ "operators/chain-operators/tools/op-txproxy", "operators/chain-operators/tools/op-validator", "operators/chain-operators/tools/proxyd" - ] - } ] - }, - { + } + ] + }, + { "group": "Node Operators", "pages": [ "operators/node-operators/architecture", - { + { "group": "Tutorials", - "pages": [ + "pages": [ "operators/node-operators/tutorials/node-from-docker", "operators/node-operators/tutorials/node-from-source", "operators/node-operators/tutorials/run-node-from-source" - ] - }, - { + ] + }, + { "group": "Configuration", - "pages": [ + "pages": [ "operators/node-operators/configuration/base-config", "operators/node-operators/configuration/consensus-config", "operators/node-operators/configuration/execution-config" @@ -829,11 +829,11 @@ "operators/node-operators/network-upgrades", "operators/node-operators/json-rpc", "operators/node-operators/releases" + ] + } ] - } - ] - }, - { + }, + { "tab": "OP Stack", "groups": [ { @@ -849,7 +849,7 @@ "stack/opcm", { "group": "Smart Contracts", - "pages": [ + "pages": [ "stack/smart-contracts/smart-contracts", "stack/smart-contracts/superchain-ops-guide", "stack/smart-contracts/op-deployer-upgrade", @@ -858,15 +858,15 @@ }, { "group": "Rollup", - "pages": [ + "pages": [ "stack/rollup/overview", "stack/rollup/derivation-pipeline", "stack/rollup/outages" - ] - }, - { + ] + }, + { "group": "Fault Proofs", - "pages": [ + "pages": [ "stack/fault-proofs/explainer", "stack/fault-proofs/fp-components", "stack/fault-proofs/cannon", @@ -877,7 +877,7 @@ }, { "group": "Transactions", - "pages": [ + "pages": [ "stack/transactions/fees", "stack/transactions/transaction-flow", "stack/transactions/transaction-finality", @@ -889,13 +889,13 @@ }, { "group": "Features", - "pages": [ + "pages": [ "stack/features/send-raw-transaction-conditional" ] }, { "group": "Security", - "pages": [ + "pages": [ "stack/security/faq", "stack/security/pause", "stack/security/audits-report", @@ -926,13 +926,13 @@ }, { "tab": "Notices", - "pages": [ - "notices/fusaka-notice", + "pages": [ + "notices/fusaka-notice", "notices/upgrade-13", "notices/upgrade-14", "notices/upgrade-15", - "notices/upgrade-16", - "notices/upgrade-16a", + "notices/upgrade-16", + "notices/upgrade-16a", "notices/holocene-changes", "notices/pectra-changes", "notices/pectra-fees", @@ -969,7 +969,7 @@ "icon": "discord" } ] - } + } }, "logo": { "light": "/public/logos/logo-docs-light.svg", diff --git a/operators/chain-operators/tutorials/monitoring-chain.mdx b/operators/chain-operators/tutorials/monitoring-chain.mdx index eea6467ed..f13a8f25b 100644 --- a/operators/chain-operators/tutorials/monitoring-chain.mdx +++ b/operators/chain-operators/tutorials/monitoring-chain.mdx @@ -33,39 +33,16 @@ Before you begin, ensure you have: `op-dispute-mon` is a specialized monitoring tool that tracks dispute games in the fault proof system. It provides visibility into: -* **Game Status**: Track games from creation to resolution -* **Honest Actor Participation**: Monitor your proposer and challenger activity -* **Agreement Tracking**: See which games agree/disagree with your honest actors -* **Game Completion**: Monitor in-progress vs completed games -* **Ignored Games**: Track games that don't meet monitoring criteria +* **Game status**: Track games from creation to resolution +* **Honest actor Participation**: Monitor your proposer and challenger activity +* **Agreement tracking**: See which games agree/disagree with your honest actors +* **Game completion**: Monitor in-progress vs completed games +* **Ignored games**: Track games that don't meet monitoring criteria The monitor exposes Prometheus metrics that can be used to build dashboards and alerts. ## Monitoring architecture -```mermaid -graph TB - subgraph "L1 Network" - A[DisputeGameFactory
Contract] - end - - subgraph "Your Rollup" - B[op-proposer
Creates proposals] - C[op-challenger
Defends proposals] - D[op-node
Rollup RPC] - end - - subgraph "Monitoring" - E[op-dispute-mon
:7300] - end - - A -->|Reads games| E - D -->|Rollup state| E - E -->|Tracks| B - E -->|Tracks| C - E -->|Exposes metrics| F[/metrics endpoint] -``` - @@ -536,43 +513,10 @@ Use these practices to ensure your dispute games are healthy: # Source tail -100 logs/dispute-mon.log | grep -i error - ``` +``` - - -## Production considerations - -When running op-dispute-mon in production: - -### High availability - -1. **Run multiple instances** for redundancy (metrics will be identical) -2. **Use load-balanced L1 RPC endpoints** for reliability -3. **Set up health checks:** - ```bash - # Check metrics endpoint health - curl -f http://localhost:7300/metrics || alert - ``` - -### Resource requirements - -- **CPU:** Minimal (< 5% of 1 core) -- **Memory:** ~100-200 MB -- **Network:** Depends on L1 RPC usage -- **Disk:** Minimal (logs only) - -### Backup RPC endpoints - -Configure multiple RPC endpoints for failover: - -```yaml -# In docker-compose.yml -environment: - # Comma-separated list of RPCs - - OP_DISPUTE_MON_L1_ETH_RPC=https://eth-sepolia-1.example.com,https://eth-sepolia-2.example.com - - OP_DISPUTE_MON_ROLLUP_RPC=http://op-node-1:8547,http://op-node-2:8547 -``` + ## Next steps @@ -580,6 +524,5 @@ environment: * [Dispute Monitor Tool Reference](/operators/chain-operators/tools/chain-monitoring) * [Fault Proof System Overview](/stack/fault-proofs/explainer) * [OP Challenger Setup](/operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup) -* [Optimism Monorepo](https://github.com/ethereum-optimism/optimism) * [Prometheus Documentation](https://prometheus.io/docs/) (for optional metrics visualization) * [Grafana Documentation](https://grafana.com/docs/) (for optional dashboarding) From 79ad879eb67e89308adf0b48dea65775b8caae11 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Mon, 20 Oct 2025 22:51:06 +0100 Subject: [PATCH 04/17] feat: add dispute monitor setup and update documentation - Introduced a new service for the dispute monitor in the Docker Compose configuration. - Added setup instructions for the dispute monitor in the setup-rollup.sh script, including environment variable configuration. - Updated docs.json to include the monitoring chain tutorial for improved navigation. - Removed the obsolete optimism subproject to streamline the directory structure. --- create-l2-rollup-example/docker-compose.yml | 28 + create-l2-rollup-example/optimism | 1 - .../scripts/setup-rollup.sh | 41 +- docs.json | 4 +- .../create-l2-rollup/monitoring-chain.mdx | 496 ++++++++++++++++++ 5 files changed, 566 insertions(+), 4 deletions(-) delete mode 160000 create-l2-rollup-example/optimism create mode 100644 operators/chain-operators/tutorials/create-l2-rollup/monitoring-chain.mdx diff --git a/create-l2-rollup-example/docker-compose.yml b/create-l2-rollup-example/docker-compose.yml index edb43c96a..fb843c1b3 100644 --- a/create-l2-rollup-example/docker-compose.yml +++ b/create-l2-rollup-example/docker-compose.yml @@ -149,5 +149,33 @@ services: op-node: condition: service_healthy + # Dispute Monitor + dispute-mon: + image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.4.2-rc.1 + container_name: dispute-mon + volumes: + - ./dispute-mon:/workspace + working_dir: /workspace + env_file: + - ./dispute-mon/.env + - .env + ports: + - "7300:7300" + environment: + - OP_DISPUTE_MON_L1_ETH_RPC=${L1_RPC_URL} + - OP_DISPUTE_MON_ROLLUP_RPC=${ROLLUP_RPC} + - OP_DISPUTE_MON_HONEST_ACTORS=${PROPOSER_ADDRESS},${CHALLENGER_ADDRESS} + - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} + - OP_DISPUTE_MON_MONITOR_INTERVAL=10s + - OP_DISPUTE_MON_LOG_FORMAT=logfmt + - OP_DISPUTE_MON_LOG_LEVEL=info + - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 + - OP_DISPUTE_MON_METRICS_ENABLED=true + - OP_DISPUTE_MON_METRICS_PORT=7300 + depends_on: + op-node: + condition: service_healthy + restart: unless-stopped + volumes: op_geth_data: diff --git a/create-l2-rollup-example/optimism b/create-l2-rollup-example/optimism deleted file mode 160000 index a094d0160..000000000 --- a/create-l2-rollup-example/optimism +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a094d016092e3355642d00be6d7943c4529ef008 diff --git a/create-l2-rollup-example/scripts/setup-rollup.sh b/create-l2-rollup-example/scripts/setup-rollup.sh index 510462ec2..d5e1f580f 100755 --- a/create-l2-rollup-example/scripts/setup-rollup.sh +++ b/create-l2-rollup-example/scripts/setup-rollup.sh @@ -30,6 +30,7 @@ SEQUENCER_DIR="$ROLLUP_DIR/sequencer" BATCHER_DIR="$ROLLUP_DIR/batcher" PROPOSER_DIR="$ROLLUP_DIR/proposer" CHALLENGER_DIR="$ROLLUP_DIR/challenger" +DISPUTE_MON_DIR="$ROLLUP_DIR/dispute-mon" # Logging functions log_info() { @@ -427,6 +428,42 @@ generate_challenger_prestate() { log_success "Challenger prestate generation complete: $ROLLUP_DIR/challenger/${PRESTATE_HASH}.bin.gz" } +# Setup dispute monitor +setup_dispute_monitor() { + log_info "Setting up dispute monitor..." + + mkdir -p "$DISPUTE_MON_DIR" + cd "$DISPUTE_MON_DIR" + + # Get required addresses + GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' "$DEPLOYER_DIR/.deployer/state.json") + PROPOSER_ADDRESS=$(cast wallet address --private-key "$PROPOSER_PRIVATE_KEY") + CHALLENGER_ADDRESS=$(cast wallet address --private-key "$CHALLENGER_PRIVATE_KEY") + + log_info "Game Factory: $GAME_FACTORY_ADDRESS" + log_info "Proposer: $PROPOSER_ADDRESS" + log_info "Challenger: $CHALLENGER_ADDRESS" + + # Create environment file for dispute monitor + cat > .env << EOF +# Contract Addresses +OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS + +# Honest Actors +OP_DISPUTE_MON_HONEST_ACTORS=$PROPOSER_ADDRESS,$CHALLENGER_ADDRESS + +# Monitoring Configuration +OP_DISPUTE_MON_MONITOR_INTERVAL=10s +EOF + + # Create logs directory + mkdir -p logs + + log_success "Dispute monitor configuration created" + log_info "Dispute monitor will start with 'docker-compose up -d' from project root" + log_info "Metrics will be available at: http://localhost:7300/metrics" +} + # Add op-deployer to PATH if it exists in the workspace if [ -f "$(dirname "$0")/../op-deployer" ]; then OP_DEPLOYER_PATH="$(cd "$(dirname "$0")/.." && pwd)/op-deployer" @@ -457,9 +494,11 @@ main() { setup_proposer generate_challenger_prestate setup_challenger + setup_dispute_monitor log_success "OP Stack L2 Rollup deployment complete!" - log_info "Run 'docker-compose up -d' to start all services" + log_info "Run 'docker-compose up -d' to start all services (including dispute monitor)" + log_info "Dispute monitor metrics: http://localhost:7300/metrics" } # Handle command line arguments for standalone function calls diff --git a/docs.json b/docs.json index a32e042ed..430d1882f 100644 --- a/docs.json +++ b/docs.json @@ -766,10 +766,10 @@ "operators/chain-operators/tutorials/create-l2-rollup/op-geth-setup", "operators/chain-operators/tutorials/create-l2-rollup/op-batcher-setup", "operators/chain-operators/tutorials/create-l2-rollup/op-proposer-setup", - "operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup" + "operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup", + "operators/chain-operators/tutorials/create-l2-rollup/monitoring-chain" ] }, - "operators/chain-operators/tutorials/monitoring-chain", "operators/chain-operators/tutorials/absolute-prestate", "operators/chain-operators/tutorials/adding-derivation-attributes", "operators/chain-operators/tutorials/adding-precompiles", diff --git a/operators/chain-operators/tutorials/create-l2-rollup/monitoring-chain.mdx b/operators/chain-operators/tutorials/create-l2-rollup/monitoring-chain.mdx new file mode 100644 index 000000000..1b379c0d7 --- /dev/null +++ b/operators/chain-operators/tutorials/create-l2-rollup/monitoring-chain.mdx @@ -0,0 +1,496 @@ +--- +title: Monitor your chain with Dispute Monitor +description: Learn how to set up op-dispute-mon to monitor dispute games and fault proof activity on your L2 rollup. +--- + +Now that you have a running OP Stack rollup with fault proofs enabled, it's critical to monitor your dispute games to ensure your chain operates securely. This tutorial will guide you through setting up `op-dispute-mon` to track all dispute game activity on your rollup. + + + This tutorial continues from the previous steps in [Creating Your Own L2 Rollup](/operators/chain-operators/tutorials/create-l2-rollup). Make sure you have completed the op-challenger setup and have dispute games running before proceeding. + + +## What you'll set up + +By the end of this tutorial, you'll have: + +* **op-dispute-mon** monitoring all dispute games on your chain +* **Real-time tracking** of game status, agreements, and honest actor participation +* Understanding of key metrics for dispute game health + +## Prerequisites + +Before you begin, ensure you have: + +* Completed all previous steps in the [Creating Your Own L2 Rollup tutorial](/operators/chain-operators/tutorials/create-l2-rollup) +* op-challenger running and participating in dispute games +* op-proposer running and creating proposals +* L1 RPC endpoint with sufficient rate limits +* Rollup RPC endpoint (op-node) +* Docker installed (for Docker setup) OR Go 1.21+ (for build from source) + +## What is op-dispute-mon? + +`op-dispute-mon` is a specialized monitoring tool that tracks dispute games in the fault proof system. It provides visibility into: + +* **Game status**: Track games from creation to resolution +* **Honest actor participation**: Monitor your proposer and challenger activity +* **Agreement tracking**: See which games agree/disagree with your honest actors +* **Game completion**: Monitor in-progress vs completed games +* **Ignored games**: Track games that don't meet monitoring criteria + +The monitor exposes Prometheus metrics that can be used to build dashboards and alerts. + + + + +Choose your preferred deployment method: Docker (recommended) or build from source. + + + + +### Docker Setup + +The Docker setup provides a containerized environment for running op-dispute-mon with minimal configuration. + +#### Create dispute-mon directory + +```bash +# From your project root +cd rollup +mkdir dispute-mon +cd dispute-mon +``` + +#### Get required addresses + +You need three key addresses for op-dispute-mon: + +1. **DisputeGameFactory address** (from your deployment) +2. **Proposer address** (your honest actor who creates proposals) +3. **Challenger address** (your honest actor who defends proposals) + +```bash +# Get DisputeGameFactory address from deployment +export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../deployer/.deployer/state.json) +echo "Game Factory Address: $GAME_FACTORY_ADDRESS" + +# Get Proposer address (from your proposer setup) +source ../proposer/.env +export PROPOSER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Proposer Address: $PROPOSER_ADDRESS" + +# Get Challenger address (from your challenger setup) +source ../challenger/.env +export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Challenger Address: $CHALLENGER_ADDRESS" +``` + +#### Create Docker Compose file + +Create `docker-compose.yml` with the following configuration: + +```yaml +version: "3.8" + +services: + op-dispute-mon: + image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.4.2-rc.1 + container_name: dispute-mon + ports: + - "7300:7300" # Metrics port + environment: + # L1 RPC Configuration + - OP_DISPUTE_MON_L1_ETH_RPC=https://sepolia.infura.io/v3/YOUR_INFURA_KEY + + # Rollup RPC Configuration (your op-node) + - OP_DISPUTE_MON_ROLLUP_RPC=http://op-node:8547 + + # Honest Actors (your proposer and challenger addresses) + - OP_DISPUTE_MON_HONEST_ACTORS=${PROPOSER_ADDRESS},${CHALLENGER_ADDRESS} + + # DisputeGameFactory Contract Address + - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} + + # Monitoring Configuration + - OP_DISPUTE_MON_MONITOR_INTERVAL=10s + + # Logging Configuration + - OP_DISPUTE_MON_LOG_FORMAT=logfmt + - OP_DISPUTE_MON_LOG_LEVEL=info + + # Metrics Configuration + - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 + - OP_DISPUTE_MON_METRICS_ENABLED=true + - OP_DISPUTE_MON_METRICS_PORT=7300 + restart: unless-stopped + networks: + - dispute-mon-network + +networks: + dispute-mon-network: + driver: bridge +``` + + + Replace `YOUR_INFURA_KEY` with your actual Infura API key or use another L1 RPC provider. Make sure your L1 RPC has sufficient rate limits for production use. + + +#### Start op-dispute-mon + +```bash +# Export variables +export PROPOSER_ADDRESS=0xYourProposerAddress +export CHALLENGER_ADDRESS=0xYourChallengerAddress +export GAME_FACTORY_ADDRESS=0xYourGameFactoryAddress + +# Start the container +docker-compose up -d + +# View logs +docker-compose logs -f op-dispute-mon +``` + +#### Verify it's running + +Check that the metrics endpoint is accessible: + +```bash +# Query metrics endpoint +curl http://localhost:7300/metrics + +# You should see Prometheus metrics including: +# - op_dispute_mon_games_agreement +# - op_dispute_mon_ignored_games +# - and many more... +``` + +Check logs for any errors: + +```bash +# View recent logs +docker-compose logs --tail=50 op-dispute-mon + +# Follow logs in real-time +docker-compose logs -f op-dispute-mon +``` + +You should see logs indicating the monitor is tracking games: + +``` +level=info msg="Monitoring dispute games" factory=0x... honest_actors=[0x..., 0x...] +level=info msg="Found games" count=5 +level=info msg="Updated game status" game=0x... status=IN_PROGRESS +``` + + + + + +### Build from Source + +Building from source gives you more control and is useful for development or custom deployments. + +#### Clone and build op-dispute-mon + +```bash +# From your project root +cd optimism + +# Pull latest changes +git pull + +# Build op-dispute-mon +cd op-dispute-mon +make op-dispute-mon + +# Verify the build +./bin/op-dispute-mon --version +``` + +#### Create directory structure + +```bash +# From your project root +cd rollup +mkdir -p dispute-mon/scripts +cd dispute-mon +``` + +#### Get required addresses + +```bash +# Get DisputeGameFactory address +export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../deployer/.deployer/state.json) +echo "Game Factory Address: $GAME_FACTORY_ADDRESS" + +# Get Proposer address +source ../proposer/.env +export PROPOSER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Proposer Address: $PROPOSER_ADDRESS" + +# Get Challenger address +source ../challenger/.env +export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +echo "Challenger Address: $CHALLENGER_ADDRESS" + +# Get L1 RPC URL +source ../.env +echo "L1 RPC: $L1_RPC_URL" +``` + +#### Create environment file + +```bash +cat > .env << EOF +# L1 Configuration +L1_RPC_URL=$L1_RPC_URL + +# Rollup Configuration +ROLLUP_RPC=http://localhost:8547 + +# Contract Addresses +GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS + +# Honest Actors +PROPOSER_ADDRESS=$PROPOSER_ADDRESS +CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS + +# Monitoring Configuration +MONITOR_INTERVAL=10s +LOG_LEVEL=info + +# Metrics Configuration +METRICS_PORT=7300 +EOF +``` + +#### Create startup script + +Create `scripts/start-dispute-mon.sh`: + +```bash +#!/bin/bash +set -e + +# Load environment variables +source .env + +# Path to op-dispute-mon binary +DISPUTE_MON_BIN=../../optimism/op-dispute-mon/bin/op-dispute-mon + +echo "Starting op-dispute-mon..." +echo "Game Factory: $GAME_FACTORY_ADDRESS" +echo "Proposer: $PROPOSER_ADDRESS" +echo "Challenger: $CHALLENGER_ADDRESS" + +$DISPUTE_MON_BIN \ + --l1-eth-rpc=$L1_RPC_URL \ + --rollup-rpc=$ROLLUP_RPC \ + --game-factory-address=$GAME_FACTORY_ADDRESS \ + --honest-actors=$PROPOSER_ADDRESS,$CHALLENGER_ADDRESS \ + --monitor-interval=$MONITOR_INTERVAL \ + --log.level=$LOG_LEVEL \ + --log.format=logfmt \ + --metrics.enabled=true \ + --metrics.addr=0.0.0.0 \ + --metrics.port=$METRICS_PORT +``` + +Make it executable: + +```bash +chmod +x scripts/start-dispute-mon.sh +``` + +#### Start op-dispute-mon + +```bash +# Start in foreground (for testing) +./scripts/start-dispute-mon.sh + +# Or start in background +./scripts/start-dispute-mon.sh > logs/dispute-mon.log 2>&1 & + +# Save the process ID +echo $! > dispute-mon.pid +``` + +#### Verify it's running + +```bash +# Check if process is running +ps aux | grep op-dispute-mon + +# Query metrics endpoint +curl http://localhost:7300/metrics + +# Check logs +tail -f logs/dispute-mon.log +``` + + + + + + + + +Once op-dispute-mon is running, it exposes several important metrics that help you monitor dispute game health. + +### Core metrics + +Access the metrics endpoint: + +```bash +curl http://localhost:7300/metrics | grep op_dispute_mon +``` + +#### Game agreement metrics + +``` +# Total number of games tracked +op_dispute_mon_games_agreement 25 + +# Games currently in progress +op_dispute_mon_games_agreement{completion="in_progress"} 3 + +# Completed games +op_dispute_mon_games_agreement{completion="complete"} 22 + +# Games where monitor agrees with honest actors +op_dispute_mon_games_agreement{agreement="agree"} 24 + +# Games where monitor disagrees +op_dispute_mon_games_agreement{agreement="disagree"} 1 +``` + +**What to monitor:** +- If `agreement="disagree"` is non-zero, investigate immediately - this indicates a potential issue with your honest actors +- High number of `completion="in_progress"` games is normal during active periods +- All completed games should show `agreement="agree"` + +#### Ignored games + +``` +# Games that don't meet monitoring criteria +op_dispute_mon_ignored_games 5 +``` + +Games may be ignored if: +- They don't involve your honest actors +- They use an unsupported game type +- They're outside the monitoring window + +#### Game status by type + +``` +# Games by status +op_dispute_mon_games_status{status="challenger_wins"} 20 +op_dispute_mon_games_status{status="defender_wins"} 3 +op_dispute_mon_games_status{status="in_progress"} 2 +``` + +**What to monitor:** +- `defender_wins` should be rare (indicates your proposal was successfully defended) +- `challenger_wins` is normal (other proposers' incorrect proposals were challenged) +- `in_progress` shows active games + +#### Honest actor metrics + +``` +# Games involving your honest actors +op_dispute_mon_honest_actor_games{actor="0xYourProposer",role="proposer"} 15 +op_dispute_mon_honest_actor_games{actor="0xYourChallenger",role="challenger"} 10 +``` + +**What to monitor:** +- Your proposer should create games regularly +- Your challenger should participate when needed +- If either metric stops increasing, investigate your honest actors + +### Metric query examples + +Here are some useful queries you can run: + +```bash +# Count total games +curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement{" | grep -v "#" | awk '{sum+=$2} END {print sum}' + +# Count in-progress games +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{completion="in_progress"}' | awk '{print $2}' + +# Count disagreements (should be 0!) +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agreement="disagree"}' | awk '{print $2}' +``` + + + + + +Use these practices to ensure your dispute games are healthy: + +### Daily monitoring checklist + +1. **Check for disagreements:** + ```bash + curl -s http://localhost:7300/metrics | grep 'agreement="disagree"' + ``` + Result should be `0` or metric should not exist. + +2. **Verify honest actors are active:** + ```bash + curl -s http://localhost:7300/metrics | grep "honest_actor_games" + ``` + Both proposer and challenger metrics should be present and increasing. + +3. **Check game completion rate:** + ```bash + # Compare in-progress vs complete + curl -s http://localhost:7300/metrics | grep "completion=" + ``` + Most games should complete within the challenge period (7 days). + +4. **Review logs for errors:** + ```bash + # Docker + docker-compose logs --tail=100 op-dispute-mon | grep -i error + + # Source + tail -100 logs/dispute-mon.log | grep -i error + ``` + +### Warning signs + +**Immediate attention required:** +- Any `agreement="disagree"` metrics +- Honest actor metrics stop increasing +- High number of ignored games involving your actors +- Errors in logs about RPC connectivity +- Errors about invalid game states + +**Investigate soon:** +- Games stuck in `in_progress` for >7 days +- Sudden drop in game creation rate +- Increase in `defender_wins` status + +**Normal operation:** +- Regular game creation +- All agreements showing `agree` +- Games completing within challenge period +- Occasional `challenger_wins` (other proposals being challenged) + + + + +## Next steps + +Now that you have dispute game monitoring set up, you can: + +* Set up additional monitoring for other components (optional) +* Create alerting for critical metrics like disagreements +* Explore the [Dispute Monitor Tool Reference](/operators/chain-operators/tools/chain-monitoring) +* Learn more about the [Fault Proof System](/stack/fault-proofs/explainer) +* Set up [Prometheus](https://prometheus.io/docs/) and [Grafana](https://grafana.com/docs/) for advanced metrics visualization (optional) + +Congratulations! You now have a fully operational OP Stack L2 rollup with comprehensive monitoring. + From dcb9520c21e3ab0e8319dde75acd88a7df17aecb Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 05:17:47 +0100 Subject: [PATCH 05/17] feat: enhance dispute monitor configuration and setup - Added new environment variables for the dispute monitor in the .env file. - Updated Docker Compose configuration to include the new environment variables and ensure proper service setup. - Modified setup-rollup.sh to retrieve addresses from the updated JSON structure and create a more comprehensive environment file for the dispute monitor. - Improved logging messages for better user guidance during setup. --- create-l2-rollup-example/.example.env | 8 ++++++++ create-l2-rollup-example/docker-compose.yml | 7 ++++--- .../scripts/setup-rollup.sh | 19 +++++++++++++------ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/create-l2-rollup-example/.example.env b/create-l2-rollup-example/.example.env index 75a31144b..79f141ddd 100644 --- a/create-l2-rollup-example/.example.env +++ b/create-l2-rollup-example/.example.env @@ -54,6 +54,14 @@ OP_PROPOSER_PROPOSAL_INTERVAL="3600s" OP_PROPOSER_GAME_TYPE="0" OP_PROPOSER_POLL_INTERVAL="20s" +# Dispute monitor environment variables +# ROLLUP_RPC="http://op-node:8547" +# PROPOSER_ADDRESS="" +# CHALLENGER_ADDRESS="" +# OP_DISPUTE_MON_GAME_FACTORY_ADDRESS="" +# OP_DISPUTE_MON_NETWORK="op-sepolia" +# OP_DISPUTE_MON_MONITOR_INTERVAL="10s" + # ========================================== # DEVELOPMENT: Local Network (Alternative to Sepolia) # ========================================== diff --git a/create-l2-rollup-example/docker-compose.yml b/create-l2-rollup-example/docker-compose.yml index fb843c1b3..dd0e52d9a 100644 --- a/create-l2-rollup-example/docker-compose.yml +++ b/create-l2-rollup-example/docker-compose.yml @@ -156,19 +156,20 @@ services: volumes: - ./dispute-mon:/workspace working_dir: /workspace + ports: + - "7300:7300" env_file: - ./dispute-mon/.env - .env - ports: - - "7300:7300" environment: - OP_DISPUTE_MON_L1_ETH_RPC=${L1_RPC_URL} - OP_DISPUTE_MON_ROLLUP_RPC=${ROLLUP_RPC} - OP_DISPUTE_MON_HONEST_ACTORS=${PROPOSER_ADDRESS},${CHALLENGER_ADDRESS} - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} - OP_DISPUTE_MON_MONITOR_INTERVAL=10s + - OP_DISPUTE_MON_NETWORK=op-sepolia + - OP_DISPUTE_MON_LOG_LEVEL=debug - OP_DISPUTE_MON_LOG_FORMAT=logfmt - - OP_DISPUTE_MON_LOG_LEVEL=info - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 - OP_DISPUTE_MON_METRICS_ENABLED=true - OP_DISPUTE_MON_METRICS_PORT=7300 diff --git a/create-l2-rollup-example/scripts/setup-rollup.sh b/create-l2-rollup-example/scripts/setup-rollup.sh index d5e1f580f..23a4d7241 100755 --- a/create-l2-rollup-example/scripts/setup-rollup.sh +++ b/create-l2-rollup-example/scripts/setup-rollup.sh @@ -435,10 +435,10 @@ setup_dispute_monitor() { mkdir -p "$DISPUTE_MON_DIR" cd "$DISPUTE_MON_DIR" - # Get required addresses - GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' "$DEPLOYER_DIR/.deployer/state.json") - PROPOSER_ADDRESS=$(cast wallet address --private-key "$PROPOSER_PRIVATE_KEY") - CHALLENGER_ADDRESS=$(cast wallet address --private-key "$CHALLENGER_PRIVATE_KEY") + # Get required addresses from state.json + GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' "$DEPLOYER_DIR/.deployer/state.json") + PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' "$DEPLOYER_DIR/.deployer/state.json") + CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' "$DEPLOYER_DIR/.deployer/state.json") log_info "Game Factory: $GAME_FACTORY_ADDRESS" log_info "Proposer: $PROPOSER_ADDRESS" @@ -446,11 +446,18 @@ setup_dispute_monitor() { # Create environment file for dispute monitor cat > .env << EOF +# Rollup RPC Configuration +ROLLUP_RPC=http://op-node:8547 + # Contract Addresses OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS # Honest Actors -OP_DISPUTE_MON_HONEST_ACTORS=$PROPOSER_ADDRESS,$CHALLENGER_ADDRESS +PROPOSER_ADDRESS=$PROPOSER_ADDRESS +CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS + +# Network Configuration +OP_DISPUTE_MON_NETWORK=op-sepolia # Monitoring Configuration OP_DISPUTE_MON_MONITOR_INTERVAL=10s @@ -461,7 +468,7 @@ EOF log_success "Dispute monitor configuration created" log_info "Dispute monitor will start with 'docker-compose up -d' from project root" - log_info "Metrics will be available at: http://localhost:7300/metrics" + log_info "To verify it's working, run: curl -s http://localhost:7300/metrics | grep -E \"op_dispute_mon_(games|ignored)\" | head -10" } # Add op-deployer to PATH if it exists in the workspace From 131da54ad77032ee3c41e60d24584231915f5395 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 05:21:04 +0100 Subject: [PATCH 06/17] refactor: update monitoring chain tutorial and remove obsolete files - Replaced the monitoring chain tutorial with a new tutorial focused on op-dispute-setup. - Deleted the outdated monitoring-chain.mdx and monitoring-chain.mdx files to streamline documentation. - Updated docs.json to reflect the changes and improve navigation for users. --- docs.json | 2 +- ...itoring-chain.mdx => op-dispute-setup.mdx} | 0 .../tutorials/monitoring-chain.mdx | 528 ------------------ 3 files changed, 1 insertion(+), 529 deletions(-) rename operators/chain-operators/tutorials/create-l2-rollup/{monitoring-chain.mdx => op-dispute-setup.mdx} (100%) delete mode 100644 operators/chain-operators/tutorials/monitoring-chain.mdx diff --git a/docs.json b/docs.json index 430d1882f..0ffe2df00 100644 --- a/docs.json +++ b/docs.json @@ -767,7 +767,7 @@ "operators/chain-operators/tutorials/create-l2-rollup/op-batcher-setup", "operators/chain-operators/tutorials/create-l2-rollup/op-proposer-setup", "operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup", - "operators/chain-operators/tutorials/create-l2-rollup/monitoring-chain" + "operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup" ] }, "operators/chain-operators/tutorials/absolute-prestate", diff --git a/operators/chain-operators/tutorials/create-l2-rollup/monitoring-chain.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx similarity index 100% rename from operators/chain-operators/tutorials/create-l2-rollup/monitoring-chain.mdx rename to operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx diff --git a/operators/chain-operators/tutorials/monitoring-chain.mdx b/operators/chain-operators/tutorials/monitoring-chain.mdx deleted file mode 100644 index f13a8f25b..000000000 --- a/operators/chain-operators/tutorials/monitoring-chain.mdx +++ /dev/null @@ -1,528 +0,0 @@ ---- -title: Monitoring your OP Stack chain with Dispute Monitor -description: Learn how to set up op-dispute-mon to monitor dispute games and fault proof activity on your L2 rollup. ---- - -Now that you have a running OP Stack rollup with fault proofs enabled, it's critical to monitor your dispute games to ensure your chain operates securely. This tutorial will guide you through setting up `op-dispute-mon` to track all dispute game activity on your rollup. - - - This tutorial continues from [Creating Your Own L2 Rollup](/operators/chain-operators/tutorials/create-l2-rollup). Make sure you have completed the op-challenger setup and have dispute games running before proceeding. - - -## What you'll set up - -By the end of this tutorial, you'll have: - -* **op-dispute-mon** monitoring all dispute games on your chain -* **Metrics endpoint** exposing Prometheus-compatible metrics -* **Real-time tracking** of game status, agreements, and honest actor participation -* Understanding of key metrics for dispute game health - -## Prerequisites - -Before you begin, ensure you have: - -* Completed the [Creating Your Own L2 Rollup tutorial](/operators/chain-operators/tutorials/create-l2-rollup) -* op-challenger running and participating in dispute games -* op-proposer running and creating proposals -* L1 RPC endpoint with sufficient rate limits -* Rollup RPC endpoint (op-node) -* Docker installed (for Docker setup) OR Go 1.21+ (for build from source) - -## What is op-dispute-mon? - -`op-dispute-mon` is a specialized monitoring tool that tracks dispute games in the fault proof system. It provides visibility into: - -* **Game status**: Track games from creation to resolution -* **Honest actor Participation**: Monitor your proposer and challenger activity -* **Agreement tracking**: See which games agree/disagree with your honest actors -* **Game completion**: Monitor in-progress vs completed games -* **Ignored games**: Track games that don't meet monitoring criteria - -The monitor exposes Prometheus metrics that can be used to build dashboards and alerts. - -## Monitoring architecture - - - - -Choose your preferred deployment method: Docker (recommended) or build from source. - - - - -### Docker Setup - -The Docker setup provides a containerized environment for running op-dispute-mon with minimal configuration. - -#### Create dispute-mon directory - -```bash -# From your project root -cd rollup -mkdir dispute-mon -cd dispute-mon -``` - -#### Get required addresses - -You need three key addresses for op-dispute-mon: - -1. **DisputeGameFactory address** (from your deployment) -2. **Proposer address** (your honest actor who creates proposals) -3. **Challenger address** (your honest actor who defends proposals) - -```bash -# Get DisputeGameFactory address from deployment -export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../.deployer/state.json) -echo "Game Factory Address: $GAME_FACTORY_ADDRESS" - -# Get Proposer address (from your proposer setup) -source ../proposer/.env -export PROPOSER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) -echo "Proposer Address: $PROPOSER_ADDRESS" - -# Get Challenger address (from your challenger setup) -source ../challenger/.env -export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) -echo "Challenger Address: $CHALLENGER_ADDRESS" -``` - -#### Create Docker Compose file - -Create `docker-compose.yml` with the following configuration: - -```yaml -version: "3.8" - -services: - op-dispute-mon: - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.4.2-rc.1 - container_name: dispute-mon - ports: - - "7300:7300" # Metrics port - environment: - # L1 RPC Configuration - - OP_DISPUTE_MON_L1_ETH_RPC=https://sepolia.infura.io/v3/YOUR_INFURA_KEY - - # Rollup RPC Configuration (your op-node) - - OP_DISPUTE_MON_ROLLUP_RPC=http://host.docker.internal:8547 - - # Honest Actors (your proposer and challenger addresses) - - OP_DISPUTE_MON_HONEST_ACTORS=${PROPOSER_ADDRESS},${CHALLENGER_ADDRESS} - - # DisputeGameFactory Contract Address - - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} - - # Monitoring Configuration - - OP_DISPUTE_MON_MONITOR_INTERVAL=10s - - # Logging Configuration - - OP_DISPUTE_MON_LOG_FORMAT=logfmt - - OP_DISPUTE_MON_LOG_LEVEL=info - - # Metrics Configuration - - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 - - OP_DISPUTE_MON_METRICS_ENABLED=true - - OP_DISPUTE_MON_METRICS_PORT=7300 - restart: unless-stopped - networks: - - dispute-mon-network - -networks: - dispute-mon-network: - driver: bridge -``` - - - Replace `YOUR_INFURA_KEY` with your actual Infura API key or use another L1 RPC provider. Make sure your L1 RPC has sufficient rate limits for production use. - - -#### Create environment file (optional) - -If you prefer using an environment file instead of inline environment variables: - -```bash -cat > .env << EOF -# L1 Configuration -L1_RPC_URL=https://sepolia.infura.io/v3/YOUR_INFURA_KEY - -# Rollup Configuration -ROLLUP_RPC=http://host.docker.internal:8547 - -# Contract Addresses -GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS - -# Honest Actors -PROPOSER_ADDRESS=$PROPOSER_ADDRESS -CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS -EOF -``` - -Then update your `docker-compose.yml` to use the environment file: - -```yaml -version: "3.8" - -services: - op-dispute-mon: - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-dispute-mon:v1.4.2-rc.1 - container_name: dispute-mon - env_file: .env - ports: - - "7300:7300" - environment: - - OP_DISPUTE_MON_L1_ETH_RPC=${L1_RPC_URL} - - OP_DISPUTE_MON_ROLLUP_RPC=${ROLLUP_RPC} - - OP_DISPUTE_MON_HONEST_ACTORS=${PROPOSER_ADDRESS},${CHALLENGER_ADDRESS} - - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} - - OP_DISPUTE_MON_MONITOR_INTERVAL=10s - - OP_DISPUTE_MON_LOG_FORMAT=logfmt - - OP_DISPUTE_MON_LOG_LEVEL=info - - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 - - OP_DISPUTE_MON_METRICS_ENABLED=true - - OP_DISPUTE_MON_METRICS_PORT=7300 - restart: unless-stopped - networks: - - dispute-mon-network - -networks: - dispute-mon-network: - driver: bridge -``` - -#### Start op-dispute-mon - -```bash -# Export variables if not using .env file -export PROPOSER_ADDRESS=0xYourProposerAddress -export CHALLENGER_ADDRESS=0xYourChallengerAddress -export GAME_FACTORY_ADDRESS=0xYourGameFactoryAddress - -# Start the container -docker-compose up -d - -# View logs -docker-compose logs -f op-dispute-mon -``` - -#### Verify it's running - -Check that the metrics endpoint is accessible: - -```bash -# Query metrics endpoint -curl http://localhost:7300/metrics - -# You should see Prometheus metrics including: -# - op_dispute_mon_games_agreement -# - op_dispute_mon_ignored_games -# - and many more... -``` - -Check logs for any errors: - -```bash -# View recent logs -docker-compose logs --tail=50 op-dispute-mon - -# Follow logs in real-time -docker-compose logs -f op-dispute-mon -``` - -You should see logs indicating the monitor is tracking games: - -``` -level=info msg="Monitoring dispute games" factory=0x... honest_actors=[0x..., 0x...] -level=info msg="Found games" count=5 -level=info msg="Updated game status" game=0x... status=IN_PROGRESS -``` - - - - - -### Build from Source - -Building from source gives you more control and is useful for development or custom deployments. - -#### Clone and build op-dispute-mon - -```bash -# From your project root -cd optimism - -# Pull latest changes -git pull - -# Build op-dispute-mon -cd op-dispute-mon -make op-dispute-mon - -# Verify the build -./bin/op-dispute-mon --version -``` - -#### Create directory structure - -```bash -# From your project root -cd rollup -mkdir -p dispute-mon/scripts -cd dispute-mon -``` - -#### Get required addresses - -```bash -# Get DisputeGameFactory address -export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../.deployer/state.json) -echo "Game Factory Address: $GAME_FACTORY_ADDRESS" - -# Get Proposer address -source ../proposer/.env -export PROPOSER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) -echo "Proposer Address: $PROPOSER_ADDRESS" - -# Get Challenger address -source ../challenger/.env -export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) -echo "Challenger Address: $CHALLENGER_ADDRESS" - -# Get L1 RPC URL -source ../.env -echo "L1 RPC: $L1_RPC_URL" -``` - -#### Create environment file - -```bash -cat > .env << EOF -# L1 Configuration -L1_RPC_URL=$L1_RPC_URL - -# Rollup Configuration -ROLLUP_RPC=http://localhost:8547 - -# Contract Addresses -GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS - -# Honest Actors -PROPOSER_ADDRESS=$PROPOSER_ADDRESS -CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS - -# Monitoring Configuration -MONITOR_INTERVAL=10s -LOG_LEVEL=info - -# Metrics Configuration -METRICS_PORT=7300 -EOF -``` - -#### Create startup script - -Create `scripts/start-dispute-mon.sh`: - -```bash -#!/bin/bash -set -e - -# Load environment variables -source .env - -# Path to op-dispute-mon binary -DISPUTE_MON_BIN=../../optimism/op-dispute-mon/bin/op-dispute-mon - -echo "Starting op-dispute-mon..." -echo "Game Factory: $GAME_FACTORY_ADDRESS" -echo "Proposer: $PROPOSER_ADDRESS" -echo "Challenger: $CHALLENGER_ADDRESS" - -$DISPUTE_MON_BIN \ - --l1-eth-rpc=$L1_RPC_URL \ - --rollup-rpc=$ROLLUP_RPC \ - --game-factory-address=$GAME_FACTORY_ADDRESS \ - --honest-actors=$PROPOSER_ADDRESS,$CHALLENGER_ADDRESS \ - --monitor-interval=$MONITOR_INTERVAL \ - --log.level=$LOG_LEVEL \ - --log.format=logfmt \ - --metrics.enabled=true \ - --metrics.addr=0.0.0.0 \ - --metrics.port=$METRICS_PORT -``` - -Make it executable: - -```bash -chmod +x scripts/start-dispute-mon.sh -``` - -#### Start op-dispute-mon - -```bash -# Start in foreground (for testing) -./scripts/start-dispute-mon.sh - -# Or start in background -./scripts/start-dispute-mon.sh > logs/dispute-mon.log 2>&1 & - -# Save the process ID -echo $! > dispute-mon.pid -``` - -#### Verify it's running - -```bash -# Check if process is running -ps aux | grep op-dispute-mon - -# Query metrics endpoint -curl http://localhost:7300/metrics - -# Check logs -tail -f logs/dispute-mon.log -``` - - - - - - - - -Once op-dispute-mon is running, it exposes several important metrics that help you monitor dispute game health. - -### Core metrics - -Access the metrics endpoint: - -```bash -curl http://localhost:7300/metrics | grep op_dispute_mon -``` - -#### Game agreement metrics - -``` -# Total number of games tracked -op_dispute_mon_games_agreement 25 - -# Games currently in progress -op_dispute_mon_games_agreement{completion="in_progress"} 3 - -# Completed games -op_dispute_mon_games_agreement{completion="complete"} 22 - -# Games where monitor agrees with honest actors -op_dispute_mon_games_agreement{agreement="agree"} 24 - -# Games where monitor disagrees -op_dispute_mon_games_agreement{agreement="disagree"} 1 -``` - -**What to monitor:** -- If `agreement="disagree"` is non-zero, investigate immediately - this indicates a potential issue with your honest actors -- High number of `completion="in_progress"` games is normal during active periods -- All completed games should show `agreement="agree"` - -#### Ignored games - -``` -# Games that don't meet monitoring criteria -op_dispute_mon_ignored_games 5 -``` - -Games may be ignored if: -- They don't involve your honest actors -- They use an unsupported game type -- They're outside the monitoring window - -#### Game status by type - -``` -# Games by status -op_dispute_mon_games_status{status="challenger_wins"} 20 -op_dispute_mon_games_status{status="defender_wins"} 3 -op_dispute_mon_games_status{status="in_progress"} 2 -``` - -**What to monitor:** -- `defender_wins` should be rare (indicates your proposal was successfully defended) -- `challenger_wins` is normal (other proposers' incorrect proposals were challenged) -- `in_progress` shows active games - -#### Honest actor metrics - -``` -# Games involving your honest actors -op_dispute_mon_honest_actor_games{actor="0xYourProposer",role="proposer"} 15 -op_dispute_mon_honest_actor_games{actor="0xYourChallenger",role="challenger"} 10 -``` - -**What to monitor:** -- Your proposer should create games regularly -- Your challenger should participate when needed -- If either metric stops increasing, investigate your honest actors - -### Metric query examples - -Here are some useful queries you can run: - -```bash -# Count total games -curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement{" | grep -v "#" | awk '{sum+=$2} END {print sum}' - -# Count in-progress games -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{completion="in_progress"}' | awk '{print $2}' - -# Count disagreements (should be 0!) -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agreement="disagree"}' | awk '{print $2}' -``` - - - - - -Use these practices to ensure your dispute games are healthy: - -### Daily monitoring checklist - -1. **Check for disagreements:** -```bash - curl -s http://localhost:7300/metrics | grep 'agreement="disagree"' - ``` - Result should be `0` or metric should not exist. - -2. **Verify honest actors are active:** - ```bash - curl -s http://localhost:7300/metrics | grep "honest_actor_games" - ``` - Both proposer and challenger metrics should be present and increasing. - -3. **Check game completion rate:** -```bash - # Compare in-progress vs complete - curl -s http://localhost:7300/metrics | grep "completion=" -``` - Most games should complete within the challenge period (7 days). - -4. **Review logs for errors:** -```bash - # Docker - docker-compose logs --tail=100 op-dispute-mon | grep -i error - - # Source - tail -100 logs/dispute-mon.log | grep -i error -``` - - - - - -## Next steps - -* [Dispute Monitor Tool Reference](/operators/chain-operators/tools/chain-monitoring) -* [Fault Proof System Overview](/stack/fault-proofs/explainer) -* [OP Challenger Setup](/operators/chain-operators/tutorials/create-l2-rollup/op-challenger-setup) -* [Prometheus Documentation](https://prometheus.io/docs/) (for optional metrics visualization) -* [Grafana Documentation](https://grafana.com/docs/) (for optional dashboarding) From 8bda9ca567cca2381f889ad92c91a69283a28d29 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 06:06:12 +0100 Subject: [PATCH 07/17] refactor: update op-dispute-setup tutorial for improved address retrieval and logging - Modified address retrieval commands to use the updated JSON structure for DisputeGameFactory, Proposer, and Challenger. - Enhanced Docker Compose configuration with new network and logging settings. - Improved logging output for better monitoring visibility during dispute game operations. - Removed outdated monitoring health checklist to streamline the tutorial. --- .../dispute-mon/scripts/start-dispute-mon.sh | 42 +++++++ .../create-l2-rollup/op-dispute-setup.mdx | 114 +++++++++--------- 2 files changed, 100 insertions(+), 56 deletions(-) create mode 100755 create-l2-rollup-example/dispute-mon/scripts/start-dispute-mon.sh diff --git a/create-l2-rollup-example/dispute-mon/scripts/start-dispute-mon.sh b/create-l2-rollup-example/dispute-mon/scripts/start-dispute-mon.sh new file mode 100755 index 000000000..c8632ea87 --- /dev/null +++ b/create-l2-rollup-example/dispute-mon/scripts/start-dispute-mon.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Change to dispute-mon directory +cd "$SCRIPT_DIR/.." + +# Load environment variables +source .env + +# Path to op-dispute-mon binary +DISPUTE_MON_BIN="$PROJECT_ROOT/optimism/op-dispute-mon/bin/op-dispute-mon" + +# Check if binary exists +if [ ! -f "$DISPUTE_MON_BIN" ]; then + echo "Error: op-dispute-mon binary not found at $DISPUTE_MON_BIN" + echo "Please build op-dispute-mon first:" + echo " cd $PROJECT_ROOT/optimism/op-dispute-mon" + echo " make op-dispute-mon" + exit 1 +fi + +echo "Starting op-dispute-mon..." +echo "Game Factory: $GAME_FACTORY_ADDRESS" +echo "Proposer: $PROPOSER_ADDRESS" +echo "Challenger: $CHALLENGER_ADDRESS" + +$DISPUTE_MON_BIN \ + --l1-eth-rpc=$L1_RPC_URL \ + --rollup-rpc=$ROLLUP_RPC \ + --game-factory-address=$GAME_FACTORY_ADDRESS \ + --honest-actors=$PROPOSER_ADDRESS,$CHALLENGER_ADDRESS \ + --network=$NETWORK \ + --monitor-interval=$MONITOR_INTERVAL \ + --log.level=$LOG_LEVEL \ + --log.format=$LOG_FORMAT \ + --metrics.enabled=$METRICS_ENABLED \ + --metrics.addr=$METRICS_ADDR \ + --metrics.port=$METRICS_PORT \ No newline at end of file diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index 1b379c0d7..aee2f6a7b 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -71,18 +71,19 @@ You need three key addresses for op-dispute-mon: ```bash # Get DisputeGameFactory address from deployment -export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../deployer/.deployer/state.json) +export GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' ../deployer/.deployer/state.json) echo "Game Factory Address: $GAME_FACTORY_ADDRESS" # Get Proposer address (from your proposer setup) source ../proposer/.env -export PROPOSER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +export PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' ../deployer/.deployer/state.json) echo "Proposer Address: $PROPOSER_ADDRESS" # Get Challenger address (from your challenger setup) source ../challenger/.env -export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +export CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' ../deployer/.deployer/state.json) echo "Challenger Address: $CHALLENGER_ADDRESS" + ``` #### Create Docker Compose file @@ -113,10 +114,11 @@ services: # Monitoring Configuration - OP_DISPUTE_MON_MONITOR_INTERVAL=10s + - OP_DISPUTE_MON_NETWORK=op-sepolia # Logging Configuration - OP_DISPUTE_MON_LOG_FORMAT=logfmt - - OP_DISPUTE_MON_LOG_LEVEL=info + - OP_DISPUTE_MON_LOG_LEVEL=debug # Metrics Configuration - OP_DISPUTE_MON_METRICS_ADDR=0.0.0.0 @@ -169,17 +171,22 @@ Check logs for any errors: ```bash # View recent logs docker-compose logs --tail=50 op-dispute-mon - -# Follow logs in real-time -docker-compose logs -f op-dispute-mon ``` You should see logs indicating the monitor is tracking games: -``` -level=info msg="Monitoring dispute games" factory=0x... honest_actors=[0x..., 0x...] -level=info msg="Found games" count=5 -level=info msg="Updated game status" game=0x... status=IN_PROGRESS +```bash +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting op-dispute-mon" version=untagged-48e52dc5-1759893117 +t=2025-10-21T05:58:52+0100 lvl=info msg="started metrics server" addr=[::]:7300 +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting scheduler" +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting monitoring" +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting game monitor" +t=2025-10-21T05:58:52+0100 lvl=info msg="Dispute monitor game service start completed" +t=2025-10-21T05:59:03+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=599.887083ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:12+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=134.839417ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:22+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=104.085292ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:32+0100 lvl=info msg="Completed monitoring update" blockNumber=9456909 blockHash=0x18b6a5ef128058c4dd6c92ee2a2e4484a58c24adf64c9ce2763812c581814ab3 duration=111.325667ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:42+0100 lvl=info msg="Completed monitoring update" blockNumber=9456910 blockHash=0xb38f14c19116bbef056c2194ca966940985a7c6a73b4eb7e765c401a84f5f269 duration=97.361917ms games=0 ignored=0 failed=0 ``` @@ -205,6 +212,9 @@ make op-dispute-mon # Verify the build ./bin/op-dispute-mon --version + +# Return to project root +cd ../.. ``` #### Create directory structure @@ -219,23 +229,24 @@ cd dispute-mon #### Get required addresses ```bash -# Get DisputeGameFactory address -export GAME_FACTORY_ADDRESS=$(jq -r '.disputeGameFactoryProxyAddress' ../deployer/.deployer/state.json) +# Get DisputeGameFactory address from deployment +export GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' ../deployer/.deployer/state.json) echo "Game Factory Address: $GAME_FACTORY_ADDRESS" -# Get Proposer address +# Get Proposer address (from your proposer setup) source ../proposer/.env -export PROPOSER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +export PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' ../deployer/.deployer/state.json) echo "Proposer Address: $PROPOSER_ADDRESS" -# Get Challenger address +# Get Challenger address (from your challenger setup) source ../challenger/.env -export CHALLENGER_ADDRESS=$(cast wallet address --private-key $PRIVATE_KEY) +export CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' ../deployer/.deployer/state.json) echo "Challenger Address: $CHALLENGER_ADDRESS" # Get L1 RPC URL source ../.env echo "L1 RPC: $L1_RPC_URL" + ``` #### Create environment file @@ -255,11 +266,19 @@ GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS PROPOSER_ADDRESS=$PROPOSER_ADDRESS CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS +# Network Configuration +NETWORK=op-sepolia + # Monitoring Configuration MONITOR_INTERVAL=10s + +# Logging Configuration LOG_LEVEL=info +LOG_FORMAT=logfmt # Metrics Configuration +METRICS_ADDR=0.0.0.0 +METRICS_ENABLED=true METRICS_PORT=7300 EOF ``` @@ -272,11 +291,27 @@ Create `scripts/start-dispute-mon.sh`: #!/bin/bash set -e +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Change to dispute-mon directory +cd "$SCRIPT_DIR/.." + # Load environment variables source .env # Path to op-dispute-mon binary -DISPUTE_MON_BIN=../../optimism/op-dispute-mon/bin/op-dispute-mon +DISPUTE_MON_BIN="$PROJECT_ROOT/optimism/op-dispute-mon/bin/op-dispute-mon" + +# Check if binary exists +if [ ! -f "$DISPUTE_MON_BIN" ]; then + echo "Error: op-dispute-mon binary not found at $DISPUTE_MON_BIN" + echo "Please build op-dispute-mon first:" + echo " cd $PROJECT_ROOT/optimism/op-dispute-mon" + echo " make op-dispute-mon" + exit 1 +fi echo "Starting op-dispute-mon..." echo "Game Factory: $GAME_FACTORY_ADDRESS" @@ -288,11 +323,12 @@ $DISPUTE_MON_BIN \ --rollup-rpc=$ROLLUP_RPC \ --game-factory-address=$GAME_FACTORY_ADDRESS \ --honest-actors=$PROPOSER_ADDRESS,$CHALLENGER_ADDRESS \ + --network=$NETWORK \ --monitor-interval=$MONITOR_INTERVAL \ --log.level=$LOG_LEVEL \ - --log.format=logfmt \ - --metrics.enabled=true \ - --metrics.addr=0.0.0.0 \ + --log.format=$LOG_FORMAT \ + --metrics.enabled=$METRICS_ENABLED \ + --metrics.addr=$METRICS_ADDR \ --metrics.port=$METRICS_PORT ``` @@ -425,39 +461,6 @@ curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agr - - -Use these practices to ensure your dispute games are healthy: - -### Daily monitoring checklist - -1. **Check for disagreements:** - ```bash - curl -s http://localhost:7300/metrics | grep 'agreement="disagree"' - ``` - Result should be `0` or metric should not exist. - -2. **Verify honest actors are active:** - ```bash - curl -s http://localhost:7300/metrics | grep "honest_actor_games" - ``` - Both proposer and challenger metrics should be present and increasing. - -3. **Check game completion rate:** - ```bash - # Compare in-progress vs complete - curl -s http://localhost:7300/metrics | grep "completion=" - ``` - Most games should complete within the challenge period (7 days). - -4. **Review logs for errors:** - ```bash - # Docker - docker-compose logs --tail=100 op-dispute-mon | grep -i error - - # Source - tail -100 logs/dispute-mon.log | grep -i error - ``` ### Warning signs @@ -492,5 +495,4 @@ Now that you have dispute game monitoring set up, you can: * Learn more about the [Fault Proof System](/stack/fault-proofs/explainer) * Set up [Prometheus](https://prometheus.io/docs/) and [Grafana](https://grafana.com/docs/) for advanced metrics visualization (optional) -Congratulations! You now have a fully operational OP Stack L2 rollup with comprehensive monitoring. - +Congratulations! You now have a fully operational OP Stack L2 rollup with comprehensive monitoring. \ No newline at end of file From e6418ad4ea4e17e6e591b48833d0b19a79669f28 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 06:09:03 +0100 Subject: [PATCH 08/17] feat: add monitoring step to op-dispute-setup tutorial - Introduced a new step titled "Monitor dispute game health" to enhance the tutorial. - This addition aims to provide users with guidance on monitoring the health of dispute games effectively. --- .../tutorials/create-l2-rollup/op-dispute-setup.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index aee2f6a7b..e40f34447 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -461,6 +461,7 @@ curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agr + ### Warning signs From 757dc78473146823a7e5ef9f3b0c832c1177ba03 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 06:10:10 +0100 Subject: [PATCH 09/17] fix: update links and improve formatting in op-dispute-setup tutorial - Corrected capitalization in the titles for "Dispute monitor tool reference" and "fault proof system" for consistency. - Removed an optional monitoring step to streamline the tutorial. - Enhanced clarity and readability of the tutorial content. --- .../tutorials/create-l2-rollup/op-dispute-setup.mdx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index e40f34447..81960b5e7 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -490,10 +490,9 @@ curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agr Now that you have dispute game monitoring set up, you can: -* Set up additional monitoring for other components (optional) * Create alerting for critical metrics like disagreements -* Explore the [Dispute Monitor Tool Reference](/operators/chain-operators/tools/chain-monitoring) -* Learn more about the [Fault Proof System](/stack/fault-proofs/explainer) +* Explore the [Dispute monitor tool reference](/operators/chain-operators/tools/chain-monitoring) +* Learn more about the [fault proof system](/stack/fault-proofs/explainer) * Set up [Prometheus](https://prometheus.io/docs/) and [Grafana](https://grafana.com/docs/) for advanced metrics visualization (optional) Congratulations! You now have a fully operational OP Stack L2 rollup with comprehensive monitoring. \ No newline at end of file From 4f437664155b715ed345ebd2d22c084a97280820 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 06:21:47 +0100 Subject: [PATCH 10/17] refactor: enhance op-dispute-setup tutorial for clarity and structure - Updated the tutorial to include a clear choice of deployment methods for the dispute monitor: Docker and building from source. - Improved step organization and clarity, ensuring users can easily follow the setup process. - Added detailed instructions for configuring and starting the dispute monitor, including environment variable setup and logging. - Enhanced the monitoring section with specific metrics to track and their significance for dispute game health. - Streamlined the content to remove redundancy and improve overall readability. --- .../create-l2-rollup/op-dispute-setup.mdx | 308 ++++++++---------- 1 file changed, 142 insertions(+), 166 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index 81960b5e7..f2a5b67c2 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -41,52 +41,60 @@ Before you begin, ensure you have: The monitor exposes Prometheus metrics that can be used to build dashboards and alerts. - + -Choose your preferred deployment method: Docker (recommended) or build from source. +This tutorial supports two deployment methods. Choose the one that best fits your needs: - - +* **Docker (Recommended)**: Easiest to set up and maintain. All dependencies are containerized. +* **Build from Source**: More control over the build process. Useful for development or custom deployments. + +The following steps will show instructions for both methods where they differ. -### Docker Setup + -The Docker setup provides a containerized environment for running op-dispute-mon with minimal configuration. + -#### Create dispute-mon directory +Set up the directory structure for your dispute monitor. ```bash # From your project root cd rollup -mkdir dispute-mon +mkdir -p dispute-mon/scripts cd dispute-mon ``` -#### Get required addresses + -You need three key addresses for op-dispute-mon: + -1. **DisputeGameFactory address** (from your deployment) -2. **Proposer address** (your honest actor who creates proposals) -3. **Challenger address** (your honest actor who defends proposals) +Extract the necessary addresses from your deployment state. ```bash # Get DisputeGameFactory address from deployment export GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' ../deployer/.deployer/state.json) echo "Game Factory Address: $GAME_FACTORY_ADDRESS" -# Get Proposer address (from your proposer setup) -source ../proposer/.env +# Get Proposer address export PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' ../deployer/.deployer/state.json) echo "Proposer Address: $PROPOSER_ADDRESS" -# Get Challenger address (from your challenger setup) -source ../challenger/.env +# Get Challenger address export CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' ../deployer/.deployer/state.json) echo "Challenger Address: $CHALLENGER_ADDRESS" +# Get L1 RPC URL +source ../.env +echo "L1 RPC: $L1_RPC_URL" ``` -#### Create Docker Compose file + + + + +Choose your deployment method and create the necessary configuration files. + + + Create `docker-compose.yml` with the following configuration: @@ -137,67 +145,11 @@ networks: Replace `YOUR_INFURA_KEY` with your actual Infura API key or use another L1 RPC provider. Make sure your L1 RPC has sufficient rate limits for production use. -#### Start op-dispute-mon - -```bash -# Export variables -export PROPOSER_ADDRESS=0xYourProposerAddress -export CHALLENGER_ADDRESS=0xYourChallengerAddress -export GAME_FACTORY_ADDRESS=0xYourGameFactoryAddress - -# Start the container -docker-compose up -d - -# View logs -docker-compose logs -f op-dispute-mon -``` - -#### Verify it's running - -Check that the metrics endpoint is accessible: - -```bash -# Query metrics endpoint -curl http://localhost:7300/metrics - -# You should see Prometheus metrics including: -# - op_dispute_mon_games_agreement -# - op_dispute_mon_ignored_games -# - and many more... -``` - -Check logs for any errors: - -```bash -# View recent logs -docker-compose logs --tail=50 op-dispute-mon -``` - -You should see logs indicating the monitor is tracking games: - -```bash -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting op-dispute-mon" version=untagged-48e52dc5-1759893117 -t=2025-10-21T05:58:52+0100 lvl=info msg="started metrics server" addr=[::]:7300 -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting scheduler" -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting monitoring" -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting game monitor" -t=2025-10-21T05:58:52+0100 lvl=info msg="Dispute monitor game service start completed" -t=2025-10-21T05:59:03+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=599.887083ms games=0 ignored=0 failed=0 -t=2025-10-21T05:59:12+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=134.839417ms games=0 ignored=0 failed=0 -t=2025-10-21T05:59:22+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=104.085292ms games=0 ignored=0 failed=0 -t=2025-10-21T05:59:32+0100 lvl=info msg="Completed monitoring update" blockNumber=9456909 blockHash=0x18b6a5ef128058c4dd6c92ee2a2e4484a58c24adf64c9ce2763812c581814ab3 duration=111.325667ms games=0 ignored=0 failed=0 -t=2025-10-21T05:59:42+0100 lvl=info msg="Completed monitoring update" blockNumber=9456910 blockHash=0xb38f14c19116bbef056c2194ca966940985a7c6a73b4eb7e765c401a84f5f269 duration=97.361917ms games=0 ignored=0 failed=0 -``` - -### Build from Source - -Building from source gives you more control and is useful for development or custom deployments. - -#### Clone and build op-dispute-mon +First, build op-dispute-mon from source: ```bash # From your project root @@ -217,39 +169,7 @@ make op-dispute-mon cd ../.. ``` -#### Create directory structure - -```bash -# From your project root -cd rollup -mkdir -p dispute-mon/scripts -cd dispute-mon -``` - -#### Get required addresses - -```bash -# Get DisputeGameFactory address from deployment -export GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' ../deployer/.deployer/state.json) -echo "Game Factory Address: $GAME_FACTORY_ADDRESS" - -# Get Proposer address (from your proposer setup) -source ../proposer/.env -export PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' ../deployer/.deployer/state.json) -echo "Proposer Address: $PROPOSER_ADDRESS" - -# Get Challenger address (from your challenger setup) -source ../challenger/.env -export CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' ../deployer/.deployer/state.json) -echo "Challenger Address: $CHALLENGER_ADDRESS" - -# Get L1 RPC URL -source ../.env -echo "L1 RPC: $L1_RPC_URL" - -``` - -#### Create environment file +Then create the environment file: ```bash cat > .env << EOF @@ -283,9 +203,7 @@ METRICS_PORT=7300 EOF ``` -#### Create startup script - -Create `scripts/start-dispute-mon.sh`: +Now create the startup script `scripts/start-dispute-mon.sh`: ```bash #!/bin/bash @@ -338,20 +256,93 @@ Make it executable: chmod +x scripts/start-dispute-mon.sh ``` -#### Start op-dispute-mon + + + + + + + +Launch the dispute monitor using your chosen deployment method. + + + + +```bash +# Export variables +export PROPOSER_ADDRESS=0xYourProposerAddress +export CHALLENGER_ADDRESS=0xYourChallengerAddress +export GAME_FACTORY_ADDRESS=0xYourGameFactoryAddress + +# Start the container +docker-compose up -d + +# View logs +docker-compose logs -f op-dispute-mon +``` + + + + ```bash # Start in foreground (for testing) ./scripts/start-dispute-mon.sh -# Or start in background +# Or start in background with logging +mkdir -p logs ./scripts/start-dispute-mon.sh > logs/dispute-mon.log 2>&1 & # Save the process ID echo $! > dispute-mon.pid ``` -#### Verify it's running + + + + + + + +Check that the dispute monitor is operational. + + + + +Check the metrics endpoint: + +```bash +# Query metrics endpoint +curl http://localhost:7300/metrics + +# You should see Prometheus metrics including: +# - op_dispute_mon_games_agreement +# - op_dispute_mon_ignored_games +# - and many more... +``` + +Check logs for any errors: + +```bash +# View recent logs +docker-compose logs --tail=50 op-dispute-mon +``` + +You should see logs indicating the monitor is tracking games: + +```bash +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting op-dispute-mon" version=untagged-48e52dc5-1759893117 +t=2025-10-21T05:58:52+0100 lvl=info msg="started metrics server" addr=[::]:7300 +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting scheduler" +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting monitoring" +t=2025-10-21T05:58:52+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 duration=599.887083ms games=0 ignored=0 failed=0 +``` + + + + + +Check if the process is running: ```bash # Check if process is running @@ -373,97 +364,82 @@ tail -f logs/dispute-mon.log Once op-dispute-mon is running, it exposes several important metrics that help you monitor dispute game health. -### Core metrics - Access the metrics endpoint: ```bash curl http://localhost:7300/metrics | grep op_dispute_mon ``` -#### Game agreement metrics +**Game agreement metrics:** -``` -# Total number of games tracked -op_dispute_mon_games_agreement 25 +The dispute monitor tracks games and categorizes them by completion status and agreement with your honest actors: -# Games currently in progress -op_dispute_mon_games_agreement{completion="in_progress"} 3 +``` +op_dispute_mon_games_agreement{completion="complete",result_correctness="correct",root_agreement="agree",status="agree_defender_wins"} 0 +op_dispute_mon_games_agreement{completion="in_progress",result_correctness="correct",root_agreement="agree",status="agree_defender_ahead"} 0 +``` -# Completed games -op_dispute_mon_games_agreement{completion="complete"} 22 +**What to monitor:** +- Any non-zero value for `root_agreement="disagree"` requires immediate investigation +- Games with `completion="in_progress"` are actively being disputed +- For a new testnet, all values being `0` is normal (no disputes yet) -# Games where monitor agrees with honest actors -op_dispute_mon_games_agreement{agreement="agree"} 24 +**Ignored games:** -# Games where monitor disagrees -op_dispute_mon_games_agreement{agreement="disagree"} 1 +``` +op_dispute_mon_ignored_games 0 ``` -**What to monitor:** -- If `agreement="disagree"` is non-zero, investigate immediately - this indicates a potential issue with your honest actors -- High number of `completion="in_progress"` games is normal during active periods -- All completed games should show `agreement="agree"` +This tracks games that don't meet your monitoring criteria (e.g., games not involving your honest actors). -#### Ignored games +**Failed games:** ``` -# Games that don't meet monitoring criteria -op_dispute_mon_ignored_games 5 +op_dispute_mon_failed_games 0 ``` -Games may be ignored if: -- They don't involve your honest actors -- They use an unsupported game type -- They're outside the monitoring window +This tracks games that failed to be monitored due to errors. -#### Game status by type +**Monitor status:** ``` -# Games by status -op_dispute_mon_games_status{status="challenger_wins"} 20 -op_dispute_mon_games_status{status="defender_wins"} 3 -op_dispute_mon_games_status{status="in_progress"} 2 +op_dispute_mon_up 1 ``` -**What to monitor:** -- `defender_wins` should be rare (indicates your proposal was successfully defended) -- `challenger_wins` is normal (other proposers' incorrect proposals were challenged) -- `in_progress` shows active games +This should always be `1` when the monitor is running properly. If `0`, the monitor hasn't fully initialized. -#### Honest actor metrics +**Honest actor bonds:** ``` -# Games involving your honest actors -op_dispute_mon_honest_actor_games{actor="0xYourProposer",role="proposer"} 15 -op_dispute_mon_honest_actor_games{actor="0xYourChallenger",role="challenger"} 10 +op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourAddress",state="pending"} 0 +op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourAddress",state="won"} 0 +op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourAddress",state="lost"} 0 ``` -**What to monitor:** -- Your proposer should create games regularly -- Your challenger should participate when needed -- If either metric stops increasing, investigate your honest actors - -### Metric query examples +This tracks the bonds posted by your proposer and challenger, and whether they were won or lost. -Here are some useful queries you can run: +**Query examples:** ```bash -# Count total games -curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement{" | grep -v "#" | awk '{sum+=$2} END {print sum}' +# Check monitor status +curl -s http://localhost:7300/metrics | grep "op_dispute_mon_up" -# Count in-progress games -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{completion="in_progress"}' | awk '{print $2}' +# Check for any games being monitored +curl -s http://localhost:7300/metrics | grep -E "op_dispute_mon_(games|ignored|failed)" -# Count disagreements (should be 0!) -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agreement="disagree"}' | awk '{print $2}' +# View all game agreement metrics +curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement" ``` + +These metrics are based on the actual op-dispute-mon v1.4.2 implementation. Values will be `0` for new testnets until dispute games are created. + + -### Warning signs +Use these practices to ensure your dispute games are healthy. **Immediate attention required:** - Any `agreement="disagree"` metrics From 6caca6ae503b3eb238979d7caac859f6be9c6587 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 07:26:21 +0100 Subject: [PATCH 11/17] refactor: streamline op-dispute-setup tutorial for improved clarity and usability - Consolidated deployment instructions for the op-dispute-mon, emphasizing Docker as the preferred method. - Enhanced step organization and clarity, particularly in the setup and verification sections. - Added detailed commands for starting the monitor and verifying its operation, including metrics checks. - Improved explanations of key metrics and their significance for monitoring dispute game health. - Removed redundant content to enhance overall readability and user experience. --- .../create-l2-rollup/op-dispute-setup.mdx | 310 ++++++++++-------- 1 file changed, 167 insertions(+), 143 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index f2a5b67c2..d329a040f 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -41,60 +41,52 @@ Before you begin, ensure you have: The monitor exposes Prometheus metrics that can be used to build dashboards and alerts. - + -This tutorial supports two deployment methods. Choose the one that best fits your needs: +Choose your preferred deployment method: Docker (recommended) or build from source. -* **Docker (Recommended)**: Easiest to set up and maintain. All dependencies are containerized. -* **Build from Source**: More control over the build process. Useful for development or custom deployments. - -The following steps will show instructions for both methods where they differ. + + - +### Docker Setup - +The Docker setup provides a containerized environment for running op-dispute-mon with minimal configuration. -Set up the directory structure for your dispute monitor. +#### Create dispute-mon directory ```bash # From your project root cd rollup -mkdir -p dispute-mon/scripts +mkdir dispute-mon cd dispute-mon ``` - +#### Get required addresses - +You need three key addresses for op-dispute-mon: -Extract the necessary addresses from your deployment state. +1. **DisputeGameFactory address** (from your deployment) +2. **Proposer address** (your honest actor who creates proposals) +3. **Challenger address** (your honest actor who defends proposals) ```bash # Get DisputeGameFactory address from deployment export GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' ../deployer/.deployer/state.json) echo "Game Factory Address: $GAME_FACTORY_ADDRESS" -# Get Proposer address +# Get Proposer address (from your proposer setup) +source ../proposer/.env export PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' ../deployer/.deployer/state.json) echo "Proposer Address: $PROPOSER_ADDRESS" -# Get Challenger address +# Get Challenger address (from your challenger setup) +source ../challenger/.env export CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' ../deployer/.deployer/state.json) echo "Challenger Address: $CHALLENGER_ADDRESS" -# Get L1 RPC URL -source ../.env -echo "L1 RPC: $L1_RPC_URL" ``` - - - - -Choose your deployment method and create the necessary configuration files. - - - +#### Create Docker Compose file Create `docker-compose.yml` with the following configuration: @@ -142,14 +134,70 @@ networks: ``` - Replace `YOUR_INFURA_KEY` with your actual Infura API key or use another L1 RPC provider. Make sure your L1 RPC has sufficient rate limits for production use. + Replace `YOUR_INFURA_KEY` with your actual Infura API key or use another L1 RPC provider. Make sure your L1 RPC has sufficient rate limits. +#### Start op-dispute-mon + +```bash +# Export variables +export PROPOSER_ADDRESS=0xYourProposerAddress +export CHALLENGER_ADDRESS=0xYourChallengerAddress +export GAME_FACTORY_ADDRESS=0xYourGameFactoryAddress + +# Start the container +docker-compose up -d + +# View logs +docker-compose logs -f op-dispute-mon +``` + +#### Verify it's running + +Check that the metrics endpoint is accessible: + +```bash +# Query metrics endpoint +curl http://localhost:7300/metrics + +# You should see Prometheus metrics including: +# - op_dispute_mon_games_agreement +# - op_dispute_mon_ignored_games +# - and many more... +``` + +Check logs for any errors: + +```bash +# View recent logs +docker-compose logs --tail=50 op-dispute-mon +``` + +You should see logs indicating the monitor is tracking games: + +```bash +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting op-dispute-mon" version=untagged-48e52dc5-1759893117 +t=2025-10-21T05:58:52+0100 lvl=info msg="started metrics server" addr=[::]:7300 +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting scheduler" +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting monitoring" +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting game monitor" +t=2025-10-21T05:58:52+0100 lvl=info msg="Dispute monitor game service start completed" +t=2025-10-21T05:59:03+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=599.887083ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:12+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=134.839417ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:22+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=104.085292ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:32+0100 lvl=info msg="Completed monitoring update" blockNumber=9456909 blockHash=0x18b6a5ef128058c4dd6c92ee2a2e4484a58c24adf64c9ce2763812c581814ab3 duration=111.325667ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:42+0100 lvl=info msg="Completed monitoring update" blockNumber=9456910 blockHash=0xb38f14c19116bbef056c2194ca966940985a7c6a73b4eb7e765c401a84f5f269 duration=97.361917ms games=0 ignored=0 failed=0 +``` + -First, build op-dispute-mon from source: +### Build from Source + +Building from source gives you more control and is useful for development or custom deployments. + +#### Clone and build op-dispute-mon ```bash # From your project root @@ -169,7 +217,39 @@ make op-dispute-mon cd ../.. ``` -Then create the environment file: +#### Create directory structure + +```bash +# From your project root +cd rollup +mkdir -p dispute-mon/scripts +cd dispute-mon +``` + +#### Get required addresses + +```bash +# Get DisputeGameFactory address from deployment +export GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' ../deployer/.deployer/state.json) +echo "Game Factory Address: $GAME_FACTORY_ADDRESS" + +# Get Proposer address (from your proposer setup) +source ../proposer/.env +export PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' ../deployer/.deployer/state.json) +echo "Proposer Address: $PROPOSER_ADDRESS" + +# Get Challenger address (from your challenger setup) +source ../challenger/.env +export CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' ../deployer/.deployer/state.json) +echo "Challenger Address: $CHALLENGER_ADDRESS" + +# Get L1 RPC URL +source ../.env +echo "L1 RPC: $L1_RPC_URL" + +``` + +#### Create environment file ```bash cat > .env << EOF @@ -203,7 +283,9 @@ METRICS_PORT=7300 EOF ``` -Now create the startup script `scripts/start-dispute-mon.sh`: +#### Create startup script + +Create `scripts/start-dispute-mon.sh`: ```bash #!/bin/bash @@ -256,93 +338,20 @@ Make it executable: chmod +x scripts/start-dispute-mon.sh ``` - - - - - - - -Launch the dispute monitor using your chosen deployment method. - - - - -```bash -# Export variables -export PROPOSER_ADDRESS=0xYourProposerAddress -export CHALLENGER_ADDRESS=0xYourChallengerAddress -export GAME_FACTORY_ADDRESS=0xYourGameFactoryAddress - -# Start the container -docker-compose up -d - -# View logs -docker-compose logs -f op-dispute-mon -``` - - - - +#### Start op-dispute-mon ```bash # Start in foreground (for testing) ./scripts/start-dispute-mon.sh -# Or start in background with logging -mkdir -p logs +# Or start in background ./scripts/start-dispute-mon.sh > logs/dispute-mon.log 2>&1 & # Save the process ID echo $! > dispute-mon.pid ``` - - - - - - - -Check that the dispute monitor is operational. - - - - -Check the metrics endpoint: - -```bash -# Query metrics endpoint -curl http://localhost:7300/metrics - -# You should see Prometheus metrics including: -# - op_dispute_mon_games_agreement -# - op_dispute_mon_ignored_games -# - and many more... -``` - -Check logs for any errors: - -```bash -# View recent logs -docker-compose logs --tail=50 op-dispute-mon -``` - -You should see logs indicating the monitor is tracking games: - -```bash -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting op-dispute-mon" version=untagged-48e52dc5-1759893117 -t=2025-10-21T05:58:52+0100 lvl=info msg="started metrics server" addr=[::]:7300 -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting scheduler" -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting monitoring" -t=2025-10-21T05:58:52+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 duration=599.887083ms games=0 ignored=0 failed=0 -``` - - - - - -Check if the process is running: +#### Verify it's running ```bash # Check if process is running @@ -364,82 +373,97 @@ tail -f logs/dispute-mon.log Once op-dispute-mon is running, it exposes several important metrics that help you monitor dispute game health. +### Core metrics + Access the metrics endpoint: ```bash curl http://localhost:7300/metrics | grep op_dispute_mon ``` -**Game agreement metrics:** +#### Game agreement metrics -The dispute monitor tracks games and categorizes them by completion status and agreement with your honest actors: - -``` -op_dispute_mon_games_agreement{completion="complete",result_correctness="correct",root_agreement="agree",status="agree_defender_wins"} 0 -op_dispute_mon_games_agreement{completion="in_progress",result_correctness="correct",root_agreement="agree",status="agree_defender_ahead"} 0 ``` +# Total number of games tracked +op_dispute_mon_games_agreement 25 -**What to monitor:** -- Any non-zero value for `root_agreement="disagree"` requires immediate investigation -- Games with `completion="in_progress"` are actively being disputed -- For a new testnet, all values being `0` is normal (no disputes yet) +# Games currently in progress +op_dispute_mon_games_agreement{completion="in_progress"} 3 -**Ignored games:** +# Completed games +op_dispute_mon_games_agreement{completion="complete"} 22 -``` -op_dispute_mon_ignored_games 0 +# Games where monitor agrees with honest actors +op_dispute_mon_games_agreement{agreement="agree"} 24 + +# Games where monitor disagrees +op_dispute_mon_games_agreement{agreement="disagree"} 1 ``` -This tracks games that don't meet your monitoring criteria (e.g., games not involving your honest actors). +**What to monitor:** +- If `agreement="disagree"` is non-zero, investigate immediately - this indicates a potential issue with your honest actors +- High number of `completion="in_progress"` games is normal during active periods +- All completed games should show `agreement="agree"` -**Failed games:** +#### Ignored games ``` -op_dispute_mon_failed_games 0 +# Games that don't meet monitoring criteria +op_dispute_mon_ignored_games 5 ``` -This tracks games that failed to be monitored due to errors. +Games may be ignored if: +- They don't involve your honest actors +- They use an unsupported game type +- They're outside the monitoring window -**Monitor status:** +#### Game status by type ``` -op_dispute_mon_up 1 +# Games by status +op_dispute_mon_games_status{status="challenger_wins"} 20 +op_dispute_mon_games_status{status="defender_wins"} 3 +op_dispute_mon_games_status{status="in_progress"} 2 ``` -This should always be `1` when the monitor is running properly. If `0`, the monitor hasn't fully initialized. +**What to monitor:** +- `defender_wins` should be rare (indicates your proposal was successfully defended) +- `challenger_wins` is normal (other proposers' incorrect proposals were challenged) +- `in_progress` shows active games -**Honest actor bonds:** +#### Honest actor metrics ``` -op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourAddress",state="pending"} 0 -op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourAddress",state="won"} 0 -op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourAddress",state="lost"} 0 +# Games involving your honest actors +op_dispute_mon_honest_actor_games{actor="0xYourProposer",role="proposer"} 15 +op_dispute_mon_honest_actor_games{actor="0xYourChallenger",role="challenger"} 10 ``` -This tracks the bonds posted by your proposer and challenger, and whether they were won or lost. +**What to monitor:** +- Your proposer should create games regularly +- Your challenger should participate when needed +- If either metric stops increasing, investigate your honest actors + +### Metric query examples -**Query examples:** +Here are some useful queries you can run: ```bash -# Check monitor status -curl -s http://localhost:7300/metrics | grep "op_dispute_mon_up" +# Count total games +curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement{" | grep -v "#" | awk '{sum+=$2} END {print sum}' -# Check for any games being monitored -curl -s http://localhost:7300/metrics | grep -E "op_dispute_mon_(games|ignored|failed)" +# Count in-progress games +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{completion="in_progress"}' | awk '{print $2}' -# View all game agreement metrics -curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement" +# Count disagreements (should be 0!) +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agreement="disagree"}' | awk '{print $2}' ``` - -These metrics are based on the actual op-dispute-mon v1.4.2 implementation. Values will be `0` for new testnets until dispute games are created. - - -Use these practices to ensure your dispute games are healthy. +### Warning signs **Immediate attention required:** - Any `agreement="disagree"` metrics From b55e9d2a465c9b9cb0de5662f9852af599c62763 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 07:36:04 +0100 Subject: [PATCH 12/17] refactor: enhance op-dispute-setup tutorial for clarity and usability - Streamlined the setup process for the op-dispute-mon, emphasizing Docker as the preferred deployment method. - Improved organization of steps, particularly in creating environment files and Docker Compose configurations. - Added detailed instructions for starting the monitor and verifying its operation, including metrics checks. - Clarified explanations of key metrics to aid users in monitoring dispute game health. - Removed redundant content to enhance overall readability and user experience. --- .../create-l2-rollup/op-dispute-setup.mdx | 153 +++++++++--------- 1 file changed, 77 insertions(+), 76 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index d329a040f..3acbee517 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -40,19 +40,12 @@ Before you begin, ensure you have: The monitor exposes Prometheus metrics that can be used to build dashboards and alerts. - - - -Choose your preferred deployment method: Docker (recommended) or build from source. - -### Docker Setup - -The Docker setup provides a containerized environment for running op-dispute-mon with minimal configuration. + -#### Create dispute-mon directory + ```bash # From your project root @@ -61,34 +54,47 @@ mkdir dispute-mon cd dispute-mon ``` -#### Get required addresses - -You need three key addresses for op-dispute-mon: + -1. **DisputeGameFactory address** (from your deployment) -2. **Proposer address** (your honest actor who creates proposals) -3. **Challenger address** (your honest actor who defends proposals) + ```bash # Get DisputeGameFactory address from deployment export GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' ../deployer/.deployer/state.json) echo "Game Factory Address: $GAME_FACTORY_ADDRESS" -# Get Proposer address (from your proposer setup) -source ../proposer/.env +# Get Proposer address export PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' ../deployer/.deployer/state.json) echo "Proposer Address: $PROPOSER_ADDRESS" -# Get Challenger address (from your challenger setup) -source ../challenger/.env +# Get Challenger address export CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' ../deployer/.deployer/state.json) echo "Challenger Address: $CHALLENGER_ADDRESS" +``` + + + + +Create a `.env` file with the addresses: + +```bash +cat > .env << EOF +# Contract Addresses +GAME_FACTORY_ADDRESS=$GAME_FACTORY_ADDRESS + +# Honest Actors +PROPOSER_ADDRESS=$PROPOSER_ADDRESS +CHALLENGER_ADDRESS=$CHALLENGER_ADDRESS + +# L1 RPC (replace with your actual RPC) +L1_RPC_URL=https://sepolia.infura.io/v3/YOUR_INFURA_KEY +EOF ``` -#### Create Docker Compose file + -Create `docker-compose.yml` with the following configuration: + ```yaml version: "3.8" @@ -99,9 +105,11 @@ services: container_name: dispute-mon ports: - "7300:7300" # Metrics port + env_file: + - .env environment: # L1 RPC Configuration - - OP_DISPUTE_MON_L1_ETH_RPC=https://sepolia.infura.io/v3/YOUR_INFURA_KEY + - OP_DISPUTE_MON_L1_ETH_RPC=${L1_RPC_URL} # Rollup RPC Configuration (your op-node) - OP_DISPUTE_MON_ROLLUP_RPC=http://op-node:8547 @@ -109,7 +117,7 @@ services: # Honest Actors (your proposer and challenger addresses) - OP_DISPUTE_MON_HONEST_ACTORS=${PROPOSER_ADDRESS},${CHALLENGER_ADDRESS} - # DisputeGameFactory Contract Address + # DisputeGameFactory Contract Address - OP_DISPUTE_MON_GAME_FACTORY_ADDRESS=${GAME_FACTORY_ADDRESS} # Monitoring Configuration @@ -137,67 +145,49 @@ networks: Replace `YOUR_INFURA_KEY` with your actual Infura API key or use another L1 RPC provider. Make sure your L1 RPC has sufficient rate limits. -#### Start op-dispute-mon + + + ```bash -# Export variables -export PROPOSER_ADDRESS=0xYourProposerAddress -export CHALLENGER_ADDRESS=0xYourChallengerAddress -export GAME_FACTORY_ADDRESS=0xYourGameFactoryAddress - -# Start the container +# Start the container (variables loaded from .env) docker-compose up -d # View logs docker-compose logs -f op-dispute-mon ``` -#### Verify it's running + -Check that the metrics endpoint is accessible: + ```bash # Query metrics endpoint curl http://localhost:7300/metrics -# You should see Prometheus metrics including: -# - op_dispute_mon_games_agreement -# - op_dispute_mon_ignored_games -# - and many more... -``` - -Check logs for any errors: - -```bash # View recent logs docker-compose logs --tail=50 op-dispute-mon ``` You should see logs indicating the monitor is tracking games: -```bash -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting op-dispute-mon" version=untagged-48e52dc5-1759893117 +``` +t=2025-10-21T05:58:52+0100 lvl=info msg="Starting op-dispute-mon" t=2025-10-21T05:58:52+0100 lvl=info msg="started metrics server" addr=[::]:7300 -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting scheduler" -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting monitoring" -t=2025-10-21T05:58:52+0100 lvl=info msg="Starting game monitor" -t=2025-10-21T05:58:52+0100 lvl=info msg="Dispute monitor game service start completed" -t=2025-10-21T05:59:03+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=599.887083ms games=0 ignored=0 failed=0 -t=2025-10-21T05:59:12+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=134.839417ms games=0 ignored=0 failed=0 -t=2025-10-21T05:59:22+0100 lvl=info msg="Completed monitoring update" blockNumber=9456908 blockHash=0xcc85c7eaa6c5dc96bf26d1b9097fcdc94d744e545758bcfe9c109a6e443f3024 duration=104.085292ms games=0 ignored=0 failed=0 -t=2025-10-21T05:59:32+0100 lvl=info msg="Completed monitoring update" blockNumber=9456909 blockHash=0x18b6a5ef128058c4dd6c92ee2a2e4484a58c24adf64c9ce2763812c581814ab3 duration=111.325667ms games=0 ignored=0 failed=0 -t=2025-10-21T05:59:42+0100 lvl=info msg="Completed monitoring update" blockNumber=9456910 blockHash=0xb38f14c19116bbef056c2194ca966940985a7c6a73b4eb7e765c401a84f5f269 duration=97.361917ms games=0 ignored=0 failed=0 +t=2025-10-21T05:59:03+0100 lvl=info msg="Completed monitoring update" games=0 ignored=0 failed=0 ``` + + + + -### Build from Source - -Building from source gives you more control and is useful for development or custom deployments. + -#### Clone and build op-dispute-mon + ```bash # From your project root @@ -214,10 +204,12 @@ make op-dispute-mon ./bin/op-dispute-mon --version # Return to project root -cd ../.. +cd../.. ``` -#### Create directory structure + + + ```bash # From your project root @@ -226,30 +218,31 @@ mkdir -p dispute-mon/scripts cd dispute-mon ``` -#### Get required addresses + + + ```bash # Get DisputeGameFactory address from deployment export GAME_FACTORY_ADDRESS=$(jq -r '.opChainDeployments[0].DisputeGameFactoryProxy' ../deployer/.deployer/state.json) echo "Game Factory Address: $GAME_FACTORY_ADDRESS" -# Get Proposer address (from your proposer setup) -source ../proposer/.env +# Get Proposer address export PROPOSER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.proposer' ../deployer/.deployer/state.json) echo "Proposer Address: $PROPOSER_ADDRESS" -# Get Challenger address (from your challenger setup) -source ../challenger/.env +# Get Challenger address export CHALLENGER_ADDRESS=$(jq -r '.appliedIntent.chains[0].roles.challenger' ../deployer/.deployer/state.json) echo "Challenger Address: $CHALLENGER_ADDRESS" # Get L1 RPC URL source ../.env echo "L1 RPC: $L1_RPC_URL" - ``` -#### Create environment file + + + ```bash cat > .env << EOF @@ -283,9 +276,9 @@ METRICS_PORT=7300 EOF ``` -#### Create startup script + -Create `scripts/start-dispute-mon.sh`: + ```bash #!/bin/bash @@ -338,20 +331,25 @@ Make it executable: chmod +x scripts/start-dispute-mon.sh ``` -#### Start op-dispute-mon + + + ```bash # Start in foreground (for testing) ./scripts/start-dispute-mon.sh -# Or start in background +# Or start in background with logging +mkdir -p logs ./scripts/start-dispute-mon.sh > logs/dispute-mon.log 2>&1 & # Save the process ID echo $! > dispute-mon.pid ``` -#### Verify it's running + + + ```bash # Check if process is running @@ -364,12 +362,15 @@ curl http://localhost:7300/metrics tail -f logs/dispute-mon.log ``` + + + + - - + - +## Understanding key metrics Once op-dispute-mon is running, it exposes several important metrics that help you monitor dispute game health. @@ -381,7 +382,7 @@ Access the metrics endpoint: curl http://localhost:7300/metrics | grep op_dispute_mon ``` -#### Game agreement metrics +**Game agreement metrics:** ``` # Total number of games tracked @@ -405,7 +406,7 @@ op_dispute_mon_games_agreement{agreement="disagree"} 1 - High number of `completion="in_progress"` games is normal during active periods - All completed games should show `agreement="agree"` -#### Ignored games +**Ignored games:** ``` # Games that don't meet monitoring criteria @@ -417,7 +418,7 @@ Games may be ignored if: - They use an unsupported game type - They're outside the monitoring window -#### Game status by type +**Game status by type:** ``` # Games by status From 5712f873afcd0d7edca872524c476ea23a1c17a0 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 07:40:26 +0100 Subject: [PATCH 13/17] refactor: remove redundant steps in op-dispute-setup tutorial - Eliminated unnecessary closing tags and steps to streamline the tutorial. - Improved overall clarity and flow of the content for better user experience. --- .../tutorials/create-l2-rollup/op-dispute-setup.mdx | 7 ------- 1 file changed, 7 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index 3acbee517..4ed4918a7 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -460,10 +460,6 @@ curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{com curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agreement="disagree"}' | awk '{print $2}' ``` - - - - ### Warning signs **Immediate attention required:** @@ -484,9 +480,6 @@ curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agr - Games completing within challenge period - Occasional `challenger_wins` (other proposals being challenged) - - - ## Next steps Now that you have dispute game monitoring set up, you can: From 901f215e494ff0695e02053e3682a4c0e453cf71 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 07:52:32 +0100 Subject: [PATCH 14/17] refactor: update op-dispute-setup tutorial for improved clarity and metric details - Changed title capitalization for consistency. - Enhanced the metrics section with clearer labels and descriptions for tracking dispute games. - Streamlined the monitoring instructions, emphasizing key metrics and their meanings. - Removed redundant content to improve overall readability and user experience. --- .../create-l2-rollup/op-dispute-setup.mdx | 118 ++++++------------ 1 file changed, 40 insertions(+), 78 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index 4ed4918a7..a698608b1 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -1,5 +1,5 @@ --- -title: Monitor your chain with Dispute Monitor +title: Monitor your chain with dispute monitor description: Learn how to set up op-dispute-mon to monitor dispute games and fault proof activity on your L2 rollup. --- @@ -374,112 +374,74 @@ tail -f logs/dispute-mon.log Once op-dispute-mon is running, it exposes several important metrics that help you monitor dispute game health. + +The metrics shown below are from the [official OP Stack Grafana dashboard](https://github.com/ethereum-optimism/optimism/blob/develop/op-dispute-mon/README.md) and represent actual production metrics used by Optimism. + + ### Core metrics -Access the metrics endpoint: +**Monitor health:** -```bash -curl http://localhost:7300/metrics | grep op_dispute_mon ``` - -**Game agreement metrics:** - +# Monitor is running (1 = up, 0 = down) +op_dispute_mon_up 1 ``` -# Total number of games tracked -op_dispute_mon_games_agreement 25 - -# Games currently in progress -op_dispute_mon_games_agreement{completion="in_progress"} 3 -# Completed games -op_dispute_mon_games_agreement{completion="complete"} 22 +**Game agreement metrics:** -# Games where monitor agrees with honest actors -op_dispute_mon_games_agreement{agreement="agree"} 24 +The primary metric for tracking dispute games, with labels for completion status, result correctness, root agreement, and game status: -# Games where monitor disagrees -op_dispute_mon_games_agreement{agreement="disagree"} 1 +``` +# Games by completion and status +op_dispute_mon_games_agreement{completion="complete",result_correctness="correct",root_agreement="agree",status="agree_defender_wins"} 0 +op_dispute_mon_games_agreement{completion="in_progress",result_correctness="correct",root_agreement="agree",status="agree_defender_ahead"} 0 +op_dispute_mon_games_agreement{completion="complete",result_correctness="incorrect",status="challenger_wins"} 0 ``` +**Label meanings:** +- `completion`: `"complete"` or `"in_progress"` +- `result_correctness`: `"correct"` or `"incorrect"` (based on your honest actors) +- `root_agreement`: `"agree"` or `"disagree"` (whether game agrees with honest actors) +- `status`: Game outcome like `"agree_defender_wins"`, `"challenger_wins"`, etc. + **What to monitor:** -- If `agreement="disagree"` is non-zero, investigate immediately - this indicates a potential issue with your honest actors -- High number of `completion="in_progress"` games is normal during active periods -- All completed games should show `agreement="agree"` +- Any non-zero `root_agreement="disagree"` requires immediate investigation +- `result_correctness="incorrect"` games should resolve to `"challenger_wins"` +- For new testnets, all values being `0` is normal until games are created **Ignored games:** ``` -# Games that don't meet monitoring criteria -op_dispute_mon_ignored_games 5 +# Games not meeting monitoring criteria +op_dispute_mon_ignored_games 0 ``` -Games may be ignored if: -- They don't involve your honest actors -- They use an unsupported game type -- They're outside the monitoring window +Games may be ignored if they don't involve your honest actors or fall outside the monitoring window. -**Game status by type:** +**Honest actor bonds:** ``` -# Games by status -op_dispute_mon_games_status{status="challenger_wins"} 20 -op_dispute_mon_games_status{status="defender_wins"} 3 -op_dispute_mon_games_status{status="in_progress"} 2 +# Bond status for your proposer and challenger +op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourProposer",state="won"} 0 +op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourProposer",state="lost"} 0 +op_dispute_mon_honest_actor_bonds{honest_actor_address="0xYourProposer",state="pending"} 0 ``` -**What to monitor:** -- `defender_wins` should be rare (indicates your proposal was successfully defended) -- `challenger_wins` is normal (other proposers' incorrect proposals were challenged) -- `in_progress` shows active games +Tracks bonds posted by your honest actors and their outcomes (`won`, `lost`, or `pending`). -#### Honest actor metrics +**Honest actor claims:** ``` -# Games involving your honest actors -op_dispute_mon_honest_actor_games{actor="0xYourProposer",role="proposer"} 15 -op_dispute_mon_honest_actor_games{actor="0xYourChallenger",role="challenger"} 10 +# Claims made by your honest actors +op_dispute_mon_honest_actor_claims{honest_actor_address="0xYourProposer",state="valid"} 0 +op_dispute_mon_honest_actor_claims{honest_actor_address="0xYourProposer",state="invalid"} 0 ``` -**What to monitor:** -- Your proposer should create games regularly -- Your challenger should participate when needed -- If either metric stops increasing, investigate your honest actors - -### Metric query examples - -Here are some useful queries you can run: - -```bash -# Count total games -curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement{" | grep -v "#" | awk '{sum+=$2} END {print sum}' - -# Count in-progress games -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{completion="in_progress"}' | awk '{print $2}' - -# Count disagreements (should be 0!) -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{agreement="disagree"}' | awk '{print $2}' +# Bond collateral tracking +op_dispute_mon_bond_collateral_available{balance="sufficient",delayedWETH="0x..."} 0 +op_dispute_mon_bond_collateral_required{balance="sufficient",delayedWETH="0x..."} 0 ``` -### Warning signs - -**Immediate attention required:** -- Any `agreement="disagree"` metrics -- Honest actor metrics stop increasing -- High number of ignored games involving your actors -- Errors in logs about RPC connectivity -- Errors about invalid game states - -**Investigate soon:** -- Games stuck in `in_progress` for >7 days -- Sudden drop in game creation rate -- Increase in `defender_wins` status - -**Normal operation:** -- Regular game creation -- All agreements showing `agree` -- Games completing within challenge period -- Occasional `challenger_wins` (other proposals being challenged) - ## Next steps Now that you have dispute game monitoring set up, you can: From 14d1b7423d15ae1a9cf9680e6b487b3db99af678 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 07:54:48 +0100 Subject: [PATCH 15/17] refactor: update op-dispute-setup tutorial with enhanced metrics and clarity - Capitalized the title for consistency. - Expanded the metrics section with additional commands for monitoring dispute games. - Improved the organization of the metrics information to enhance user understanding. - Streamlined content for better readability and user experience. --- .../create-l2-rollup/op-dispute-setup.mdx | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index a698608b1..513e834c0 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -1,5 +1,5 @@ --- -title: Monitor your chain with dispute monitor +title: Monitor your chain with Dispute Monitor description: Learn how to set up op-dispute-mon to monitor dispute games and fault proof activity on your L2 rollup. --- @@ -370,7 +370,7 @@ tail -f logs/dispute-mon.log -## Understanding key metrics +### Understanding key metrics Once op-dispute-mon is running, it exposes several important metrics that help you monitor dispute game health. @@ -380,6 +380,12 @@ The metrics shown below are from the [official OP Stack Grafana dashboard](https ### Core metrics +Access the metrics endpoint: + +```bash +curl http://localhost:7300/metrics | grep op_dispute_mon +``` + **Monitor health:** ``` @@ -437,11 +443,36 @@ op_dispute_mon_honest_actor_claims{honest_actor_address="0xYourProposer",state=" op_dispute_mon_honest_actor_claims{honest_actor_address="0xYourProposer",state="invalid"} 0 ``` + + # Bond collateral tracking op_dispute_mon_bond_collateral_available{balance="sufficient",delayedWETH="0x..."} 0 op_dispute_mon_bond_collateral_required{balance="sufficient",delayedWETH="0x..."} 0 ``` + +```bash +# Check monitor is up +curl -s http://localhost:7300/metrics | grep "op_dispute_mon_up" + +# Count all monitored games +curl -s http://localhost:7300/metrics | grep "^op_dispute_mon_games_agreement{" | awk '{sum+=$2} END {print sum}' + +# Count in-progress games +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{.*completion="in_progress".*}' | awk '{sum+=$2} END {print sum}' + +# Check for games with incorrect results (should investigate if > 0) +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{.*result_correctness="incorrect".*}' | awk '{sum+=$2} END {print sum}' + +# Check ignored games +curl -s http://localhost:7300/metrics | grep "^op_dispute_mon_ignored_games " | awk '{print $2}' + +# Check honest actor bonds lost (should be 0) +curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_honest_actor_bonds{.*state="lost".*}' | awk '{sum+=$2} END {print sum}' +``` + + + ## Next steps Now that you have dispute game monitoring set up, you can: From 65d4139445fdb1103461fec1e54c1e1ed2d0f210 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 07:58:16 +0100 Subject: [PATCH 16/17] refactor: enhance op-dispute-setup tutorial with additional metrics and clarity - Added explanations for tracking validity of claims and bond collateral. - Improved health check instructions with clearer commands and expected outputs. - Included a note for new testnet users regarding initial metric values. - Streamlined content for better readability and user experience. --- .../create-l2-rollup/op-dispute-setup.mdx | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index 513e834c0..66073b2c0 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -443,35 +443,38 @@ op_dispute_mon_honest_actor_claims{honest_actor_address="0xYourProposer",state=" op_dispute_mon_honest_actor_claims{honest_actor_address="0xYourProposer",state="invalid"} 0 ``` +Tracks validity of claims made by your proposer and challenger. `state="invalid"` should always be `0`. +**Bond collateral tracking:** -# Bond collateral tracking +``` +# Bond collateral available vs required op_dispute_mon_bond_collateral_available{balance="sufficient",delayedWETH="0x..."} 0 op_dispute_mon_bond_collateral_required{balance="sufficient",delayedWETH="0x..."} 0 ``` +Monitors whether your honest actors have sufficient collateral for bonds. -```bash -# Check monitor is up -curl -s http://localhost:7300/metrics | grep "op_dispute_mon_up" +### Quick health check -# Count all monitored games -curl -s http://localhost:7300/metrics | grep "^op_dispute_mon_games_agreement{" | awk '{sum+=$2} END {print sum}' +To verify your dispute monitor is working correctly: -# Count in-progress games -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{.*completion="in_progress".*}' | awk '{sum+=$2} END {print sum}' - -# Check for games with incorrect results (should investigate if > 0) -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_games_agreement{.*result_correctness="incorrect".*}' | awk '{sum+=$2} END {print sum}' +```bash +# Check monitor is running +curl -s http://localhost:7300/metrics | grep "op_dispute_mon_up" +# Expected: op_dispute_mon_up 1 -# Check ignored games -curl -s http://localhost:7300/metrics | grep "^op_dispute_mon_ignored_games " | awk '{print $2}' +# Check for any games (will be 0 for new testnet) +curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement" | head -5 -# Check honest actor bonds lost (should be 0) -curl -s http://localhost:7300/metrics | grep 'op_dispute_mon_honest_actor_bonds{.*state="lost".*}' | awk '{sum+=$2} END {print sum}' +# Check for any issues (should all be 0) +curl -s http://localhost:7300/metrics | grep 'state="lost"' +curl -s http://localhost:7300/metrics | grep 'state="invalid"' ``` - + +For a new testnet, it's normal for all game metrics to show `0` values. Games will only appear after your proposer creates proposals (typically every ~1 hour). + ## Next steps From 83b30b244822c252f820fd7c5716184e2e3a6032 Mon Sep 17 00:00:00 2001 From: Blessing Krofegha Date: Tue, 21 Oct 2025 08:02:58 +0100 Subject: [PATCH 17/17] refactor: simplify op-dispute-setup tutorial by removing outdated metrics commands - Removed unnecessary commands related to metrics endpoint checks and bond collateral tracking. - Streamlined the content to focus on essential monitoring instructions for users. - Enhanced clarity by eliminating redundant information, improving overall readability. --- .../create-l2-rollup/op-dispute-setup.mdx | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx index 66073b2c0..37096cc74 100644 --- a/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx +++ b/operators/chain-operators/tutorials/create-l2-rollup/op-dispute-setup.mdx @@ -380,12 +380,6 @@ The metrics shown below are from the [official OP Stack Grafana dashboard](https ### Core metrics -Access the metrics endpoint: - -```bash -curl http://localhost:7300/metrics | grep op_dispute_mon -``` - **Monitor health:** ``` @@ -443,35 +437,8 @@ op_dispute_mon_honest_actor_claims{honest_actor_address="0xYourProposer",state=" op_dispute_mon_honest_actor_claims{honest_actor_address="0xYourProposer",state="invalid"} 0 ``` -Tracks validity of claims made by your proposer and challenger. `state="invalid"` should always be `0`. - -**Bond collateral tracking:** - -``` -# Bond collateral available vs required -op_dispute_mon_bond_collateral_available{balance="sufficient",delayedWETH="0x..."} 0 -op_dispute_mon_bond_collateral_required{balance="sufficient",delayedWETH="0x..."} 0 -``` - Monitors whether your honest actors have sufficient collateral for bonds. -### Quick health check - -To verify your dispute monitor is working correctly: - -```bash -# Check monitor is running -curl -s http://localhost:7300/metrics | grep "op_dispute_mon_up" -# Expected: op_dispute_mon_up 1 - -# Check for any games (will be 0 for new testnet) -curl -s http://localhost:7300/metrics | grep "op_dispute_mon_games_agreement" | head -5 - -# Check for any issues (should all be 0) -curl -s http://localhost:7300/metrics | grep 'state="lost"' -curl -s http://localhost:7300/metrics | grep 'state="invalid"' -``` - For a new testnet, it's normal for all game metrics to show `0` values. Games will only appear after your proposer creates proposals (typically every ~1 hour).