feat(loadtest): per-transaction calldata from stdin for contract-call mode

cmd/loadtest/cmd.go

@@ -206,6 +206,8 @@ v3, uniswapv3 - perform UniswapV3 swaps`)
 	f.StringVar(&cfg.ContractAddress, "contract-address", "", "contract address for --mode contract-call (requires --calldata)")
 	f.StringVar(&cfg.ContractCallData, "calldata", "", "hex encoded calldata: function signature + encoded arguments (requires --mode contract-call and --contract-address)")
 	f.StringVar(&cfg.ContractCallDataFile, "calldata-file", "", "path to a file containing hex encoded calldata (alternative to --calldata; mutually exclusive with it)")
+	f.BoolVar(&cfg.ContractCallDataStdin, "calldata-stdin", false, "read raw calldata from stdin, one --calldata-size chunk per transaction; test stops at EOF (requires --mode contract-call, mutually exclusive with --calldata and --calldata-file)")
+	f.Uint64Var(&cfg.ContractCallDataSize, "calldata-size", 0, "bytes of stdin consumed as calldata for each transaction (requires --calldata-stdin)")
 	f.BoolVar(&cfg.ContractCallPayable, "contract-call-payable", false, "mark function as payable using value from --eth-amount-in-wei (requires --mode contract-call and --contract-address)")
 	f.StringVar(&cfg.Proxy, "proxy", "", "use the proxy specified")
 	f.BoolVar(&cfg.WaitForReceipt, "wait-for-receipt", false, "wait for transaction receipt to be mined instead of just sending")

cmd/loadtest/loadtestUsage.md

@@ -24,8 +24,9 @@ The `--mode` flag is important for this command.
 - `b`/`blob` will send EIP-4844 blob transactions. Use `--blob-fee-cap`
   to set the maximum blob fee per chunk.
 - `cc`/`contract-call` will call a specific contract function. Requires
-  `--contract-address` and either `--calldata` (hex string) or
-  `--calldata-file` (path to a file containing the hex calldata). Use
+  `--contract-address` and one of `--calldata` (hex string),
+  `--calldata-file` (path to a file containing the hex calldata), or
+  `--calldata-stdin` (raw bytes streamed from stdin, see below). Use
   `--contract-call-payable` if the function is payable.
 - `R`/`recall` will attempt to replay all of the transactions from the
   previous blocks. You can use `--recall-blocks` to specify how many
@@ -53,6 +54,45 @@ Here is a simple example that runs 1000 requests at a max rate of 1 request per
 $ polycli loadtest --verbosity 700 --chain-id 1256 --concurrency 1 --requests 1000 --rate-limit 1 --mode t --rpc-url http://localhost:8888
 ```
 
+### Per-Transaction Calldata from Stdin
+
+`--calldata` and `--calldata-file` send the same calldata in every
+transaction. `--calldata-stdin` instead reads raw bytes from stdin and
+gives each transaction its own `--calldata-size`-byte chunk, so a shell
+pipeline can generate a different payload per transaction. The test
+stops cleanly when stdin reaches EOF or when the request count is
+reached, whichever comes first.
+
+```bash
+$ for i in $(seq 1 10000); do
+    zstdcat receipt-addresses.txt.zst | shuf | head -n 1500 | sed 's/0x//' | tr -d '\n' | xxd -r -p
+  done | polycli loadtest --mode contract-call --contract-address 0x... \
+      --calldata-stdin --calldata-size 30000 --gas-limit 8000000 \
+      --requests 100000 --concurrency 4
+```
+
+Rules for the input:
+
+- Stdin is treated as raw bytes, not hex. Pipe through `xxd -r -p` or
+  similar to convert hex text into bytes.
+- Every chunk is exactly `--calldata-size` bytes. A trailing partial
+  chunk at EOF is discarded with a warning.
+- No function selector is added. Prepend one in the input if the target
+  function needs it.
+- `--calldata-stdin` requires `contract-call` to be the only mode and is
+  mutually exclusive with `--calldata`, `--calldata-file`, and
+  `--reverse-nonce-order`. Stdin must be a pipe or file, not a terminal,
+  and `--calldata-size` is capped at 1 MiB.
+- A read error on stdin, such as the producer dying mid-stream, stops the
+  test like EOF does but exits non-zero so a broken pipeline is not
+  mistaken for a clean finish.
+- Set `--gas-limit`. Without it every transaction is gas estimated,
+  adding one RPC call per send.
+- The producer is the throughput ceiling. If the pipeline generates
+  chunks slower than polycli can send them, the rate limiter is never the
+  binding constraint. Pregenerate to a file and redirect it if that
+  matters.
+
 ### Separate Broadcast Endpoint
 
 By default, all RPC calls (gas estimation, chain ID, nonces, receipts, and transaction broadcast) go to `--rpc-url`. The `--send-rpc-url` flag routes only the transaction broadcast (`eth_sendRawTransaction`, or `eth_sendRawTransactionPrivate` when combined with `--private-txs`) to a secondary endpoint while everything else, including account funding, stays on `--rpc-url`. This is useful for:

doc/polycli_loadtest.md

@@ -45,8 +45,9 @@ The `--mode` flag is important for this command.
 - `b`/`blob` will send EIP-4844 blob transactions. Use `--blob-fee-cap`
   to set the maximum blob fee per chunk.
 - `cc`/`contract-call` will call a specific contract function. Requires
-  `--contract-address` and either `--calldata` (hex string) or
-  `--calldata-file` (path to a file containing the hex calldata). Use
+  `--contract-address` and one of `--calldata` (hex string),
+  `--calldata-file` (path to a file containing the hex calldata), or
+  `--calldata-stdin` (raw bytes streamed from stdin, see below). Use
   `--contract-call-payable` if the function is payable.
 - `R`/`recall` will attempt to replay all of the transactions from the
   previous blocks. You can use `--recall-blocks` to specify how many
@@ -74,6 +75,45 @@ Here is a simple example that runs 1000 requests at a max rate of 1 request per
 $ polycli loadtest --verbosity 700 --chain-id 1256 --concurrency 1 --requests 1000 --rate-limit 1 --mode t --rpc-url http://localhost:8888
 ```
 
+### Per-Transaction Calldata from Stdin
+
+`--calldata` and `--calldata-file` send the same calldata in every
+transaction. `--calldata-stdin` instead reads raw bytes from stdin and
+gives each transaction its own `--calldata-size`-byte chunk, so a shell
+pipeline can generate a different payload per transaction. The test
+stops cleanly when stdin reaches EOF or when the request count is
+reached, whichever comes first.
+
+```bash
+$ for i in $(seq 1 10000); do
+    zstdcat receipt-addresses.txt.zst | shuf | head -n 1500 | sed 's/0x//' | tr -d '\n' | xxd -r -p
+  done | polycli loadtest --mode contract-call --contract-address 0x... \
+      --calldata-stdin --calldata-size 30000 --gas-limit 8000000 \
+      --requests 100000 --concurrency 4
+```
+
+Rules for the input:
+
+- Stdin is treated as raw bytes, not hex. Pipe through `xxd -r -p` or
+  similar to convert hex text into bytes.
+- Every chunk is exactly `--calldata-size` bytes. A trailing partial
+  chunk at EOF is discarded with a warning.
+- No function selector is added. Prepend one in the input if the target
+  function needs it.
+- `--calldata-stdin` requires `contract-call` to be the only mode and is
+  mutually exclusive with `--calldata`, `--calldata-file`, and
+  `--reverse-nonce-order`. Stdin must be a pipe or file, not a terminal,
+  and `--calldata-size` is capped at 1 MiB.
+- A read error on stdin, such as the producer dying mid-stream, stops the
+  test like EOF does but exits non-zero so a broken pipeline is not
+  mistaken for a clean finish.
+- Set `--gas-limit`. Without it every transaction is gas estimated,
+  adding one RPC call per send.
+- The producer is the throughput ceiling. If the pipeline generates
+  chunks slower than polycli can send them, the rate limiter is never the
+  binding constraint. Pregenerate to a file and redirect it if that
+  matters.
+
 ### Separate Broadcast Endpoint
 
 By default, all RPC calls (gas estimation, chain ID, nonces, receipts, and transaction broadcast) go to `--rpc-url`. The `--send-rpc-url` flag routes only the transaction broadcast (`eth_sendRawTransaction`, or `eth_sendRawTransactionPrivate` when combined with `--private-txs`) to a secondary endpoint while everything else, including account funding, stays on `--rpc-url`. This is useful for:
@@ -229,6 +269,8 @@ The codebase has a contract that used for load testing. It's written in Solidity
       --block-batch-size uint                            number of blocks to fetch per RPC batch request for recall and rpc modes (default 25)
       --calldata string                                  hex encoded calldata: function signature + encoded arguments (requires --mode contract-call and --contract-address)
       --calldata-file string                             path to a file containing hex encoded calldata (alternative to --calldata; mutually exclusive with it)
+      --calldata-size uint                               bytes of stdin consumed as calldata for each transaction (requires --calldata-stdin)
+      --calldata-stdin                                   read raw calldata from stdin, one --calldata-size chunk per transaction; test stops at EOF (requires --mode contract-call, mutually exclusive with --calldata and --calldata-file)
       --chain-id uint                                    chain ID for the transactions
       --check-balance-before-funding                     check account balance before funding sending accounts (saves gas when accounts are already funded)
       --check-preconf                                    check for preconf status after sending tx

loadtest/config/config.go

@@ -14,6 +14,7 @@ import (
 	"github.com/0xPolygon/polygon-cli/loadtest/uniswapv3"
 	"github.com/0xPolygon/polygon-cli/util"
 	"github.com/ethereum/go-ethereum/common"
+	"golang.org/x/term"
 )
 
 // Mode represents the type of load test to perform.
@@ -36,6 +37,11 @@ const (
 	ModeUniswapV3
 )
 
+// MaxContractCallDataSize caps --calldata-size. No known chain accepts a
+// transaction anywhere near this large, and each chunk is allocated in full,
+// so the cap mostly guards against typos exhausting memory.
+const MaxContractCallDataSize = 1 << 20
+
 // Config holds all load test parameters.
 type Config struct {
 	// Network connection
@@ -111,8 +117,13 @@ type Config struct {
 	ContractAddress      string
 	ContractCallData     string
 	ContractCallDataFile string
-	ContractCallPayable  bool
-	BlobFeeCap           uint64
+	// ContractCallDataStdin makes contract-call mode read raw calldata from
+	// stdin, one ContractCallDataSize-byte chunk per transaction, and stop
+	// the test at EOF.
+	ContractCallDataStdin bool
+	ContractCallDataSize  uint64
+	ContractCallPayable   bool
+	BlobFeeCap            uint64
 
 	// Account pool options
 	SendingAccountsCount      uint64
@@ -273,6 +284,29 @@ func (c *Config) Validate() error {
 		}
 	}
 
+	if c.ContractCallDataStdin {
+		if c.ContractCallData != "" || c.ContractCallDataFile != "" {
+			return errors.New("--calldata-stdin is mutually exclusive with --calldata and --calldata-file")
+		}
+		if c.ContractCallDataSize == 0 {
+			return errors.New("--calldata-stdin requires --calldata-size to be greater than zero")
+		}
+		if c.ContractCallDataSize > MaxContractCallDataSize {
+			return fmt.Errorf("--calldata-size %d exceeds the maximum of %d bytes", c.ContractCallDataSize, MaxContractCallDataSize)
+		}
+		if c.ReverseNonceOrder {
+			return errors.New("--calldata-stdin is incompatible with --reverse-nonce-order (stopping at EOF would leave the lowest planned nonces unsent, so nothing could ever mine)")
+		}
+		if err := c.validateSoleMode(ModeContractCall, "--calldata-stdin", "contract-call"); err != nil {
+			return err
+		}
+		if stdinIsTerminal() {
+			return errors.New("--calldata-stdin requires stdin to be a pipe or file, not a terminal")
+		}
+	} else if c.ContractCallDataSize != 0 {
+		return errors.New("--calldata-size requires --calldata-stdin")
+	}
+
 	if c.ContractCallDataFile != "" {
 		if c.ContractCallData != "" {
 			return errors.New("--calldata and --calldata-file are mutually exclusive")
@@ -336,6 +370,29 @@ func (c *Config) validateModesSupportRawSend(flagName string) error {
 	return nil
 }
 
+// validateSoleMode checks that the selected modes consist of exactly one mode
+// and that it is want. Used by flags that only make sense for a single mode
+// and cannot be shared with a mode list or random mode.
+func (c *Config) validateSoleMode(want Mode, flagName, modeName string) error {
+	if len(c.Modes) != 1 {
+		return fmt.Errorf("%s requires %s to be the only mode, got %d modes", flagName, modeName, len(c.Modes))
+	}
+	parsed, err := ParseMode(c.Modes[0])
+	if err != nil {
+		return fmt.Errorf("%s: %w", flagName, err)
+	}
+	if parsed != want {
+		return fmt.Errorf("%s requires --mode %s, got %q", flagName, modeName, c.Modes[0])
+	}
+	return nil
+}
+
+// stdinIsTerminal reports whether stdin is an interactive terminal rather
+// than a pipe or file. It is a variable so tests can override it.
+var stdinIsTerminal = func() bool {
+	return term.IsTerminal(int(os.Stdin.Fd()))
+}
+
 // Validate validates the UniswapV3Config and returns an error if any validation fails.
 func (c *UniswapV3Config) Validate() error {
 	switch fees := c.PoolFees; fees {

loadtest/config/config_test.go

@@ -384,3 +384,141 @@ func TestValidateReceiptPollInterval(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateCalldataStdin(t *testing.T) {
+	tests := []struct {
+		name     string
+		stdin    bool
+		size     uint64
+		calldata string
+		file     string
+		modes    []string
+		terminal bool
+		reverse  bool
+		wantErr  string
+	}{
+		{
+			name:  "valid with cc alias",
+			stdin: true,
+			size:  32,
+			modes: []string{"cc"},
+		},
+		{
+			name:  "valid with full mode name",
+			stdin: true,
+			size:  32,
+			modes: []string{"contract-call"},
+		},
+		{
+			name:    "stdin without size",
+			stdin:   true,
+			modes:   []string{"cc"},
+			wantErr: "--calldata-stdin requires --calldata-size",
+		},
+		{
+			name:    "size without stdin",
+			size:    32,
+			modes:   []string{"cc"},
+			wantErr: "--calldata-size requires --calldata-stdin",
+		},
+		{
+			name:     "stdin with calldata",
+			stdin:    true,
+			size:     32,
+			calldata: "0xdeadbeef",
+			modes:    []string{"cc"},
+			wantErr:  "--calldata-stdin is mutually exclusive with --calldata and --calldata-file",
+		},
+		{
+			name:    "stdin with calldata file",
+			stdin:   true,
+			size:    32,
+			file:    "calldata.hex",
+			modes:   []string{"cc"},
+			wantErr: "--calldata-stdin is mutually exclusive with --calldata and --calldata-file",
+		},
+		{
+			name:    "stdin with transaction mode",
+			stdin:   true,
+			size:    32,
+			modes:   []string{"t"},
+			wantErr: "--calldata-stdin requires --mode contract-call",
+		},
+		{
+			name:    "stdin with mode list",
+			stdin:   true,
+			size:    32,
+			modes:   []string{"cc", "t"},
+			wantErr: "--calldata-stdin requires contract-call to be the only mode",
+		},
+		{
+			name:    "stdin with random mode",
+			stdin:   true,
+			size:    32,
+			modes:   []string{"r"},
+			wantErr: "--calldata-stdin requires --mode contract-call",
+		},
+		{
+			name:     "stdin is a terminal",
+			stdin:    true,
+			size:     32,
+			modes:    []string{"cc"},
+			terminal: true,
+			wantErr:  "--calldata-stdin requires stdin to be a pipe or file",
+		},
+		{
+			name:    "size above cap",
+			stdin:   true,
+			size:    MaxContractCallDataSize + 1,
+			modes:   []string{"cc"},
+			wantErr: "exceeds the maximum",
+		},
+		{
+			name:  "size at cap",
+			stdin: true,
+			size:  MaxContractCallDataSize,
+			modes: []string{"cc"},
+		},
+		{
+			name:    "reverse nonce order",
+			stdin:   true,
+			size:    32,
+			modes:   []string{"cc"},
+			reverse: true,
+			wantErr: "--calldata-stdin is incompatible with --reverse-nonce-order",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			orig := stdinIsTerminal
+			stdinIsTerminal = func() bool { return tt.terminal }
+			t.Cleanup(func() { stdinIsTerminal = orig })
+
+			cfg := validConfig()
+			cfg.ContractCallDataStdin = tt.stdin
+			cfg.ContractCallDataSize = tt.size
+			cfg.ContractCallData = tt.calldata
+			cfg.ContractCallDataFile = tt.file
+			cfg.Modes = tt.modes
+			if tt.reverse {
+				cfg.ReverseNonceOrder = true
+				cfg.FireAndForget = true
+			}
+
+			err := cfg.Validate()
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("Validate() unexpected error: %v", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatalf("Validate() expected error containing %q, got nil", tt.wantErr)
+			}
+			if !strings.Contains(err.Error(), tt.wantErr) {
+				t.Fatalf("Validate() error %q does not contain %q", err.Error(), tt.wantErr)
+			}
+		})
+	}
+}