diff --git a/CHANGELOG.md b/CHANGELOG.md index 398701e7..fbe784d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to virtkit will be documented in this file. ## [Unreleased] +### Added + +- **DAX maps only the files worth mapping.** A share's window now serves regular files of + 1M and more by default (`dax=inode`); smaller files read through the guest page cache as + without DAX, so a source tree no longer spends the window's overhead on files too small to + benefit. `vk run --dax`, `x-virtkit.dax` and `[executor.vm] dax` take `:always` for + the previous behaviour and `:inode=` for another floor. + +### Changed + +- **Reading a large tree from a read-only share is faster.** A guest that reads many small + files — from an `:overlay` volume or the executor's host checkout — no longer pays a + per-file host cost that could dominate file-heavy jobs. + +### Fixed + +- **A stalled network transfer inside a guest now fails fast instead of hanging for the rest + of the job.** When the host was briefly overloaded a guest download could wedge and never + recover — a `cargo` fetch stuck at 0 bytes was the usual symptom. It now errors out + promptly and the application reconnects. +- **`host_checkout` jobs no longer stall on the first git command inside the guest.** The + job's first `git status` or `git checkout` now runs at full speed instead of stalling once + per job. + ## [0.68.0] - 2026-09-11 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 8fd8826e..b0a67729 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1860,8 +1860,6 @@ checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "ipstack" version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e889a45c1ce3e97268ad2249951528563ec1d02a120fecbd4db690beac34f7fd" dependencies = [ "ahash", "etherparse 0.20.3", diff --git a/Cargo.toml b/Cargo.toml index d5975e94..88d4340e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ members = [ # # third_party/imago is a single vendored leaf crate (see third_party/imago/VENDOR.md), a # dependency of third_party/libkrun's block device — excluded for the same reason. -exclude = ["third_party/libkrun", "third_party/imago"] +exclude = ["third_party/libkrun", "third_party/imago", "third_party/ipstack"] [workspace.package] version = "0.68.0" @@ -161,3 +161,9 @@ indicatif = "0.18" unicode-width = "0.2" # Use indicatif's transitive ANSI classifier in dashboard layout tests. console = { version = "0.16", default-features = false, features = ["ansi-parsing"] } + +# Vendored ipstack (third_party/ipstack/VENDOR.md): 1.0.1 plus one local patch — a TCP +# connection whose retransmissions are exhausted is reset instead of left Established with +# a hole in its stream. +[patch.crates-io] +ipstack = { path = "third_party/ipstack" } diff --git a/README.md b/README.md index dcc74b0b..15bc208f 100644 --- a/README.md +++ b/README.md @@ -283,9 +283,13 @@ in per-fault latency. The window reserves address space, not memory, and costs nothing until mapped. It defaults to 8G per share; `vk run --dax`, a service's `x-virtkit.dax` and the executor's `[executor.vm] dax` -resize it or turn it `off`. Each guest supports 64G of windows — eight at the default size, -with further shares served without DAX. Guests with more than 63.25G of RAM have no room -for windows and receive none. DAX requires the built-in VMM. Under +resize it or turn it `off`. Each mapping costs the host an mmap and the guest an EPT +invalidation per 2 MiB range whatever the file's size, so by default only regular files of +1M and more go through the window (`dax=inode`; the host marks them) and smaller files read +through the guest's page cache as without DAX; `:always` maps every file, and +`:inode=` moves the floor. Each guest supports 64G of windows — eight at the +default size, with further shares served without DAX. Guests with more than 63.25G of RAM +have no room for windows and receive none. DAX requires the built-in VMM. Under `VIRTKIT_VMM=cloud-hypervisor`, and for single-file binds and `vk build` stage guests, shares are served the ordinary way. diff --git a/docs/gitlab-ci.md b/docs/gitlab-ci.md index 25a6c0da..552b71cb 100644 --- a/docs/gitlab-ci.md +++ b/docs/gitlab-ci.md @@ -629,14 +629,20 @@ copy: ten concurrent job VMs reading a tools tree held eleven copies on the host ```toml [executor.vm] -dax = "8G" # per share; the default +dax = "8G" # per share; the default: files of 1M and more through the window +# dax = "8G:always" # every file through the window +# dax = "8G:inode=64K" # another size floor ``` The window reserves guest address space, not memory, and costs nothing until mapped. -`"off"` disables it. Each guest supports 64G of windows — eight at the default size, with -further shares served without DAX. Guests with more than 63.25G of RAM have no room for -windows and receive none. Mappings are 4 KiB-granular, so the benefit is avoiding a tools -tree's memory cost per VM, not per-fault latency. Compose services inherit the job VM's +`"off"` disables it. Each mapping costs the host an mmap and the guest an EPT invalidation +per 2 MiB range whatever the file's size, which a source tree's small files never repay: +by default only regular files of 1M and more are mapped (`dax=inode`; the host marks them), +the rest read through the guest page cache as without DAX. Each guest supports 64G of +windows — eight at the default size, with further shares served without DAX. Guests with +more than 63.25G of RAM have no room for windows and receive none. +Mappings are 4 KiB-granular, so the benefit is avoiding a tools tree's memory cost per +VM, not per-fault latency. Compose services inherit the job VM's setting unless they declare `x-virtkit.dax`. DAX requires the built-in VMM; cloud-hypervisor has no DAX path and serves shares the ordinary way whatever this says. diff --git a/third_party/ipstack/Cargo.lock b/third_party/ipstack/Cargo.lock new file mode 100644 index 00000000..02070cff --- /dev/null +++ b/third_party/ipstack/Cargo.lock @@ -0,0 +1,1455 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "c2rust-bitfields" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b43c3f07ab0ef604fa6f595aa46ec2f8a22172c975e186f6f5bf9829a3b72c41" +dependencies = [ + "c2rust-bitfields-derive", +] + +[[package]] +name = "c2rust-bitfields-derive" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3cbc102e2597c9744c8bd8c15915d554300601c91a079430d309816b0912545" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "etherparse" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeebdb565da33451794551a771bd0b4e6f416d3182adc3ed2837f614d6168f9a" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "ipstack" +version = "1.0.1" +dependencies = [ + "ahash", + "clap", + "criterion", + "dotenvy", + "env_logger", + "etherparse", + "log", + "rand", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tun", + "udp-stream", + "wintun", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tun" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb55e468c585e02ef89ba7a1a9848871ac942dfc64547ad4d1166b0f61a98be" +dependencies = [ + "bytes", + "cfg-if", + "futures", + "futures-core", + "ipnet", + "libc", + "log", + "nix", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "windows-sys 0.61.2", + "wintun-bindings", +] + +[[package]] +name = "udp-stream" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf021324a3dc10f5b46ab1c1cf5635e6a81b5559971967b806674673a5f2a18e" +dependencies = [ + "bytes", + "log", + "tokio", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.5", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wintun" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da99be64b5aa3de869c16977994314d0759a698d9a73ab0a5b1d52e2282033ae" +dependencies = [ + "c2rust-bitfields", + "libloading 0.8.9", + "log", + "thiserror 1.0.69", + "windows-sys 0.52.0", +] + +[[package]] +name = "wintun-bindings" +version = "0.7.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4316764300a7eb4aecf4770c81ae629ff1161b54905d48721a6832fd296ce5f" +dependencies = [ + "blocking", + "futures", + "libloading 0.9.0", + "log", + "thiserror 2.0.20", + "windows-sys 0.61.2", +] + +[[package]] +name = "zerocopy" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/third_party/ipstack/Cargo.toml b/third_party/ipstack/Cargo.toml new file mode 100644 index 00000000..39dc3e26 --- /dev/null +++ b/third_party/ipstack/Cargo.toml @@ -0,0 +1,129 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +name = "ipstack" +version = "1.0.1" +authors = ["Narrowlink "] +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Asynchronous lightweight userspace implementation of TCP/IP stack for Tun device" +homepage = "https://narrowlink.com" +readme = "README.md" +license = "Apache-2.0" +repository = "https://github.com/narrowlink/ipstack" + +[lib] +name = "ipstack" +path = "src/lib.rs" + +[[example]] +name = "tun" +path = "examples/tun.rs" + +[[example]] +name = "tun_wintun" +path = "examples/tun_wintun.rs" + +[dependencies.ahash] +version = "0.8" +features = ["std"] +default-features = false + +[dependencies.etherparse] +version = "0.20" +features = ["std"] +default-features = false + +[dependencies.log] +version = "0.4" +default-features = false + +[dependencies.rand] +version = "0.10" +features = ["thread_rng"] +default-features = false + +[dependencies.thiserror] +version = "2.0" +default-features = false + +[dependencies.tokio] +version = "1.52" +features = [ + "sync", + "rt", + "time", + "io-util", + "macros", + "rt-multi-thread", +] +default-features = false + +[dev-dependencies.clap] +version = "4.6" +features = ["derive"] +default-features = false + +[dev-dependencies.criterion] +version = "0.8" + +[dev-dependencies.dotenvy] +version = "0.15" + +[dev-dependencies.env_logger] +version = "0.11" + +[dev-dependencies.tokio] +version = "1.52" +features = ["rt-multi-thread"] +default-features = false + +[dev-dependencies.tun] +version = "0.8" +features = ["async"] +default-features = false + +[dev-dependencies.udp-stream] +version = "0.0" +default-features = false + +[build-dependencies.serde_json] +version = "1" + +[target.'cfg(target_os = "windows")'.dev-dependencies.wintun] +version = "0.5" +default-features = false + +[profile.bench] +opt-level = 3 +lto = false +codegen-units = 16 +debug = 2 +incremental = true +strip = "none" + +[profile.release] +opt-level = "s" +lto = true +codegen-units = 1 +debug = 0 +debug-assertions = false +rpath = false +panic = "abort" +overflow-checks = false +incremental = false +strip = true diff --git a/third_party/ipstack/LICENSE b/third_party/ipstack/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/third_party/ipstack/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/ipstack/README.md b/third_party/ipstack/README.md new file mode 100644 index 00000000..364463b8 --- /dev/null +++ b/third_party/ipstack/README.md @@ -0,0 +1,89 @@ +IpStack +======= + +An asynchronous lightweight userspace implementation of TCP/IP stack for Tun device. +Unstable, under development. + +[![Crates.io](https://img.shields.io/crates/v/ipstack.svg)](https://crates.io/crates/ipstack) +[![ipstack](https://docs.rs/ipstack/badge.svg)](https://docs.rs/ipstack) +[![Documentation](https://img.shields.io/badge/docs-release-brightgreen.svg?style=flat)](https://docs.rs/ipstack) +[![Download](https://img.shields.io/crates/d/ipstack.svg)](https://crates.io/crates/ipstack) +[![License](https://img.shields.io/crates/l/ipstack.svg?style=flat)](https://github.com/narrowlink/ipstack/blob/main/LICENSE) + +### Usage + +```rust, no_run +use etherparse::Icmpv4Header; +use ipstack::{IpNumber, IpStackStream}; +use std::net::{Ipv4Addr, SocketAddr}; +use tokio::{io::AsyncWriteExt, net::TcpStream}; +use udp_stream::UdpStream; + +#[tokio::main] +async fn main() -> Result<(), Box> { + const MTU: u16 = 1500; + let ipv4 = Ipv4Addr::new(10, 0, 0, 1); + let netmask = Ipv4Addr::new(255, 255, 255, 0); + let mut config = tun::Configuration::default(); + config.address(ipv4).netmask(netmask).mtu(MTU).up(); + + #[cfg(target_os = "linux")] + config.platform_config(|config| { + config.ensure_root_privileges(true); + }); + + #[cfg(target_os = "windows")] + config.platform_config(|config| { + config.device_guid(12324323423423434234_u128); + }); + + let mut ipstack_config = ipstack::IpStackConfig::default(); + ipstack_config.mtu(MTU)?; + let mut ip_stack = ipstack::IpStack::new(ipstack_config, tun::create_as_async(&config)?); + + while let Ok(stream) = ip_stack.accept().await { + match stream { + IpStackStream::Tcp(mut tcp) => { + let mut rhs = TcpStream::connect("1.1.1.1:80").await?; + tokio::spawn(async move { + let _ = tokio::io::copy_bidirectional(&mut tcp, &mut rhs).await; + let _ = rhs.shutdown().await; + let _ = tcp.shutdown().await; + }); + } + IpStackStream::Udp(mut udp) => { + let addr: SocketAddr = "1.1.1.1:53".parse()?; + let mut rhs = UdpStream::connect(addr).await?; + tokio::spawn(async move { + let _ = tokio::io::copy_bidirectional(&mut udp, &mut rhs).await; + rhs.shutdown(); + let _ = udp.shutdown().await; + }); + } + IpStackStream::UnknownTransport(u) => { + if u.src_addr().is_ipv4() && u.ip_protocol() == IpNumber::ICMP { + let (icmp_header, req_payload) = Icmpv4Header::from_slice(u.payload())?; + if let etherparse::Icmpv4Type::EchoRequest(echo) = icmp_header.icmp_type { + println!("ICMPv4 echo"); + let mut resp = Icmpv4Header::new(etherparse::Icmpv4Type::EchoReply(echo)); + resp.update_checksum(req_payload); + let mut payload = resp.to_bytes().to_vec(); + payload.extend_from_slice(req_payload); + u.send(payload)?; + } else { + println!("ICMPv4"); + } + continue; + } + println!("unknown transport - Ip Protocol {:?}", u.ip_protocol()); + } + IpStackStream::UnknownNetwork(pkt) => { + println!("unknown transport - {} bytes", pkt.len()); + } + } + } + Ok(()) +} +``` + +We also suggest that you take a look at the complete [examples](./examples). diff --git a/third_party/ipstack/VENDOR.md b/third_party/ipstack/VENDOR.md new file mode 100644 index 00000000..6faa3c20 --- /dev/null +++ b/third_party/ipstack/VENDOR.md @@ -0,0 +1,24 @@ +# Vendored ipstack + +Source: https://github.com/narrowlink/ipstack +Revision: `a343ea8c696e761acce8dbcd6687c862ecd8aacd` (crates.io 1.0.1) + +The crates.io 1.0.1 sources are vendored (`Cargo.toml`, `Cargo.lock`, `LICENSE`, `README.md`, +`build.rs`, `src/`); `examples/` and `scripts/` are dropped. `Cargo.lock` is kept for a standalone build +of this workspace-excluded crate. The root workspace's `[patch.crates-io]` points the +`ipstack` dependency here, so the switch (`vk-driver/src/switch.rs`) builds against this copy. + +## Local patches + ++ `src/stream/tcb.rs` + `src/stream/tcp.rs` — reset a connection whose retransmissions are + exhausted. Upstream drops the abandoned segment from the in-flight queue and leaves the + connection Established: the peer never receives those bytes, its duplicate ACKs can no + longer be served, and the application reads nothing for good. `collect_timed_out_inflight_packets` + now also reports the exhaustion; the TCP task then sends RST|ACK, moves to Closed and exits, + so the guest's socket errors out and the application can reconnect. Covered by + `exhausted_retransmissions_are_reported`. + +## Refreshing + +Copy the new crates.io sources over this directory, re-apply the patch above, update the +revision here. diff --git a/third_party/ipstack/build.rs b/third_party/ipstack/build.rs new file mode 100644 index 00000000..6ef1fc94 --- /dev/null +++ b/third_party/ipstack/build.rs @@ -0,0 +1,77 @@ +fn main() -> Result<(), Box> { + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_else(|_| "unknown".to_string()); + + if target_os == "windows" { + let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?); + let profile = std::env::var("PROFILE").unwrap_or_else(|_| "debug".to_string()); + + let Ok(cargo_target_dir) = extract_matching_parent_dir(&out_dir, &profile) else { + println!("cargo:warning=Could not find target directory"); + return Ok(()); + }; + // The wintun crate's root directory + let crate_dir = get_crate_dir("wintun")?; + + // The path to the DLL file, relative to the crate root, depending on the target architecture + let dll_path = get_wintun_bin_relative_path()?; + let src_path = crate_dir.join(dll_path); + + let dst_path = cargo_target_dir.join("examples/wintun.dll"); + + // Copy to the target directory + std::fs::copy(src_path, &dst_path)?; + + // Set the modified time to the current time, or the publishing process will fail. + let file = std::fs::OpenOptions::new().write(true).open(&dst_path)?; + file.set_modified(std::time::SystemTime::now())?; + } + Ok(()) +} + +fn extract_matching_parent_dir>(path: P, match_name: &str) -> std::io::Result { + let target_dir = std::path::Path::new(path.as_ref()) + .ancestors() + .find(|p| p.file_name().map(|n| *n == *match_name).unwrap_or(false)) + .ok_or(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("No parent directory matching '{match_name}'"), + ))?; + Ok(target_dir.to_path_buf()) +} + +fn get_wintun_bin_relative_path() -> Result> { + let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH")?; + + let dll_path = match target_arch.as_str() { + "x86" => "wintun/bin/x86/wintun.dll", + "x86_64" => "wintun/bin/amd64/wintun.dll", + "arm" => "wintun/bin/arm/wintun.dll", + "aarch64" => "wintun/bin/arm64/wintun.dll", + _ => return Err("Unsupported architecture".into()), + }; + + Ok(dll_path.into()) +} + +fn get_crate_dir(crate_name: &str) -> Result> { + let output = std::process::Command::new("cargo") + .arg("metadata") + .arg("--format-version=1") + .output()?; + + let metadata = serde_json::from_slice::(&output.stdout)?; + let packages = metadata["packages"].as_array().ok_or("packages")?; + + let mut crate_dir = None; + + for package in packages { + let name = package["name"].as_str().ok_or("name")?; + if name == crate_name { + let path = package["manifest_path"].as_str().ok_or("manifest_path")?; + let path = std::path::PathBuf::from(path); + crate_dir = Some(path.parent().ok_or("parent")?.to_path_buf()); + break; + } + } + Ok(crate_dir.ok_or("crate_dir")?) +} diff --git a/third_party/ipstack/src/error.rs b/third_party/ipstack/src/error.rs new file mode 100644 index 00000000..643f637b --- /dev/null +++ b/third_party/ipstack/src/error.rs @@ -0,0 +1,71 @@ +/// Error types for the IP stack. +/// +/// This enum represents all possible errors that can occur when working with the IP stack. +#[derive(thiserror::Error, Debug)] +pub enum IpStackError { + /// The transport protocol is not supported. + #[error("The transport protocol is not supported")] + UnsupportedTransportProtocol, + + /// The packet is invalid or malformed. + #[error("The packet is invalid")] + InvalidPacket, + + /// A value is too large to fit in a u16. + #[error("ValueTooBigError {0}")] + ValueTooBigErrorU16(#[from] etherparse::err::ValueTooBigError), + + /// A value is too large to fit in a usize. + #[error("ValueTooBigError {0}")] + ValueTooBigErrorUsize(#[from] etherparse::err::ValueTooBigError), + + /// The TCP packet is invalid. + #[error("Invalid Tcp packet")] + InvalidTcpPacket, + + /// An I/O error occurred. + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), + + /// Error accepting a new stream. + #[error("Accept Error")] + AcceptError, + + /// Error sending data through a channel. + #[error("Send Error {0}")] + SendError(#[from] Box>), + + /// Invalid MTU size. The minimum MTU is 1280 bytes to comply with IPv6 standards. + #[error("Invalid MTU size: {0} (bytes). Minimum MTU is 1280 bytes.")] + InvalidMtuSize(u16), +} + +impl From> for IpStackError { + fn from(e: tokio::sync::mpsc::error::SendError) -> Self { + IpStackError::SendError(Box::new(e)) + } +} + +// Safety: All variants of IpStackError either contain no data or wrap types that are `Send`. +// This ensures that IpStackError as a whole is safe to send between threads. +unsafe impl Send for IpStackError {} + +// Safety: All variants of IpStackError either contain no data or wrap types that are `Sync`. +// This ensures that IpStackError as a whole is safe to share between threads. +unsafe impl Sync for IpStackError {} + +impl From for std::io::Error { + fn from(e: IpStackError) -> Self { + match e { + IpStackError::IoError(e) => e, + _ => std::io::Error::other(e), + } + } +} + +/// A specialized [`Result`] type for IP stack operations. +/// +/// This type is used throughout the IP stack for any operation which may produce an error. +/// +/// [`Result`]: std::result::Result +pub type Result = std::result::Result; diff --git a/third_party/ipstack/src/lib.rs b/third_party/ipstack/src/lib.rs new file mode 100644 index 00000000..e67072af --- /dev/null +++ b/third_party/ipstack/src/lib.rs @@ -0,0 +1,441 @@ +#![doc = include_str!("../README.md")] + +use ahash::AHashMap; +use packet::{NetworkPacket, NetworkTuple, TransportHeader}; +use std::{sync::Arc, time::Duration}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, + select, + sync::mpsc::{self, UnboundedReceiver, UnboundedSender}, + task::JoinHandle, +}; + +pub(crate) type PacketSender = UnboundedSender; +pub(crate) type PacketReceiver = UnboundedReceiver; +pub(crate) type SessionCollection = AHashMap; + +mod error; +mod packet; +mod stream; + +pub use self::error::{IpStackError, Result}; +pub use self::stream::{IpStackStream, IpStackTcpStream, IpStackUdpStream, IpStackUnknownTransport}; +pub use self::stream::{TcpConfig, TcpOptions}; +pub use etherparse::IpNumber; + +#[cfg(unix)] +const TTL: u8 = 64; + +#[cfg(windows)] +const TTL: u8 = 128; + +#[cfg(unix)] +const TUN_FLAGS: [u8; 2] = [0x00, 0x00]; + +#[cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd", target_os = "espidf"))] +const TUN_PROTO_IP6: [u8; 2] = [0x86, 0xdd]; +#[cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd", target_os = "espidf"))] +const TUN_PROTO_IP4: [u8; 2] = [0x08, 0x00]; + +#[cfg(any(target_os = "macos", target_os = "ios"))] +const TUN_PROTO_IP6: [u8; 2] = [0x00, 0x0A]; +#[cfg(any(target_os = "macos", target_os = "ios"))] +const TUN_PROTO_IP4: [u8; 2] = [0x00, 0x02]; + +/// Minimum MTU required for IPv6 (per RFC 8200 §5: MTU ≥ 1280). +/// Also satisfies IPv4 minimum MTU (RFC 791 §3.1: 68 bytes). +const MIN_MTU: u16 = 1280; + +/// Configuration for the IP stack. +/// +/// This structure holds configuration parameters that control the behavior of the IP stack, +/// including network settings and protocol-specific timeouts. +/// +/// # Examples +/// +/// ``` +/// use ipstack::IpStackConfig; +/// use std::time::Duration; +/// +/// let mut config = IpStackConfig::default(); +/// config.mtu(1500).expect("Failed to set MTU") +/// .udp_timeout(Duration::from_secs(60)) +/// .packet_information(false); +/// ``` +#[non_exhaustive] +pub struct IpStackConfig { + /// Maximum Transmission Unit (MTU) size in bytes. + /// Default is `MIN_MTU` (1280). + pub mtu: u16, + /// Whether to include packet information headers (Unix platforms only). + /// Default is `false`. + pub packet_information: bool, + /// TCP-specific configuration parameters. + pub tcp_config: Arc, + /// Timeout for UDP connections. + /// Default is 30 seconds. + pub udp_timeout: Duration, +} + +impl Default for IpStackConfig { + fn default() -> Self { + IpStackConfig { + mtu: MIN_MTU, + packet_information: false, + tcp_config: Arc::new(TcpConfig::default()), + udp_timeout: Duration::from_secs(30), + } + } +} + +impl IpStackConfig { + /// Set custom TCP configuration. + /// + /// # Arguments + /// + /// * `config` - The TCP configuration to use + /// + /// # Examples + /// + /// ``` + /// use ipstack::{IpStackConfig, TcpConfig}; + /// + /// let mut config = IpStackConfig::default(); + /// config.with_tcp_config(TcpConfig::default()); + /// ``` + pub fn with_tcp_config(&mut self, config: TcpConfig) -> &mut Self { + self.tcp_config = Arc::new(config); + self + } + + /// Set the UDP connection timeout. + /// + /// # Arguments + /// + /// * `timeout` - The timeout duration for UDP connections + /// + /// # Examples + /// + /// ``` + /// use ipstack::IpStackConfig; + /// use std::time::Duration; + /// + /// let mut config = IpStackConfig::default(); + /// config.udp_timeout(Duration::from_secs(60)); + /// ``` + pub fn udp_timeout(&mut self, timeout: Duration) -> &mut Self { + self.udp_timeout = timeout; + self + } + + /// Set the Maximum Transmission Unit (MTU) size. + /// + /// # Arguments + /// + /// * `mtu` - The MTU size in bytes + /// + /// # Examples + /// + /// ``` + /// use ipstack::IpStackConfig; + /// + /// let mut config = IpStackConfig::default(); + /// config.mtu(1500).expect("Failed to set MTU"); + /// ``` + pub fn mtu(&mut self, mtu: u16) -> Result<&mut Self, IpStackError> { + if mtu < MIN_MTU { + return Err(IpStackError::InvalidMtuSize(mtu)); + } + self.mtu = mtu; + Ok(self) + } + + /// Set the Maximum Transmission Unit (MTU) size without validation. + pub fn mtu_unchecked(&mut self, mtu: u16) -> &mut Self { + self.mtu = mtu; + self + } + + /// Enable or disable packet information headers (Unix platforms only). + /// + /// When enabled on Unix platforms, the TUN device will include 4-byte packet + /// information headers. + /// + /// # Arguments + /// + /// * `packet_information` - Whether to include packet information headers + /// + /// # Examples + /// + /// ``` + /// use ipstack::IpStackConfig; + /// + /// let mut config = IpStackConfig::default(); + /// config.packet_information(true); + /// ``` + pub fn packet_information(&mut self, packet_information: bool) -> &mut Self { + self.packet_information = packet_information; + self + } +} + +/// The main IP stack instance. +/// +/// `IpStack` provides a userspace TCP/IP stack implementation for TUN devices. +/// It processes network packets and creates stream abstractions for TCP, UDP, and +/// unknown transport protocols. +/// +/// # Examples +/// +/// ```no_run +/// use ipstack::{IpStack, IpStackConfig, IpStackStream}; +/// use std::net::Ipv4Addr; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box> { +/// // Configure TUN device +/// let mut config = tun::Configuration::default(); +/// config +/// .address(Ipv4Addr::new(10, 0, 0, 1)) +/// .netmask(Ipv4Addr::new(255, 255, 255, 0)) +/// .up(); +/// +/// // Create IP stack +/// let ipstack_config = IpStackConfig::default(); +/// let mut ip_stack = IpStack::new(ipstack_config, tun::create_as_async(&config)?); +/// +/// // Accept incoming streams +/// while let Ok(stream) = ip_stack.accept().await { +/// match stream { +/// IpStackStream::Tcp(tcp) => { +/// // Handle TCP connection +/// } +/// IpStackStream::Udp(udp) => { +/// // Handle UDP connection +/// } +/// _ => {} +/// } +/// } +/// Ok(()) +/// } +/// ``` +pub struct IpStack { + accept_receiver: UnboundedReceiver, + handle: JoinHandle>, +} + +impl IpStack { + /// Create a new IP stack instance. + /// + /// # Arguments + /// + /// * `config` - Configuration for the IP stack + /// * `device` - An async TUN device implementing `AsyncRead` + `AsyncWrite` + /// + /// # Examples + /// + /// ```no_run + /// use ipstack::{IpStack, IpStackConfig}; + /// use std::net::Ipv4Addr; + /// + /// # async fn example() -> Result<(), Box> { + /// let mut tun_config = tun::Configuration::default(); + /// tun_config.address(Ipv4Addr::new(10, 0, 0, 1)) + /// .netmask(Ipv4Addr::new(255, 255, 255, 0)) + /// .up(); + /// + /// let ipstack_config = IpStackConfig::default(); + /// let ip_stack = IpStack::new(ipstack_config, tun::create_as_async(&tun_config)?); + /// # Ok(()) + /// # } + /// ``` + pub fn new(config: IpStackConfig, device: Device) -> IpStack + where + Device: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let (accept_sender, accept_receiver) = mpsc::unbounded_channel::(); + IpStack { + accept_receiver, + handle: run(config, device, accept_sender), + } + } + + /// Accept an incoming network stream. + /// + /// This method waits for and returns the next incoming network connection or packet. + /// The returned `IpStackStream` enum indicates the type of stream (TCP, UDP, or unknown). + /// + /// # Returns + /// + /// * `Ok(IpStackStream)` - The next incoming stream + /// * `Err(IpStackError::AcceptError)` - If the IP stack has been shut down + /// + /// # Examples + /// + /// ```no_run + /// use ipstack::{IpStack, IpStackConfig, IpStackStream}; + /// + /// # async fn example(mut ip_stack: IpStack) -> Result<(), Box> { + /// match ip_stack.accept().await? { + /// IpStackStream::Tcp(tcp) => { + /// println!("New TCP connection from {}", tcp.peer_addr()); + /// } + /// IpStackStream::Udp(udp) => { + /// println!("New UDP stream from {}", udp.peer_addr()); + /// } + /// IpStackStream::UnknownTransport(unknown) => { + /// println!("Unknown transport protocol: {:?}", unknown.ip_protocol()); + /// } + /// IpStackStream::UnknownNetwork(data) => { + /// println!("Unknown network packet: {} bytes", data.len()); + /// } + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn accept(&mut self) -> Result { + self.accept_receiver.recv().await.ok_or(IpStackError::AcceptError) + } +} + +impl Drop for IpStack { + fn drop(&mut self) { + self.handle.abort(); + } +} + +fn run( + config: IpStackConfig, + mut device: Device, + accept_sender: UnboundedSender, +) -> JoinHandle> { + let mut sessions: SessionCollection = AHashMap::new(); + let (session_remove_tx, mut session_remove_rx) = mpsc::unbounded_channel::(); + let pi = config.packet_information; + let offset = if pi && cfg!(unix) { 4 } else { 0 }; + let mut buffer = vec![0_u8; config.mtu as usize + offset]; + let (up_pkt_sender, mut up_pkt_receiver) = mpsc::unbounded_channel::(); + + tokio::spawn(async move { + loop { + select! { + Ok(n) = device.read(&mut buffer) => { + if let Err(e) = process_device_read(&buffer[offset..n], &mut sessions, &session_remove_tx, &up_pkt_sender, &config, &accept_sender).await { + let io_err: std::io::Error = e.into(); + if io_err.kind() == std::io::ErrorKind::ConnectionRefused { + log::trace!("Received junk data: {io_err}"); + } else { + log::warn!("process_device_read error: {io_err}"); + } + } + } + Some(network_tuple) = session_remove_rx.recv() => { + sessions.remove(&network_tuple); + log::debug!("session destroyed: {network_tuple}"); + } + Some(packet) = up_pkt_receiver.recv() => { + process_upstream_recv(packet, &mut device, #[cfg(unix)]pi).await?; + } + } + } + }) +} + +async fn process_device_read( + data: &[u8], + sessions: &mut SessionCollection, + session_remove_tx: &UnboundedSender, + up_pkt_sender: &PacketSender, + config: &IpStackConfig, + accept_sender: &UnboundedSender, +) -> Result<()> { + let Ok(packet) = NetworkPacket::parse(data) else { + let stream = IpStackStream::UnknownNetwork(data.to_owned()); + accept_sender.send(stream)?; + return Ok(()); + }; + + if let TransportHeader::Unknown = packet.transport_header() { + let stream = IpStackStream::UnknownTransport(IpStackUnknownTransport::new( + packet.src_addr().ip(), + packet.dst_addr().ip(), + packet.payload.unwrap_or_default(), + &packet.ip, + config.mtu, + up_pkt_sender.clone(), + )); + accept_sender.send(stream)?; + return Ok(()); + } + + let network_tuple = packet.network_tuple(); + match sessions.entry(network_tuple) { + std::collections::hash_map::Entry::Occupied(entry) => { + let len = packet.payload.as_ref().map(|p| p.len()).unwrap_or(0); + log::trace!("packet sent to stream: {network_tuple} len {len}"); + entry.get().send(packet).map_err(std::io::Error::other)?; + } + std::collections::hash_map::Entry::Vacant(entry) => { + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + let ip_stack_stream = create_stream(packet, config, up_pkt_sender.clone(), Some(tx))?; + let session_remove_tx = session_remove_tx.clone(); + tokio::spawn(async move { + rx.await.ok(); + if let Err(e) = session_remove_tx.send(network_tuple) { + log::error!("Failed to send session removal for {network_tuple}: {e}"); + } + }); + let packet_sender = ip_stack_stream.stream_sender()?; + accept_sender.send(ip_stack_stream)?; + entry.insert(packet_sender); + log::debug!("session created: {network_tuple}"); + } + } + Ok(()) +} + +fn create_stream( + packet: NetworkPacket, + cfg: &IpStackConfig, + up_pkt_sender: PacketSender, + msgr: Option<::tokio::sync::oneshot::Sender<()>>, +) -> Result { + let src_addr = packet.src_addr(); + let dst_addr = packet.dst_addr(); + match packet.transport_header() { + TransportHeader::Tcp(h) => { + let stream = IpStackTcpStream::new(src_addr, dst_addr, h.clone(), up_pkt_sender, cfg.mtu, msgr, cfg.tcp_config.clone())?; + Ok(IpStackStream::Tcp(stream)) + } + TransportHeader::Udp(_) => { + let payload = packet.payload.unwrap_or_default(); + let stream = IpStackUdpStream::new(src_addr, dst_addr, payload, up_pkt_sender, cfg.mtu, cfg.udp_timeout, msgr); + Ok(IpStackStream::Udp(stream)) + } + TransportHeader::Unknown => Err(IpStackError::UnsupportedTransportProtocol), + } +} + +async fn process_upstream_recv( + up_packet: NetworkPacket, + device: &mut Device, + #[cfg(unix)] packet_information: bool, +) -> Result<()> { + #[allow(unused_mut)] + let Ok(mut packet_bytes) = up_packet.to_bytes() else { + log::warn!("to_bytes error"); + return Ok(()); + }; + #[cfg(unix)] + if packet_information { + if up_packet.src_addr().is_ipv4() { + packet_bytes.splice(0..0, [TUN_FLAGS, TUN_PROTO_IP4].concat()); + } else { + packet_bytes.splice(0..0, [TUN_FLAGS, TUN_PROTO_IP6].concat()); + } + } + device.write_all(&packet_bytes).await?; + // device.flush().await?; + + Ok(()) +} diff --git a/third_party/ipstack/src/packet.rs b/third_party/ipstack/src/packet.rs new file mode 100644 index 00000000..2f38ec2b --- /dev/null +++ b/third_party/ipstack/src/packet.rs @@ -0,0 +1,304 @@ +use crate::error::IpStackError; +use etherparse::{Ipv4Header, Ipv6Header, NetSlice, SlicedPacket, TcpHeader, UdpHeader}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; + +#[derive(Eq, Hash, PartialEq, Debug, Clone, Copy)] +pub struct NetworkTuple { + pub src: SocketAddr, + pub dst: SocketAddr, + pub tcp: bool, +} + +impl NetworkTuple { + pub fn new(src: SocketAddr, dst: SocketAddr, tcp: bool) -> Self { + NetworkTuple { src, dst, tcp } + } +} + +impl std::fmt::Display for NetworkTuple { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let tcp = if self.tcp { "TCP" } else { "UDP" }; + write!(f, "{} {} -> {}", tcp, self.src, self.dst) + } +} + +pub mod tcp_flags { + pub const CWR: u8 = 0b10000000; + pub const ECE: u8 = 0b01000000; + pub const URG: u8 = 0b00100000; + pub const ACK: u8 = 0b00010000; + pub const PSH: u8 = 0b00001000; + pub const RST: u8 = 0b00000100; + pub const SYN: u8 = 0b00000010; + pub const FIN: u8 = 0b00000001; +} + +#[derive(Debug, Clone)] +pub(crate) enum IpHeader { + Ipv4(Ipv4Header), + Ipv6(Ipv6Header), +} + +#[derive(Debug, Clone)] +pub(crate) enum TransportHeader { + Tcp(TcpHeader), + Udp(UdpHeader), + Unknown, +} + +#[derive(Debug, Clone)] +pub struct NetworkPacket { + pub(crate) ip: IpHeader, + pub(crate) transport: TransportHeader, + pub(crate) payload: Option>, +} + +impl NetworkPacket { + pub fn parse(buf: &[u8]) -> Result { + let p = SlicedPacket::from_ip(buf).map_err(|_| IpStackError::InvalidPacket)?; + let ip = p.net.ok_or(IpStackError::InvalidPacket)?; + + let (ip, ip_payload) = match ip { + NetSlice::Ipv4(ip) => (IpHeader::Ipv4(ip.header().to_header()), ip.payload().payload), + NetSlice::Ipv6(ip) => (IpHeader::Ipv6(ip.header().to_header()), ip.payload().payload), + NetSlice::Arp(_) => return Err(IpStackError::UnsupportedTransportProtocol), + }; + let (transport, payload) = match p.transport { + Some(etherparse::TransportSlice::Tcp(h)) => (TransportHeader::Tcp(h.to_header()), h.payload()), + Some(etherparse::TransportSlice::Udp(u)) => (TransportHeader::Udp(u.to_header()), u.payload()), + _ => (TransportHeader::Unknown, ip_payload), + }; + let payload = if payload.is_empty() { None } else { Some(payload.to_vec()) }; + + Ok(NetworkPacket { ip, transport, payload }) + } + pub(crate) fn transport_header(&self) -> &TransportHeader { + &self.transport + } + pub fn src_addr(&self) -> SocketAddr { + let port = match &self.transport { + TransportHeader::Udp(udp) => udp.source_port, + TransportHeader::Tcp(tcp) => tcp.source_port, + _ => 0, + }; + match &self.ip { + IpHeader::Ipv4(ip) => SocketAddr::new(IpAddr::V4(Ipv4Addr::from(ip.source)), port), + IpHeader::Ipv6(ip) => SocketAddr::new(IpAddr::V6(Ipv6Addr::from(ip.source)), port), + } + } + pub fn dst_addr(&self) -> SocketAddr { + let port = match &self.transport { + TransportHeader::Udp(udp) => udp.destination_port, + TransportHeader::Tcp(tcp) => tcp.destination_port, + _ => 0, + }; + match &self.ip { + IpHeader::Ipv4(ip) => SocketAddr::new(IpAddr::V4(Ipv4Addr::from(ip.destination)), port), + IpHeader::Ipv6(ip) => SocketAddr::new(IpAddr::V6(Ipv6Addr::from(ip.destination)), port), + } + } + pub fn network_tuple(&self) -> NetworkTuple { + NetworkTuple { + src: self.src_addr(), + dst: self.dst_addr(), + tcp: matches!(self.transport, TransportHeader::Tcp(_)), + } + } + pub fn reverse_network_tuple(&self) -> NetworkTuple { + NetworkTuple { + src: self.dst_addr(), + dst: self.src_addr(), + tcp: matches!(self.transport, TransportHeader::Tcp(_)), + } + } + pub fn to_bytes(&self) -> Result, IpStackError> { + let mut buf = Vec::new(); + match self.ip { + IpHeader::Ipv4(ref ip) => ip.write(&mut buf)?, + IpHeader::Ipv6(ref ip) => ip.write(&mut buf)?, + } + match self.transport { + TransportHeader::Tcp(ref h) => h.write(&mut buf)?, + TransportHeader::Udp(ref h) => h.write(&mut buf)?, + _ => {} + }; + + if let Some(payload) = &self.payload { + buf.extend_from_slice(payload); + } + Ok(buf) + } + pub fn ttl(&self) -> u8 { + match &self.ip { + IpHeader::Ipv4(ip) => ip.time_to_live, + IpHeader::Ipv6(ip) => ip.hop_limit, + } + } +} + +pub fn tcp_header_fmt(header: &TcpHeader) -> String { + let mut flags = String::new(); + if header.cwr { + flags.push_str("CWR "); + } + if header.ece { + flags.push_str("ECE "); + } + if header.urg { + flags.push_str("URG "); + } + if header.ack { + flags.push_str("ACK "); + } + if header.psh { + flags.push_str("PSH "); + } + if header.rst { + flags.push_str("RST "); + } + if header.syn { + flags.push_str("SYN "); + } + if header.fin { + flags.push_str("FIN "); + } + format!( + "TcpHeader {{ seq: {}, ack: {}, flags: {} }}", + header.sequence_number, + header.acknowledgment_number, + flags.trim() + ) +} + +pub fn tcp_header_flags(inner: &TcpHeader) -> u8 { + let mut flags = 0; + if inner.cwr { + flags |= tcp_flags::CWR; + } + if inner.ece { + flags |= tcp_flags::ECE; + } + if inner.urg { + flags |= tcp_flags::URG; + } + if inner.ack { + flags |= tcp_flags::ACK; + } + if inner.psh { + flags |= tcp_flags::PSH; + } + if inner.rst { + flags |= tcp_flags::RST; + } + if inner.syn { + flags |= tcp_flags::SYN; + } + if inner.fin { + flags |= tcp_flags::FIN; + } + + flags +} + +// pub struct UdpPacket { +// header: UdpHeader, +// } + +// impl UdpPacket { +// pub fn inner(&self) -> &UdpHeader { +// &self.header +// } +// } + +// impl From<&UdpHeader> for UdpPacket { +// fn from(header: &UdpHeader) -> Self { +// UdpPacket { +// header: header.clone(), +// } +// } +// } + +#[cfg(test)] +pub mod tests { + use super::*; + use criterion::Criterion; + use rand::random; + use std::time::Duration; + + fn create_raw_packet(mtu: usize) -> Vec { + let builder = etherparse::PacketBuilder::ipv4(random(), random(), random()) + .tcp(random(), random(), random(), random()) + .fin() + .psh() + .ack(random()); + + let payload_len = mtu - builder.size(0); + assert_eq!(mtu, builder.size(payload_len)); + let payload: Vec = (0..payload_len).map(|_| random()).collect(); + + let mut buf = Vec::new(); + builder.write(&mut buf, &payload[..]).unwrap(); + assert_eq!(mtu, buf.len()); + buf + } + + fn create_packet(mtu: usize) -> NetworkPacket { + let packet = create_raw_packet(mtu); + NetworkPacket::parse(packet.as_slice()).unwrap() + } + + fn benchmarks(c: &mut Criterion) { + for mtu in [64, 1500, 4096, 16384, 65515] { + let buf = create_raw_packet(mtu); + c.bench_function(format!("decode_mtu_{mtu}").as_str(), |b| { + b.iter(|| { + let packet = std::hint::black_box(&buf[..]); + let _packet = NetworkPacket::parse(packet).unwrap(); + }) + }); + } + + for mtu in [64, 1500, 4096, 16384, 65515] { + let packet = create_packet(mtu); + c.bench_function(format!("encode_mtu_{mtu}").as_str(), |b| { + b.iter(|| { + let packet = std::hint::black_box(&packet); + let _packet = packet.to_bytes(); + }) + }); + } + } + + #[test] + fn bench() { + // `cargo test --profile bench -j1 -- --nocapture bench -- + // This workaround allows benchmarking private interfaces with `criterion` in stable rust. + let args: Vec = std::env::args().collect(); + let filter = args + .windows(3) + .filter(|p| p.len() >= 2 && p[0].ends_with("bench") && p[1] == "--") + .map(|s| s.get(2).unwrap_or(&"".to_string()).clone()) + .next(); + let filter = match filter { + Some(f) => f, + None => return, + }; + let profile_time = args + .windows(2) + .filter(|p| p.len() == 2 && p[0] == "--profile-time") + .map(|s| s[1].as_str()) + .next(); + + let mut c = Criterion::default() + .with_output_color(true) + .without_plots() + .with_filter(filter) + .warm_up_time(Duration::from_secs_f32(0.5)) + .measurement_time(Duration::from_secs_f32(0.5)) + .profile_time(profile_time.map(|s| Duration::from_secs_f32(s.parse().unwrap()))); + + benchmarks(&mut c); + + Criterion::default().final_summary(); + } +} diff --git a/third_party/ipstack/src/stream/mod.rs b/third_party/ipstack/src/stream/mod.rs new file mode 100644 index 00000000..777053f1 --- /dev/null +++ b/third_party/ipstack/src/stream/mod.rs @@ -0,0 +1,96 @@ +use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; + +pub use self::tcp::IpStackTcpStream; +pub use self::tcp::{TcpConfig, TcpOptions}; +pub use self::udp::IpStackUdpStream; +pub use self::unknown::IpStackUnknownTransport; + +mod seqnum; +mod tcb; +mod tcp; +mod udp; +mod unknown; + +/// A network stream accepted by the IP stack. +/// +/// This enum represents different types of network streams that can be accepted from the TUN device. +/// Each variant provides appropriate abstractions for handling specific protocol types. +/// +/// # Variants +/// +/// * `Tcp` - A TCP connection stream implementing `AsyncRead` + `AsyncWrite` +/// * `Udp` - A UDP stream implementing `AsyncRead` + `AsyncWrite` +/// * `UnknownTransport` - A stream for unknown transport layer protocols (e.g., ICMP, IGMP) +/// * `UnknownNetwork` - Raw network layer packets that couldn't be parsed +pub enum IpStackStream { + /// A TCP connection stream. + Tcp(IpStackTcpStream), + /// A UDP stream. + Udp(IpStackUdpStream), + /// A stream for unknown transport protocols. + UnknownTransport(IpStackUnknownTransport), + /// Raw network packets that couldn't be parsed. + UnknownNetwork(Vec), +} + +impl IpStackStream { + /// Returns the local socket address for this stream. + /// + /// For TCP and UDP streams, this returns the source address of the connection. + /// For unknown transport and network streams, this returns an unspecified address. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::{IpStack, IpStackStream}; + /// # async fn example(stream: IpStackStream) { + /// let local_addr = stream.local_addr(); + /// println!("Local address: {}", local_addr); + /// # } + /// ``` + pub fn local_addr(&self) -> SocketAddr { + match self { + IpStackStream::Tcp(tcp) => tcp.local_addr(), + IpStackStream::Udp(udp) => udp.local_addr(), + IpStackStream::UnknownNetwork(_) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)), + IpStackStream::UnknownTransport(unknown) => match unknown.src_addr() { + IpAddr::V4(addr) => SocketAddr::V4(SocketAddrV4::new(addr, 0)), + IpAddr::V6(addr) => SocketAddr::V6(SocketAddrV6::new(addr, 0, 0, 0)), + }, + } + } + + /// Returns the remote socket address for this stream. + /// + /// For TCP and UDP streams, this returns the destination address of the connection. + /// For unknown transport and network streams, this returns an unspecified address. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::{IpStack, IpStackStream}; + /// # async fn example(stream: IpStackStream) { + /// let peer_addr = stream.peer_addr(); + /// println!("Peer address: {}", peer_addr); + /// # } + /// ``` + pub fn peer_addr(&self) -> SocketAddr { + match self { + IpStackStream::Tcp(tcp) => tcp.peer_addr(), + IpStackStream::Udp(udp) => udp.peer_addr(), + IpStackStream::UnknownNetwork(_) => SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)), + IpStackStream::UnknownTransport(unknown) => match unknown.dst_addr() { + IpAddr::V4(addr) => SocketAddr::V4(SocketAddrV4::new(addr, 0)), + IpAddr::V6(addr) => SocketAddr::V6(SocketAddrV6::new(addr, 0, 0, 0)), + }, + } + } + + pub(crate) fn stream_sender(&self) -> Result { + match self { + IpStackStream::Tcp(tcp) => Ok(tcp.stream_sender()), + IpStackStream::Udp(udp) => Ok(udp.stream_sender()), + _ => Err(std::io::Error::other("Unknown transport stream does not have a sender")), + } + } +} diff --git a/third_party/ipstack/src/stream/seqnum.rs b/third_party/ipstack/src/stream/seqnum.rs new file mode 100644 index 00000000..9273a2f0 --- /dev/null +++ b/third_party/ipstack/src/stream/seqnum.rs @@ -0,0 +1,176 @@ +use std::ops::{Add, AddAssign, Sub, SubAssign}; + +const MAX_DIFF: u32 = u32::MAX / 2; + +/// A TCP sequence number that persents a 32-bit unsigned integer, suppport overflow comparison and arithmetic. +#[derive(Eq, PartialEq, Debug, Copy, Clone, Hash, Default)] +#[repr(transparent)] +pub struct SeqNum(pub u32); + +impl std::fmt::Display for SeqNum { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for SeqNum { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for u32 { + fn from(value: SeqNum) -> Self { + value.0 + } +} + +impl From for usize { + fn from(value: SeqNum) -> Self { + value.0 as usize + } +} + +impl TryFrom for SeqNum { + type Error = std::io::Error; + fn try_from(value: usize) -> Result { + if value > u32::MAX as usize { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("value 0x{value:X} is too large to convert to SeqNum"), + )); + } + Ok(Self(value as u32)) + } +} + +impl PartialEq for SeqNum { + fn eq(&self, other: &u32) -> bool { + self.0 == *other + } +} + +impl PartialOrd for SeqNum { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialOrd for SeqNum { + fn partial_cmp(&self, other: &u32) -> Option { + Some(self.cmp(&SeqNum(*other))) + } +} + +impl Ord for SeqNum { + #[inline] + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + let diff = self.0.wrapping_sub(other.0); + if diff == 0 { + std::cmp::Ordering::Equal + } else if diff < MAX_DIFF { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Less + } + } +} + +impl Add for SeqNum { + type Output = SeqNum; + #[inline] + fn add(self, rhs: Self) -> Self::Output { + SeqNum(self.0.wrapping_add(rhs.0)) + } +} + +impl Add for SeqNum { + type Output = SeqNum; + #[inline] + fn add(self, rhs: u32) -> Self::Output { + SeqNum(self.0.wrapping_add(rhs)) + } +} + +impl AddAssign for SeqNum { + fn add_assign(&mut self, rhs: Self) { + self.0 = self.0.wrapping_add(rhs.0) + } +} + +impl AddAssign for SeqNum { + fn add_assign(&mut self, rhs: u32) { + self.0 = self.0.wrapping_add(rhs) + } +} + +impl Sub for SeqNum { + type Output = SeqNum; + #[inline] + fn sub(self, rhs: Self) -> Self::Output { + SeqNum(self.0.wrapping_sub(rhs.0)) + } +} + +impl Sub for SeqNum { + type Output = SeqNum; + #[inline] + fn sub(self, rhs: u32) -> Self::Output { + SeqNum(self.0.wrapping_sub(rhs)) + } +} + +impl SubAssign for SeqNum { + fn sub_assign(&mut self, rhs: Self) { + self.0 = self.0.wrapping_sub(rhs.0) + } +} + +impl SubAssign for SeqNum { + fn sub_assign(&mut self, rhs: u32) { + self.0 = self.0.wrapping_sub(rhs) + } +} + +impl SeqNum { + pub fn distance(&self, other: Self) -> u32 { + let diff = self.0.wrapping_sub(other.0); + if diff <= MAX_DIFF { diff } else { u32::MAX - diff + 1 } + } +} + +#[test] +fn test_seq_num_near_overflow() { + let a: SeqNum = (u32::MAX - 3).into(); + let b = a + 8; + + assert_eq!(a, SeqNum(4294967292)); + assert_eq!(b, SeqNum(4)); + + assert!(a < b); + assert!(b > a); + assert!(a <= b); + assert!(b >= a); + assert!(a != b); + + assert_eq!(a.distance(b), 8); + assert_eq!(b.distance(a), 8); +} + +#[test] +fn test_seq_num_near_max_diff() { + let a = SeqNum(MAX_DIFF - 1); + let mut b = SeqNum(MAX_DIFF + 1); + + assert!(a < b); + assert!(b > a); + assert_eq!(a.distance(b), 2); + + b += 3; + assert_eq!(b.distance(a), 5); + + b -= 10; + assert_eq!(b.distance(a), 5); + + assert_eq!(b, SeqNum(MAX_DIFF - 6)); +} diff --git a/third_party/ipstack/src/stream/tcb.rs b/third_party/ipstack/src/stream/tcb.rs new file mode 100644 index 00000000..8d727241 --- /dev/null +++ b/third_party/ipstack/src/stream/tcb.rs @@ -0,0 +1,526 @@ +use super::seqnum::SeqNum; +use etherparse::TcpHeader; +use std::{collections::BTreeMap, time::Duration}; + +pub(super) const MAX_UNACK: u32 = 1024 * 16; // 16KB +pub(super) const READ_BUFFER_SIZE: usize = 1024 * 16; // 16KB +pub(super) const MAX_COUNT_FOR_DUP_ACK: usize = 3; // Maximum number of duplicate ACKs before retransmission + +/// Retransmission timeout +pub(super) const RTO: std::time::Duration = std::time::Duration::from_secs(1); + +/// Maximum count of retransmissions before dropping the packet +pub(super) const MAX_RETRANSMIT_COUNT: usize = 3; + +#[derive(Debug, PartialEq, Clone, Copy)] +pub(crate) enum TcpState { + // Init, /* Since we always act as a server, it starts from `Listen`, so we don't use states Init & SynSent. */ + // SynSent, + Listen, + SynReceived, + Established, + FinWait1, // act as a client, actively send a farewell packet to the other side, followed with FinWait2, TimeWait, Closed + FinWait2, + TimeWait, + CloseWait, // act as a server, followed with LastAck, Closed + LastAck, + Closed, +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub(super) enum PacketType { + WindowUpdate, + Invalid, + RetransmissionRequest, + NewPacket, + Ack, + KeepAlive, +} + +/// TCP Control Block +/// - `inflight_packets` is prerepresented bytes stream from upstream application, +/// which have been sent to the lower device but not yet acknowledged. +/// - `unordered_packets` is the bytes stream received from the lower device, +/// which can be acknowledged and extracted by `consume_unordered_packets` method +/// then can be read by upstream application via `Tcp::poll_read` method. +#[derive(Debug, Clone)] +pub(crate) struct Tcb { + seq: SeqNum, + ack: SeqNum, + mtu: u16, + last_received_ack: SeqNum, + send_window: u16, + state: TcpState, + inflight_packets: BTreeMap, + unordered_packets: BTreeMap>, + duplicate_ack_count: usize, + duplicate_ack_count_helper: SeqNum, + max_unacked_bytes: u32, + read_buffer_size: usize, + max_count_for_dup_ack: usize, + rto: std::time::Duration, + max_retransmit_count: usize, +} + +impl Tcb { + pub(super) fn new( + ack: SeqNum, + mtu: u16, + max_unacked_bytes: u32, + read_buffer_size: usize, + max_count_for_dup_ack: usize, + rto: std::time::Duration, + max_retransmit_count: usize, + ) -> Tcb { + #[cfg(debug_assertions)] + let seq = 100; + #[cfg(not(debug_assertions))] + let seq = rand::RngExt::random::(&mut rand::rng()); + Tcb { + seq: seq.into(), + ack, + mtu, + last_received_ack: seq.into(), + send_window: u16::MAX, + state: TcpState::Listen, + inflight_packets: BTreeMap::new(), + unordered_packets: BTreeMap::new(), + duplicate_ack_count: 0, + duplicate_ack_count_helper: seq.into(), + max_unacked_bytes, + read_buffer_size, + max_count_for_dup_ack, + rto, + max_retransmit_count, + } + } + + pub fn calculate_payload_max_len(&self, ip_header_size: usize, tcp_header_size: usize) -> usize { + let send_window = self.get_send_window() as usize; + let mtu = self.get_mtu() as usize; + std::cmp::min(send_window, mtu.saturating_sub(ip_header_size + tcp_header_size)) + } + + pub fn update_duplicate_ack_count(&mut self, rcvd_ack: SeqNum) { + // If the received rcvd_ack is the same as duplicate_ack_count_helper and not all data has been acknowledged (rcvd_ack < self.seq), increment the count. + if rcvd_ack == self.duplicate_ack_count_helper && rcvd_ack < self.seq { + self.duplicate_ack_count = self.duplicate_ack_count.saturating_add(1); + } else { + self.duplicate_ack_count_helper = rcvd_ack; + self.duplicate_ack_count = 0; // reset duplicate ACK count + } + } + + pub fn is_duplicate_ack_count_exceeded(&self) -> bool { + self.duplicate_ack_count >= self.max_count_for_dup_ack + } + + pub(super) fn add_unordered_packet(&mut self, seq: SeqNum, buf: Vec) { + if seq < self.ack { + #[rustfmt::skip] + log::warn!("{:?}: Received packet seq {seq} < self ack {}, len = {}", self.state, self.ack, buf.len()); + return; + } + self.unordered_packets.insert(seq, buf); + } + pub(super) fn get_available_read_buffer_size(&self) -> usize { + self.read_buffer_size.saturating_sub(self.get_unordered_packets_total_len()) + } + #[inline] + pub(crate) fn get_unordered_packets_total_len(&self) -> usize { + self.unordered_packets.values().map(|p| p.len()).sum() + } + + pub(super) fn consume_unordered_packets(&mut self, max_bytes: usize) -> Option> { + let mut data = Vec::new(); + let mut remaining_bytes = max_bytes; + + while remaining_bytes > 0 { + if let Some(seq) = self.unordered_packets.keys().next().copied() { + if seq != self.ack { + break; // sequence number is not continuous, stop extracting + } + + // remove and get the first packet + let mut payload = self.unordered_packets.remove(&seq).unwrap(); + let payload_len = payload.len(); + + if payload_len <= remaining_bytes { + // current packet can be fully extracted + data.extend(payload); + self.ack += payload_len as u32; + remaining_bytes -= payload_len; + } else { + // current packet can only be partially extracted + let remaining_payload = payload.split_off(remaining_bytes); + data.extend_from_slice(&payload); + self.ack += remaining_bytes as u32; + self.unordered_packets.insert(self.ack, remaining_payload); + break; + } + } else { + break; // no more packets to extract + } + } + + if data.is_empty() { None } else { Some(data) } + } + + pub(super) fn increase_seq(&mut self) { + self.seq += 1; + } + pub(super) fn get_seq(&self) -> SeqNum { + self.seq + } + pub(super) fn increase_ack(&mut self) { + self.ack += 1; + } + pub(super) fn get_ack(&self) -> SeqNum { + self.ack + } + pub(super) fn get_mtu(&self) -> u16 { + self.mtu + } + pub(super) fn get_last_received_ack(&self) -> SeqNum { + self.last_received_ack + } + pub(super) fn change_state(&mut self, state: TcpState) { + self.state = state; + } + pub(super) fn get_state(&self) -> TcpState { + self.state + } + pub(super) fn update_send_window(&mut self, window: u16) { + self.send_window = window; + } + pub(super) fn get_send_window(&self) -> u16 { + self.send_window + } + pub(super) fn get_recv_window(&self) -> u16 { + self.get_available_read_buffer_size().try_into().unwrap_or(u16::MAX) + } + // #[inline(always)] + // pub(super) fn buffer_size(&self, payload_len: u16) -> u16 { + // match MAX_UNACK - self.inflight_packets.len() as u32 { + // // b if b.saturating_sub(payload_len as u32 + 64) != 0 => payload_len, + // // b if b < 128 && b >= 4 => (b / 2) as u16, + // // b if b < 4 => b as u16, + // // b => (b - 64) as u16, + // b if b >= payload_len as u32 * 2 && b > 0 => payload_len, + // b if b < 4 => b as u16, + // b => (b / 2) as u16, + // } + // } + + pub(super) fn check_pkt_type(&self, tcp_header: &TcpHeader, payload: &[u8]) -> PacketType { + let rcvd_ack = SeqNum(tcp_header.acknowledgment_number); + let rcvd_seq = SeqNum(tcp_header.sequence_number); + let rcvd_window = tcp_header.window_size; + let len = payload.len(); + let res = if rcvd_ack > self.seq { + PacketType::Invalid + } else { + match rcvd_ack.cmp(&self.get_last_received_ack()) { + std::cmp::Ordering::Less => PacketType::Invalid, + std::cmp::Ordering::Equal => { + if self.ack - 1 == rcvd_seq && payload.len() <= 1 { + PacketType::KeepAlive + } else if !payload.is_empty() { + PacketType::NewPacket + } else if self.get_send_window() == rcvd_window && self.seq != rcvd_ack && self.is_duplicate_ack_count_exceeded() { + PacketType::RetransmissionRequest + } else { + PacketType::WindowUpdate + } + } + std::cmp::Ordering::Greater => { + if payload.is_empty() { + PacketType::Ack + } else { + PacketType::NewPacket + } + } + } + }; + #[rustfmt::skip] + log::trace!("received {{ ack = {:08X?}, seq = {:08X?}, window = {rcvd_window} }}, self {{ ack = {:08X?}, seq = {:08X?}, send_window = {} }}, len = {len}, {res:?}", rcvd_ack.0, rcvd_seq.0, self.ack.0, self.seq.0, self.get_send_window()); + res + } + + pub(super) fn add_inflight_packet(&mut self, buf: Vec) -> std::io::Result<()> { + if buf.is_empty() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Empty payload")); + } + let buf_len = buf.len() as u32; + self.inflight_packets.insert(self.seq, InflightPacket::new(self.seq, buf, self.rto)); + self.seq += buf_len; + Ok(()) + } + + pub(super) fn update_last_received_ack(&mut self, ack: SeqNum) { + self.last_received_ack = ack; + } + + pub(crate) fn update_inflight_packet_queue(&mut self, ack: SeqNum) { + match self.inflight_packets.first_key_value() { + None => return, + Some((&seq, _)) if ack < seq => return, + _ => {} + } + if let Some(seq) = self + .inflight_packets + .iter() + .find(|(_, p)| p.contains_seq_num(ack - 1)) + .map(|(&s, _)| s) + { + let mut inflight_packet = self.inflight_packets.remove(&seq).unwrap(); + let distance = ack.distance(inflight_packet.seq) as usize; + if distance < inflight_packet.payload.len() { + inflight_packet.payload.drain(0..distance); + inflight_packet.seq = ack; + self.inflight_packets.insert(ack, inflight_packet); + } + } + self.inflight_packets.retain(|_, p| ack < p.seq + p.payload.len() as u32); + } + + pub(crate) fn find_inflight_packet(&self, seq: SeqNum) -> Option<&InflightPacket> { + self.inflight_packets.get(&seq) + } + + #[must_use] + /// Collect the in-flight packets due for retransmission, and report whether any packet + /// exhausted its retransmissions. Such a packet is dropped from the queue: the peer will + /// never receive those bytes, so the caller must abort the connection rather than carry + /// on with a stream that has a permanent hole in it. + pub(crate) fn collect_timed_out_inflight_packets(&mut self) -> (Vec, bool) { + let mut retransmit_list = Vec::new(); + let mut exhausted = false; + + self.inflight_packets.retain(|_, packet| { + if packet.retransmit_count >= self.max_retransmit_count { + log::warn!("Packet with seq {:?} reached max retransmit count, dropping packet", packet.seq); + exhausted = true; + return false; // remove this packet + } + if packet.is_timed_out() { + packet.retransmit_count += 1; + packet.retransmit_timeout *= 2; // increase timeout exponentially + packet.send_time = std::time::Instant::now(); + retransmit_list.push(packet.clone()); + } + true // keep the packet in the inflight_packets + }); + (retransmit_list, exhausted) + } + + pub(crate) fn get_inflight_packets_total_len(&self) -> usize { + self.inflight_packets.values().map(|p| p.payload.len()).sum() + } + + #[allow(dead_code)] + pub(crate) fn get_all_inflight_packets(&self) -> Vec<&InflightPacket> { + self.inflight_packets.values().collect::>() + } + + pub fn is_send_buffer_full(&self) -> bool { + // To respect the receiver's window (remote_window) size and avoid sending too many unacknowledged packets, which may cause packet loss + // Simplified version: min(cwnd, rwnd) + self.seq.distance(self.get_last_received_ack()) >= self.max_unacked_bytes.min(self.get_send_window() as u32) + } +} + +#[derive(Debug, Clone)] +pub struct InflightPacket { + pub seq: SeqNum, + pub payload: Vec, + pub send_time: std::time::Instant, + pub retransmit_count: usize, + pub retransmit_timeout: std::time::Duration, // current retransmission timeout +} + +impl InflightPacket { + fn new(seq: SeqNum, payload: Vec, rto: Duration) -> Self { + Self { + seq, + payload, + send_time: std::time::Instant::now(), + retransmit_count: 0, + retransmit_timeout: rto, + } + } + pub(crate) fn contains_seq_num(&self, seq: SeqNum) -> bool { + self.seq <= seq && seq < self.seq + self.payload.len() as u32 + } + pub(crate) fn is_timed_out(&self) -> bool { + self.send_time.elapsed() >= self.retransmit_timeout + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_in_flight_packet() { + let p = InflightPacket::new((u32::MAX - 1).into(), vec![10, 20, 30, 40, 50], RTO); + + assert!(p.contains_seq_num((u32::MAX - 1).into())); + assert!(p.contains_seq_num(u32::MAX.into())); + assert!(p.contains_seq_num(0.into())); + assert!(p.contains_seq_num(1.into())); + assert!(p.contains_seq_num(2.into())); + + assert!(!p.contains_seq_num(3.into())); + } + + #[test] + fn test_get_unordered_packets_with_max_bytes() { + let mut tcb = Tcb::new( + SeqNum(1000), + 1500, + MAX_UNACK, + READ_BUFFER_SIZE, + MAX_COUNT_FOR_DUP_ACK, + RTO, + MAX_RETRANSMIT_COUNT, + ); + + // insert 3 consecutive packets + tcb.add_unordered_packet(SeqNum(1000), vec![1; 500]); // seq=1000, len=500 + tcb.add_unordered_packet(SeqNum(1500), vec![2; 500]); // seq=1500, len=500 + tcb.add_unordered_packet(SeqNum(2000), vec![3; 500]); // seq=2000, len=500 + + // test 1: extract up to 700 bytes + let data = tcb.consume_unordered_packets(700).unwrap(); + assert_eq!(data.len(), 700); // extract 500 + 200 + assert_eq!(data[..500], vec![1; 500]); // the first packet + assert_eq!(data[500..700], vec![2; 200]); // the first 200 bytes of the second packet + assert_eq!(tcb.ack, SeqNum(1700)); // ack increased by 700 + assert_eq!(tcb.unordered_packets.len(), 2); // remaining two packets + assert_eq!(tcb.unordered_packets.get(&SeqNum(1700)).unwrap().len(), 300); // the second packet remaining 300 bytes + assert_eq!(tcb.unordered_packets.get(&SeqNum(2000)).unwrap().len(), 500); // the third packet unchanged + + // test 2: extract up to 800 bytes + let data = tcb.consume_unordered_packets(800).unwrap(); + assert_eq!(data.len(), 800); // extract 300 bytes of the second packet and the third packet + assert_eq!(data[..300], vec![2; 300]); // the remaining 300 bytes of the second packet + assert_eq!(data[300..800], vec![3; 500]); // the third packet + assert_eq!(tcb.ack, SeqNum(2500)); // ack increased by 800 + assert_eq!(tcb.unordered_packets.len(), 0); // no remaining packets + + // test 3: no data to extract + let data = tcb.consume_unordered_packets(1000); + assert!(data.is_none()); + } + + #[test] + fn test_update_inflight_packet_queue() { + let mut tcb = Tcb::new( + SeqNum(1000), + 1500, + MAX_UNACK, + READ_BUFFER_SIZE, + MAX_COUNT_FOR_DUP_ACK, + RTO, + MAX_RETRANSMIT_COUNT, + ); + tcb.seq = SeqNum(100); // setting the initial seq + + // insert 3 consecutive packets + tcb.add_inflight_packet(vec![1; 500]).unwrap(); // seq=100, len=500 + tcb.add_inflight_packet(vec![2; 500]).unwrap(); // seq=600, len=500 + tcb.add_inflight_packet(vec![3; 500]).unwrap(); // seq=1100, len=500 + + // test 1: confirm partial packets (ack=800) + tcb.update_inflight_packet_queue(SeqNum(800)); + assert_eq!(tcb.inflight_packets.len(), 2); // remaining two packets + let first_packet = tcb.inflight_packets.first_key_value().unwrap().1; + assert_eq!(first_packet.seq, SeqNum(800)); // the remaining part of the first packet + assert_eq!(first_packet.payload.len(), 300); // remaining 300 bytes in the first packet + let second_packet = tcb.inflight_packets.last_key_value().unwrap().1; + assert_eq!(second_packet.seq, SeqNum(1100)); // no change in the second packet + + // test 2: confirm all packets (ack=2000) + tcb.update_inflight_packet_queue(SeqNum(2000)); + assert_eq!(tcb.inflight_packets.len(), 0); // all packets are acknowledged + } + + #[test] + fn test_update_inflight_packet_queue_cumulative_ack() { + let mut tcb = Tcb::new( + SeqNum(1000), + 1500, + MAX_UNACK, + READ_BUFFER_SIZE, + MAX_COUNT_FOR_DUP_ACK, + RTO, + MAX_RETRANSMIT_COUNT, + ); + tcb.seq = SeqNum(1000); + + // Insert 3 consecutive packets + tcb.add_inflight_packet(vec![1; 500]).unwrap(); // seq=1000, len=500 + tcb.add_inflight_packet(vec![2; 500]).unwrap(); // seq=1500, len=500 + tcb.add_inflight_packet(vec![3; 500]).unwrap(); // seq=2000, len=500 + + // Emulate cumulative ACK: ack=2500 + tcb.update_inflight_packet_queue(SeqNum(2500)); + assert_eq!(tcb.inflight_packets.len(), 0); // all packets should be removed + } + + #[test] + fn test_retransmit_with_exponential_backoff() { + let mut tcb = Tcb::new( + SeqNum(1000), + 1500, + MAX_UNACK, + READ_BUFFER_SIZE, + MAX_COUNT_FOR_DUP_ACK, + RTO, + MAX_RETRANSMIT_COUNT, + ); + + tcb.add_inflight_packet(vec![1; 500]).unwrap(); + + // Simulate retransmission timeouts + for i in 0..MAX_RETRANSMIT_COUNT { + // Simulate a timeout for the first packet + let timeout = tcb.inflight_packets.values().next().unwrap().retransmit_timeout + std::time::Duration::from_millis(100); + println!("timeout: {timeout:?}"); + std::thread::sleep(timeout); + + let (packets, _) = tcb.collect_timed_out_inflight_packets(); + assert_eq!(packets.len(), 1); + let packet = &packets[0]; + assert_eq!(packet.retransmit_count, i + 1); + assert!(packet.retransmit_timeout > RTO); + } + + let (packets, _) = tcb.collect_timed_out_inflight_packets(); + assert!(packets.is_empty()); + assert!(tcb.inflight_packets.is_empty()); + } + + /// A segment that used up its retransmissions is dropped from the queue and reported as + /// exhausted, so the connection can be reset rather than left with a hole in the stream. + #[test] + fn exhausted_retransmissions_are_reported() { + let rto = std::time::Duration::from_millis(5); + let mut tcb = Tcb::new(SeqNum(1000), 1500, MAX_UNACK, READ_BUFFER_SIZE, MAX_COUNT_FOR_DUP_ACK, rto, 2); + tcb.add_inflight_packet(vec![1; 100]).unwrap(); + let mut exhausted = false; + for _ in 0..8 { + std::thread::sleep(rto * 8); + let (_, e) = tcb.collect_timed_out_inflight_packets(); + if e { + exhausted = true; + break; + } + } + assert!(exhausted, "retransmission exhaustion was never reported"); + assert!(tcb.inflight_packets.is_empty()); + let (packets, again) = tcb.collect_timed_out_inflight_packets(); + assert!(packets.is_empty() && !again, "an empty queue reports nothing"); + } +} diff --git a/third_party/ipstack/src/stream/tcp.rs b/third_party/ipstack/src/stream/tcp.rs new file mode 100644 index 00000000..8a07552b --- /dev/null +++ b/third_party/ipstack/src/stream/tcp.rs @@ -0,0 +1,1052 @@ +use super::seqnum::SeqNum; +use crate::{ + PacketReceiver, PacketSender, TTL, + error::IpStackError, + packet::{ + IpHeader, NetworkPacket, NetworkTuple, TransportHeader, + tcp_flags::{ACK, FIN, PSH, RST, SYN}, + tcp_header_flags, tcp_header_fmt, + }, + stream::tcb::{MAX_COUNT_FOR_DUP_ACK, MAX_RETRANSMIT_COUNT, MAX_UNACK, PacketType, READ_BUFFER_SIZE, RTO, Tcb, TcpState}, +}; +use etherparse::{IpNumber, Ipv4Header, Ipv6FlowLabel, TcpHeader, TcpOptionElement}; +use std::{ + future::Future, + io::ErrorKind::{BrokenPipe, ConnectionRefused, InvalidInput, UnexpectedEof}, + net::SocketAddr, + pin::Pin, + sync::Arc, + task::{Context, Poll, Waker}, + time::Duration, +}; +use tokio::io::{AsyncRead, AsyncWrite}; + +/// 2 * MSL (Maximum Segment Lifetime) is the maximum time a TCP connection can be in the TIME_WAIT state. +const TWO_MSL: Duration = Duration::from_secs(2); + +const CLOSE_WAIT_TIMEOUT: Duration = Duration::from_secs(5); +const LAST_ACK_MAX_RETRIES: usize = 3; +const LAST_ACK_TIMEOUT: Duration = Duration::from_millis(500); +const TIMEOUT: Duration = Duration::from_secs(60); + +#[non_exhaustive] +#[derive(Debug, Clone)] +/// TCP configuration +pub struct TcpConfig { + /// Maximum number of retries for sending the last ACK in the LAST_ACK state. Default is 3. + pub last_ack_max_retries: usize, + /// Timeout for the last ACK in the LAST_ACK state. Default is 500ms. + pub last_ack_timeout: Duration, + /// Timeout for the CLOSE_WAIT state. Default is 5 seconds. + pub close_wait_timeout: Duration, + /// Timeout for TCP connections. Default is 60 seconds. + pub timeout: Duration, + /// Timeout for the TIME_WAIT state. Default is 2 seconds. + pub two_msl: Duration, + /// Maximum number of unacknowledged bytes allowed in the send buffer. + pub max_unacked_bytes: u32, + /// Size of the read buffer for incoming data. + pub read_buffer_size: usize, + /// Maximum number of duplicate ACKs before triggering fast retransmission. + pub max_count_for_dup_ack: usize, + /// Retransmission timeout duration. + pub rto: std::time::Duration, + /// Maximum number of retransmissions before giving up. + pub max_retransmit_count: usize, + /// TCP options + pub options: Option>, +} + +#[non_exhaustive] +#[derive(Debug, Clone)] +pub enum TcpOptions { + /// Maximum segment size (MSS) for TCP connections. + MaximumSegmentSize(u16), +} + +impl Default for TcpConfig { + fn default() -> Self { + TcpConfig { + last_ack_max_retries: LAST_ACK_MAX_RETRIES, + last_ack_timeout: LAST_ACK_TIMEOUT, + close_wait_timeout: CLOSE_WAIT_TIMEOUT, + timeout: TIMEOUT, + two_msl: TWO_MSL, + max_unacked_bytes: MAX_UNACK, + read_buffer_size: READ_BUFFER_SIZE, + max_count_for_dup_ack: MAX_COUNT_FOR_DUP_ACK, + rto: RTO, + max_retransmit_count: MAX_RETRANSMIT_COUNT, + options: Default::default(), + } + } +} + +#[derive(Debug)] +enum Shutdown { + None, + Pending(Waker), + Ready, +} + +impl Shutdown { + fn pending(&mut self, w: Waker) { + *self = Shutdown::Pending(w); + } + fn ready(&mut self) { + if let Shutdown::Pending(w) = self { + w.wake_by_ref(); + } + *self = Shutdown::Ready; + } + + // Just for comparison purpose + fn fake_clone(&self) -> Shutdown { + match self { + Shutdown::None => Shutdown::None, + Shutdown::Pending(_) => Shutdown::Pending(Waker::noop().clone()), + Shutdown::Ready => Shutdown::Ready, + } + } +} + +impl std::fmt::Display for Shutdown { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Shutdown::None => write!(f, "None"), + Shutdown::Pending(_) => write!(f, "Pending"), + Shutdown::Ready => write!(f, "Ready"), + } + } +} + +static SESSION_COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +type TcbPtr = std::sync::Arc>; + +/// A TCP stream in the IP stack. +/// +/// This type represents a TCP connection and implements `AsyncRead` and `AsyncWrite` +/// for bidirectional data transfer. It handles TCP state management, flow control, +/// and retransmission automatically. +/// +/// # Examples +/// +/// ```no_run +/// use ipstack::{IpStack, IpStackConfig, IpStackStream}; +/// use tokio::io::{AsyncReadExt, AsyncWriteExt}; +/// +/// # async fn example(mut ip_stack: IpStack) -> Result<(), Box> { +/// if let IpStackStream::Tcp(mut tcp_stream) = ip_stack.accept().await? { +/// println!("New TCP connection from {}", tcp_stream.peer_addr()); +/// +/// // Read data +/// let mut buffer = [0u8; 1024]; +/// let n = tcp_stream.read(&mut buffer).await?; +/// +/// // Write data +/// tcp_stream.write_all(b"HTTP/1.1 200 OK\r\n\r\n").await?; +/// +/// // Shutdown the stream +/// tcp_stream.shutdown().await?; +/// } +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug)] +pub struct IpStackTcpStream { + src_addr: SocketAddr, + dst_addr: SocketAddr, + stream_sender: PacketSender, + stream_receiver: Option, + up_packet_sender: PacketSender, + tcb: TcbPtr, + shutdown: std::sync::Arc>, + write_notify: std::sync::Arc>>, + destroy_messenger: Option<::tokio::sync::oneshot::Sender<()>>, + timeout: Pin>, + data_tx: tokio::sync::mpsc::UnboundedSender>, + data_rx: tokio::sync::mpsc::UnboundedReceiver>, + read_notify: std::sync::Arc>>, + task_handle: Option>>, + exit_notifier: Option>, + temp_read_buffer: Vec, + config: Arc, +} + +impl IpStackTcpStream { + pub(crate) fn new( + src_addr: SocketAddr, + dst_addr: SocketAddr, + tcp: TcpHeader, + up_packet_sender: PacketSender, + mtu: u16, + destroy_messenger: Option<::tokio::sync::oneshot::Sender<()>>, + config: Arc, + ) -> Result { + let tcb = Tcb::new( + SeqNum(tcp.sequence_number), + mtu, + config.max_unacked_bytes, + config.read_buffer_size, + config.max_count_for_dup_ack, + config.rto, + config.max_retransmit_count, + ); + let tuple = NetworkTuple::new(src_addr, dst_addr, true); + if !tcp.syn { + if !tcp.rst + && let Err(err) = write_packet_to_device(&up_packet_sender, tuple, &tcb, None, ACK | RST, None, None) + { + log::warn!("Error sending RST/ACK packet: {err}"); + } + let info = format!("Invalid TCP packet: {tuple} {}", tcp_header_fmt(&tcp)); + return Err(IpStackError::IoError(std::io::Error::new(ConnectionRefused, info))); + } + + let (stream_sender, stream_receiver) = tokio::sync::mpsc::unbounded_channel::(); + let (data_tx, data_rx) = tokio::sync::mpsc::unbounded_channel::>(); + let deadline = tokio::time::Instant::now() + config.timeout; + + let mut stream = IpStackTcpStream { + src_addr, + dst_addr, + stream_sender, + stream_receiver: Some(stream_receiver), + up_packet_sender, + tcb: std::sync::Arc::new(std::sync::Mutex::new(tcb.clone())), + shutdown: std::sync::Arc::new(std::sync::Mutex::new(Shutdown::None)), + write_notify: std::sync::Arc::new(std::sync::Mutex::new(None)), + destroy_messenger, + timeout: Box::pin(tokio::time::sleep_until(deadline)), + data_tx, + data_rx, + read_notify: std::sync::Arc::new(std::sync::Mutex::new(None)), + task_handle: None, + exit_notifier: None, + temp_read_buffer: Vec::new(), + config, + }; + + let sessions = SESSION_COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst).saturating_add(1); + let (seq, ack, state) = { (tcb.get_seq().0, tcb.get_ack().0, tcb.get_state()) }; + let l_info = format!("local {{ seq: {seq}, ack: {ack} }}"); + log::debug!("{tuple} {state:?}: {l_info} session begins, total TCP sessions: {sessions}"); + + stream.spawn_tasks()?; + Ok(stream) + } + + fn reset_timeout(&mut self) { + let deadline = tokio::time::Instant::now() + self.config.timeout; + self.timeout.as_mut().reset(deadline); + } + + pub(crate) fn network_tuple(&self) -> NetworkTuple { + NetworkTuple::new(self.src_addr, self.dst_addr, true) + } + + /// Returns the local socket address of the TCP connection. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackTcpStream; + /// # fn example(tcp_stream: &IpStackTcpStream) { + /// let local_addr = tcp_stream.local_addr(); + /// println!("Local address: {}", local_addr); + /// # } + /// ``` + pub fn local_addr(&self) -> SocketAddr { + self.src_addr + } + + /// Returns the remote socket address of the TCP connection. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackTcpStream; + /// # fn example(tcp_stream: &IpStackTcpStream) { + /// let peer_addr = tcp_stream.peer_addr(); + /// println!("Peer address: {}", peer_addr); + /// # } + /// ``` + pub fn peer_addr(&self) -> SocketAddr { + self.dst_addr + } + + pub fn stream_sender(&self) -> PacketSender { + self.stream_sender.clone() + } +} + +impl AsyncRead for IpStackTcpStream { + fn poll_read(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf<'_>) -> Poll> { + // if there is data in the temp buffer, read it first + if !self.temp_read_buffer.is_empty() { + let len = std::cmp::min(buf.remaining(), self.temp_read_buffer.len()); + buf.put_slice(&self.temp_read_buffer[..len]); + self.temp_read_buffer.drain(..len); // remove the read data from the temp buffer + return Poll::Ready(Ok(())); + } + + let network_tuple = self.network_tuple(); + + let state = self.tcb.lock().unwrap().get_state(); + if state == TcpState::Closed { + self.shutdown.lock().unwrap().ready(); + self.write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + return Poll::Ready(Ok(())); + } + + // handle timeout + if matches!(Pin::new(&mut self.timeout).poll(cx), Poll::Ready(_)) { + { + let mut tcb = self.tcb.lock().unwrap(); + let (seq, ack) = (tcb.get_seq().0, tcb.get_ack().0); + let l_info = format!("local {{ seq: {seq}, ack: {ack} }}"); + log::warn!("{network_tuple} {state:?}: [poll_read] {l_info}, session timeout reached, closing forcefully..."); + let sender = &self.up_packet_sender; + write_packet_to_device(sender, network_tuple, &tcb, None, ACK | RST, None, None)?; + tcb.change_state(TcpState::Closed); + let state = tcb.get_state(); + log::warn!("{network_tuple} {state:?}: [poll_read] {l_info}, session notified to close"); + } + self.shutdown.lock().unwrap().ready(); + + return Poll::Ready(Err(std::io::Error::from(std::io::ErrorKind::TimedOut))); + } + self.reset_timeout(); + + // read data from channel + match self.data_rx.poll_recv(cx) { + Poll::Ready(Some(data)) => { + let capacity = buf.remaining(); + if capacity >= data.len() { + buf.put_slice(&data); + } else { + // if `buf` is not enough, put the remaining data into the temp buffer + buf.put_slice(&data[..capacity]); + self.temp_read_buffer.extend_from_slice(&data[capacity..]); + } + Poll::Ready(Ok(())) + } + Poll::Ready(None) => Poll::Ready(Ok(())), + Poll::Pending => { + self.read_notify.lock().unwrap().replace(cx.waker().clone()); + Poll::Pending + } + } + } +} + +impl AsyncWrite for IpStackTcpStream { + fn poll_write(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + let nt = self.network_tuple(); + self.reset_timeout(); + + let mut tcb = self.tcb.lock().unwrap(); + let state = tcb.get_state(); + let send_window = tcb.get_send_window(); + let is_full = tcb.is_send_buffer_full(); + + if state == TcpState::Closed { + self.shutdown.lock().unwrap().ready(); + self.read_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + return Poll::Ready(Err(std::io::Error::new(BrokenPipe, "TCP connection closed"))); + } + + if send_window == 0 || is_full { + self.write_notify.lock().unwrap().replace(cx.waker().clone()); + let info = format!("current send window: {send_window}, send buffer full: {is_full}"); + log::trace!("{nt} {state:?}: [poll_write] {info}, waiting for the other side to send ACK..."); + return Poll::Pending; + } + + let sender = &self.up_packet_sender; + let payload_len = write_packet_to_device(sender, nt, &tcb, None, ACK | PSH, None, Some(buf.to_vec()))?; + tcb.add_inflight_packet(buf[..payload_len].to_vec())?; + + let (state, seq, ack) = (tcb.get_state(), tcb.get_seq(), tcb.get_ack()); + let l_info = format!("local {{ seq: {seq}, ack: {ack} }}"); + log::trace!("{nt} {state:?}: [poll_write] {l_info} upstream data written to device, len = {payload_len}"); + + Poll::Ready(Ok(payload_len)) + } + + fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let shutdown = { self.shutdown.lock().unwrap().fake_clone() }; + let (nt, state, seq, is_ready) = { + let tcb = self.tcb.lock().unwrap(); + let is_ready = tcb.get_inflight_packets_total_len() == 0; + (self.network_tuple(), tcb.get_state(), tcb.get_seq(), is_ready) + }; + log::trace!("{nt} {state:?}: [poll_shutdown] seq = {seq}, ready = {is_ready}, shutdown {shutdown}",); + if state == TcpState::Closed { + return Poll::Ready(Ok(())); + } + match shutdown { + Shutdown::None => { + if is_ready && state == TcpState::Established { + let mut tcb = self.tcb.lock().unwrap(); + send_fin_n_change_state_to_fin_wait1("[poll_shutdown]", nt, &self.up_packet_sender, &mut tcb)?; + } + self.shutdown.lock().unwrap().pending(cx.waker().clone()); + Poll::Pending + } + Shutdown::Pending(_) => { + if is_ready && state == TcpState::Established { + let mut tcb = self.tcb.lock().unwrap(); + send_fin_n_change_state_to_fin_wait1("[poll_shutdown]", nt, &self.up_packet_sender, &mut tcb)?; + } + Poll::Pending + } + Shutdown::Ready => Poll::Ready(Ok(())), + } + } +} + +fn send_fin_n_change_state_to_fin_wait1(hint: &str, nt: NetworkTuple, sender: &PacketSender, tcb: &mut Tcb) -> std::io::Result<()> { + let state = tcb.get_state(); + if !(tcb.get_inflight_packets_total_len() == 0 && state == TcpState::Established) { + log::debug!("{nt} {state:?}: {hint} session is not in a valid state to send FIN, skipping..."); + return Ok(()); + } + + log::debug!("{nt} {state:?}: {hint} actively send a farewell packet to the other side..."); + write_packet_to_device(sender, nt, tcb, None, ACK | FIN, None, None)?; + tcb.increase_seq(); + tcb.change_state(TcpState::FinWait1); + let state = tcb.get_state(); + log::debug!("{nt} {state:?}: {hint} now in {state:?} state"); + + Ok(()) +} + +impl Drop for IpStackTcpStream { + fn drop(&mut self) { + let (nt, state) = (self.network_tuple(), self.tcb.lock().unwrap().get_state()); + log::trace!("{nt} {state:?}: [drop] session dropping, ========================= "); + if let Some(task_handle) = self.task_handle.take() { + if !task_handle.is_finished() { + if let Some(notifier) = self.exit_notifier.take() { + _ = tokio::task::block_in_place(|| tokio::runtime::Handle::current().block_on(notifier.send(()))); + } + // synchronously wait for the task to finish + _ = tokio::task::block_in_place(|| tokio::runtime::Handle::current().block_on(task_handle)); + } else { + log::trace!("{nt} {state:?}: [drop] task already finished, no need to wait exiting"); + } + } + let sessions = SESSION_COUNTER.fetch_sub(1, std::sync::atomic::Ordering::SeqCst).saturating_sub(1); + log::debug!("{nt} {state:?}: [drop] session dropped, total TCP sessions: {sessions}"); + } +} + +impl IpStackTcpStream { + fn spawn_tasks(&mut self) -> std::io::Result<()> { + let network_tuple = self.network_tuple(); + + // task: data receiving and processing + let tcb = self.tcb.clone(); + let stream_receiver = self.stream_receiver.take().unwrap(); + let up_packet_sender = self.up_packet_sender.clone(); + let shutdown = self.shutdown.clone(); + let write_notify = self.write_notify.clone(); + let read_notify = self.read_notify.clone(); + let data_tx = self.data_tx.clone(); + let destroy_messenger = self.destroy_messenger.take(); + + let (exit_task_notifier, exit_monitor) = tokio::sync::mpsc::channel::<()>(10); + let exit_notifier = exit_task_notifier.clone(); + let config = self.config.clone(); + self.exit_notifier = Some(exit_task_notifier); + + let task_handle = tokio::spawn(async move { + let v = tcp_main_logic_loop( + tcb, + config, + stream_receiver, + up_packet_sender, + exit_notifier, + network_tuple, + write_notify, + read_notify, + data_tx, + exit_monitor, + ) + .await; + if let Err(e) = &v { + log::warn!("{network_tuple} task error: {e}"); + } + _ = destroy_messenger.map(|m| m.send(())).unwrap_or(Ok(())); + log::trace!("{network_tuple} task completed, destroy messenger sent successfully"); + shutdown.lock().unwrap().ready(); + log::trace!("{network_tuple} shutdown.lock().unwrap().ready() =========="); + v + }); + self.task_handle = Some(task_handle); + Ok(()) + } +} + +#[allow(clippy::too_many_arguments)] +async fn tcp_main_logic_loop( + tcb: TcbPtr, + config: Arc, + mut stream_receiver: PacketReceiver, + up_packet_sender: PacketSender, + exit_notifier: tokio::sync::mpsc::Sender<()>, + network_tuple: NetworkTuple, + write_notify: std::sync::Arc>>, + read_notify: std::sync::Arc>>, + data_tx: tokio::sync::mpsc::UnboundedSender>, + mut exit_monitor: tokio::sync::mpsc::Receiver<()>, +) -> std::io::Result<()> { + { + let mut tcb = tcb.lock().unwrap(); + + let state = tcb.get_state(); + if state != TcpState::Listen { + log::warn!("{network_tuple} {state:?}: Invalid TCP state, not in Listen state"); + return Ok::<(), std::io::Error>(()); + } + + tcb.increase_ack(); + let (seq, ack) = (tcb.get_seq().0, tcb.get_ack().0); + let l_info = format!("local {{ seq: {seq}, ack: {ack} }}"); + log::trace!("{network_tuple} {state:?}: {l_info} session begins"); + write_packet_to_device( + &up_packet_sender, + network_tuple, + &tcb, + config.options.as_ref(), + ACK | SYN, + None, + None, + )?; + tcb.increase_seq(); + tcb.change_state(TcpState::SynReceived); + let state = tcb.get_state(); + log::trace!("{network_tuple} {state:?}: session now in {state:?} state"); + } + + let tcb_clone = tcb.clone(); + + async fn task_wait_to_close(tcb: TcbPtr, exit_notifier: tokio::sync::mpsc::Sender<()>, nt: NetworkTuple, two_msl: Duration) { + tokio::time::sleep(two_msl).await; + { + let mut tcb = tcb.lock().unwrap(); + tcb.change_state(TcpState::Closed); + let state = tcb.get_state(); + log::debug!("{nt} {state:?}: [task_wait_to_close] session closed after {two_msl:?}"); + } + exit_notifier.send(()).await.unwrap_or(()); + } + + async fn task_last_ack( + tcb: TcbPtr, + exit_notifier: tokio::sync::mpsc::Sender<()>, + nt: NetworkTuple, + pkt_sdr: PacketSender, + last_ack_timeout: Duration, + last_ack_max_retries: usize, + ) { + let hint = "[task_last_ack]"; + for idx in 1..=last_ack_max_retries { + let state = { tcb.lock().unwrap().get_state() }; + if state == TcpState::Closed { + log::debug!("{nt} {state:?}: {hint} session closed, exiting 1..."); + return; + } + + tokio::time::sleep(last_ack_timeout).await; + + { + let tcb = tcb.lock().unwrap(); + let state = tcb.get_state(); + if state == TcpState::Closed { + log::debug!("{nt} {state:?}: {hint} session closed, exiting 2..."); + return; + } + log::debug!("{nt} {state:?}: {hint} timer expired, resending ACK|FIN (retry {idx}/{last_ack_max_retries})"); + _ = write_packet_to_device(&pkt_sdr, nt, &tcb, None, ACK | FIN, None, None); + } + } + { + let mut tcb = tcb.lock().unwrap(); + tcb.change_state(TcpState::Closed); + let state = tcb.get_state(); + log::warn!("{nt} {state:?}: {hint} max retries reached, forcibly closing session"); + } + exit_notifier.send(()).await.unwrap_or(()); + } + + async fn task_timed_out_for_close_wait( + tcb: TcbPtr, + exit_notifier: tokio::sync::mpsc::Sender<()>, + nt: NetworkTuple, + up_packet_sender: PacketSender, + close_wait_timeout: Duration, + last_ack_timeout: Duration, + last_ack_max_retries: usize, + ) -> std::io::Result<()> { + tokio::time::sleep(close_wait_timeout).await; // Wait CLOSE_WAIT_TIMEOUT for upstream + let tcb_clone = tcb.clone(); + let mut tcb = tcb.lock().unwrap(); + let state = tcb.get_state(); + if state != TcpState::CloseWait { + return Ok(()); + } + log::warn!("{nt} {state:?}: Upstream timeout, forcing FIN"); + write_packet_to_device(&up_packet_sender, nt, &tcb, None, ACK | FIN, None, None)?; + tcb.increase_seq(); + tcb.change_state(TcpState::LastAck); + let new_state = tcb.get_state(); + log::debug!("{nt} {state:?}: Forced transition to {new_state:?}"); + + // Here we set a timer to wait for the last ACK from the other side. + tokio::spawn(task_last_ack( + tcb_clone, + exit_notifier, + nt, + up_packet_sender, + last_ack_timeout, + last_ack_max_retries, + )); + + Ok::<(), std::io::Error>(()) + } + + loop { + let exit_notifier = exit_notifier.clone(); + + let network_packet = tokio::select! { + _ = exit_monitor.recv() => { + log::debug!("{network_tuple} task exited due to exit signal"); + break; + } + network_packet = stream_receiver.recv() => network_packet, + }; + + let Some(mut network_packet) = network_packet else { + let state = { tcb.lock().unwrap().get_state() }; + log::debug!("{network_tuple} {state:?}: session closed unexpectedly by pipe broken, exiting task"); + tcb.lock().unwrap().change_state(TcpState::Closed); + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + read_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + break; + }; + + let payload = network_packet.payload.take().unwrap_or_default(); + let TransportHeader::Tcp(tcp_header) = network_packet.transport_header() else { + log::warn!("{network_tuple} Invalid TCP packet"); + continue; + }; + let flags = tcp_header_flags(tcp_header); + let incoming_ack: SeqNum = tcp_header.acknowledgment_number.into(); + let incoming_seq: SeqNum = tcp_header.sequence_number.into(); + let incoming_win = tcp_header.window_size; + + let mut tcb = tcb.lock().unwrap(); + + let state = tcb.get_state(); + if state == TcpState::Closed { + log::debug!("{network_tuple} {state:?}: session finished, exiting task..."); + break; + } + + if flags & RST == RST { + tcb.change_state(TcpState::Closed); + continue; + } + + tcb.update_duplicate_ack_count(incoming_ack); + + tcb.update_inflight_packet_queue(incoming_ack); + + let (timed_out, exhausted) = tcb.collect_timed_out_inflight_packets(); + if exhausted { + // Retransmissions gave up on a segment the peer never acknowledged. Leaving the + // connection Established would hand the application a stream with a permanent hole: + // reads that never return and dup-ACKs we can no longer serve. Reset it instead so + // the application fails fast and can reconnect. + log::warn!("{network_tuple} {state:?}: retransmissions exhausted, resetting the connection"); + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, RST | ACK, None, None)?; + tcb.change_state(TcpState::Closed); + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + read_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + break; + } + for packet in timed_out { + let (seq, count) = (packet.seq, packet.retransmit_count); + log::debug!("{network_tuple} inflight packet retransmission timeout: {seq:?}, retransmit_count: {count}",); + write_packet_to_device( + &up_packet_sender, + network_tuple, + &tcb, + None, + ACK | PSH, + Some(seq), + Some(packet.payload), + )?; + } + + let pkt_type = tcb.check_pkt_type(tcp_header, &payload); + + let (state, seq, ack) = { (tcb.get_state(), tcb.get_seq(), tcb.get_ack()) }; + let (info, len) = (tcp_header_fmt(tcp_header), payload.len()); + let l_info = format!("local {{ seq: {seq}, ack: {ack} }}"); + log::trace!("{network_tuple} {state:?}: {l_info} {info}, {pkt_type:?}, len = {len}"); + if pkt_type == PacketType::Invalid { + continue; + } + + match state { + TcpState::SynReceived if flags & ACK == ACK => { + if len > 0 { + tcb.add_unordered_packet(incoming_seq, payload); + extract_data_n_write_upstream(&up_packet_sender, &mut tcb, network_tuple, &data_tx, &read_notify)?; + } + tcb.change_state(TcpState::Established); + } + TcpState::Established => { + if flags == ACK { + match pkt_type { + PacketType::WindowUpdate => { + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + } + PacketType::KeepAlive => { + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK, None, None)?; + } + PacketType::RetransmissionRequest => { + if let Some(packet) = tcb.find_inflight_packet(incoming_ack) { + let (s, p) = (packet.seq, packet.payload.clone()); + log::debug!( + "{network_tuple} {state:?}: {l_info}, {pkt_type:?}, retransmission request, seq = {s}, len = {}", + p.len() + ); + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK | PSH, Some(s), Some(p))?; + } + } + PacketType::NewPacket => { + tcb.add_unordered_packet(incoming_seq, payload); + let nt = network_tuple; + extract_data_n_write_upstream(&up_packet_sender, &mut tcb, nt, &data_tx, &read_notify)?; + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + } + PacketType::Ack => { + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + } + PacketType::Invalid => {} + } + } else if flags == (ACK | FIN) { + // The other side is closing the connection, we need to send an ACK and change state to CloseWait + tcb.increase_ack(); + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK, None, None)?; + tcb.change_state(TcpState::CloseWait); + + let s = tcb.get_state(); + let len = tcb.get_inflight_packets_total_len(); + if len == 0 { + // All upstream data sent, proceed to LastAck + log::trace!("{network_tuple} {s:?}: {l_info}, {pkt_type:?}, closed by the other side, no upstream data"); + + // Here we don't wait, just send FIN to the other side and change state to LastAck directly, + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK | FIN, None, None)?; + tcb.increase_seq(); + tcb.change_state(TcpState::LastAck); + + let s = tcb.get_state(); + log::trace!("{network_tuple} {s:?}: {l_info}, {pkt_type:?}, wait the last ack from the other side"); + + // Here we set a timer to wait for the last ACK from the other side. + // If the timer expires, we send an ACK|FIN packet to the other side again and wait anthoer timeout + // till the retries reach the limit, and then close the session forcibly. + let up = up_packet_sender.clone(); + tokio::spawn(task_last_ack( + tcb_clone.clone(), + exit_notifier, + network_tuple, + up, + config.last_ack_timeout, + config.last_ack_max_retries, + )); + } else { + // Upstream data pending, wake write_notify and wait + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + log::debug!("{network_tuple} {state:?}: Waiting for upstream data to complete, inflight packets: {len}",); + + // Spawn a timeout task to force FIN if upstream is unresponsive + let tcb = tcb_clone.clone(); + let up = up_packet_sender.clone(); + tokio::spawn(task_timed_out_for_close_wait( + tcb, + exit_notifier, + network_tuple, + up, + config.close_wait_timeout, + config.last_ack_timeout, + config.last_ack_max_retries, + )); + } + } else if flags == (ACK | PSH) && pkt_type == PacketType::NewPacket { + if !payload.is_empty() && tcb.get_ack() == incoming_seq { + tcb.add_unordered_packet(incoming_seq, payload); + extract_data_n_write_upstream(&up_packet_sender, &mut tcb, network_tuple, &data_tx, &read_notify)?; + } + } else { + // unnormal case, we do nothing here + log::trace!("{network_tuple} {state:?}: {l_info}, {pkt_type:?}, unnormal case, we do nothing here"); + } + } + TcpState::CloseWait => { + if flags & ACK == ACK && tcb.get_inflight_packets_total_len() == 0 { + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK | FIN, None, None)?; + tcb.increase_seq(); + tcb.change_state(TcpState::LastAck); + let new_state = tcb.get_state(); + log::trace!("{network_tuple} {state:?}: Received ACK|FIN, transitioned to {new_state:?}"); + + // Here we set a timer to wait for the last ACK from the other side. + // If the timer expires, we send an ACK|FIN packet to the other side again and wait anthoer timeout + // till the retries reach the limit, and then close the session forcibly. + let up = up_packet_sender.clone(); + tokio::spawn(task_last_ack( + tcb_clone.clone(), + exit_notifier, + network_tuple, + up, + config.last_ack_timeout, + config.last_ack_max_retries, + )); + } else { + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + } + } + TcpState::LastAck if flags & ACK == ACK => { + tcb.change_state(TcpState::Closed); + tokio::spawn(async move { + if let Err(e) = exit_notifier.send(()).await { + log::debug!("exit_notifier send failed: {e}"); + } + }); + let new_state = tcb.get_state(); + log::trace!("{network_tuple} {state:?}: Received final ACK, transitioned to {new_state:?}"); + } + TcpState::FinWait1 => { + if flags & (ACK | FIN) == (ACK | FIN) && len == 0 { + // If the received packet is an ACK with FIN, we need to send an ACK and change state to TimeWait directly, not to FinWait2 + tcb.increase_ack(); + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK, None, None)?; + tcb.change_state(TcpState::TimeWait); + + tokio::spawn(task_wait_to_close(tcb_clone.clone(), exit_notifier, network_tuple, config.two_msl)); + let new_state = tcb.get_state(); + log::trace!("{network_tuple} {state:?}: Final ACK|FIN received too early, transitioned to {new_state:?} directly"); + } else if flags & ACK == ACK { + tcb.change_state(TcpState::FinWait2); + if len > 0 { + // if the other side is still sending data, we need to deal with it like PacketStatus::NewPacket + tcb.add_unordered_packet(incoming_seq, payload); + extract_data_n_write_upstream(&up_packet_sender, &mut tcb, network_tuple, &data_tx, &read_notify)?; + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + } + let new_state = tcb.get_state(); + log::trace!("{network_tuple} {state:?}: Received ACK, transitioned to {new_state:?}"); + } else { + // unnormal case, we do nothing here + log::trace!("{network_tuple} {state:?}: Some unnormal case, we do nothing here"); + } + } + TcpState::FinWait2 => { + if flags & (ACK | FIN) == (ACK | FIN) && len == 0 { + tcb.increase_ack(); + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK, None, None)?; + tcb.change_state(TcpState::TimeWait); + tokio::spawn(task_wait_to_close(tcb_clone.clone(), exit_notifier, network_tuple, config.two_msl)); + let new_state = tcb.get_state(); + log::trace!("{network_tuple} {state:?}: Received final ACK|FIN, transitioned to {new_state:?}"); + } else if flags & ACK == ACK && len == 0 { + // unnormal case, we do nothing here + let l_ack = tcb.get_ack(); + if incoming_seq < l_ack { + log::trace!("{network_tuple} {state:?}: Ignoring duplicate ACK, seq {incoming_seq}, expected {l_ack}"); + } + } else if flags & ACK == ACK && len > 0 { + if pkt_type == PacketType::KeepAlive { + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK, None, None)?; + } else { + // if the other side is still sending data, we need to deal with it like PacketStatus::NewPacket + tcb.add_unordered_packet(incoming_seq, payload); + extract_data_n_write_upstream(&up_packet_sender, &mut tcb, network_tuple, &data_tx, &read_notify)?; + write_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + } + if flags & FIN == FIN { + tcb.change_state(TcpState::TimeWait); + tokio::spawn(task_wait_to_close(tcb_clone.clone(), exit_notifier, network_tuple, config.two_msl)); + let new_state = tcb.get_state(); + log::trace!("{network_tuple} {state:?}: Received final ACK|FIN, transitioned to {new_state:?}"); + } + } else { + // unnormal case, we do nothing here + log::trace!("{network_tuple} {state:?}: Some unnormal case, we do nothing here"); + } + } + TcpState::TimeWait if flags & (ACK | FIN) == (ACK | FIN) => { + write_packet_to_device(&up_packet_sender, network_tuple, &tcb, None, ACK, None, None)?; + // wait to timeout, can't call `tcb.change_state(TcpState::Closed);` to change state here + // now we need to wait for the timeout to reach... + } + _ => {} + } // end of match state + + tcb.update_last_received_ack(incoming_ack); + tcb.update_send_window(incoming_win); + } // end of loop + Ok::<(), std::io::Error>(()) +} + +fn extract_data_n_write_upstream( + up_packet_sender: &PacketSender, + tcb: &mut Tcb, + network_tuple: NetworkTuple, + data_tx: &tokio::sync::mpsc::UnboundedSender>, + read_notify: &std::sync::Arc>>, +) -> std::io::Result<()> { + let (state, seq, ack) = (tcb.get_state(), tcb.get_seq(), tcb.get_ack()); + let l_info = format!("local {{ seq: {seq}, ack: {ack} }}"); + if state == TcpState::Closed { + log::debug!("{network_tuple} {state:?}: {l_info} session closed, exiting \"data extraction task\"..."); + return Ok(()); + } + + if let Some(data) = tcb.consume_unordered_packets(8192) { + let hint = if state == TcpState::Established { "normally" } else { "still" }; + log::trace!("{network_tuple} {state:?}: {l_info} {hint} receiving data, len = {}", data.len()); + data_tx.send(data).map_err(|e| std::io::Error::new(BrokenPipe, e))?; + read_notify.lock().unwrap().take().map(|w| w.wake_by_ref()).unwrap_or(()); + write_packet_to_device(up_packet_sender, network_tuple, tcb, None, ACK, None, None)?; + } + Ok(()) +} + +/// Send a TCP packet to the downstream device, with the specified flags, sequence number, and payload. +/// The returned value is the length of the `payload` sent, it may be shorter than the length of the incoming parameter `payload`. +pub(crate) fn write_packet_to_device( + up_packet_sender: &PacketSender, + tuple: NetworkTuple, + tcb: &Tcb, + options: Option<&Vec>, + flags: u8, + seq: Option, + payload: Option>, +) -> std::io::Result { + use std::io::Error; + let seq = seq.unwrap_or(tcb.get_seq()).0; + let (ack, window_size) = (tcb.get_ack().0, tcb.get_recv_window().max(tcb.get_mtu())); + let (src, dst) = (tuple.dst, tuple.src); // Note: The address is reversed here + let calc = |ip_header_len: usize, tcp_header_len: usize| tcb.calculate_payload_max_len(ip_header_len, tcp_header_len); + let packet = create_raw_packet( + src, + dst, + calc, + flags, + TTL, + seq, + ack, + window_size, + payload.unwrap_or_default(), + options, + )?; + let len = packet.payload.as_ref().map(|p| p.len()).unwrap_or(0); + up_packet_sender.send(packet).map_err(|e| Error::new(UnexpectedEof, e))?; + Ok(len) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn create_raw_packet( + src_addr: SocketAddr, + dst_addr: SocketAddr, + calculate_payload_max_len: impl Fn(usize, usize) -> usize, + flags: u8, + ttl: u8, + seq: u32, + ack: u32, + win: u16, + mut payload: Vec, + options: Option<&Vec>, +) -> std::io::Result { + let mut tcp_header = etherparse::TcpHeader::new(src_addr.port(), dst_addr.port(), seq, win); + tcp_header.acknowledgment_number = ack; + tcp_header.syn = flags & SYN != 0; + tcp_header.ack = flags & ACK != 0; + tcp_header.rst = flags & RST != 0; + tcp_header.fin = flags & FIN != 0; + tcp_header.psh = flags & PSH != 0; + + if let Some(opts) = options { + let mut tcp_options = Vec::new(); + for opt in opts { + match opt { + TcpOptions::MaximumSegmentSize(mss) => tcp_options.push(TcpOptionElement::MaximumSegmentSize(*mss)), + } + } + tcp_header + .set_options(&tcp_options) + .map_err(|e| std::io::Error::new(InvalidInput, e))?; + } + let ip_header = match (src_addr.ip(), dst_addr.ip()) { + (std::net::IpAddr::V4(src), std::net::IpAddr::V4(dst)) => { + let mut ip_h = + Ipv4Header::new(0, ttl, IpNumber::TCP, src.octets(), dst.octets()).map_err(|e| std::io::Error::new(InvalidInput, e))?; + let payload_len = calculate_payload_max_len(ip_h.header_len(), tcp_header.header_len()); + payload.truncate(payload_len); + ip_h.set_payload_len(payload.len() + tcp_header.header_len()) + .map_err(|e| std::io::Error::new(InvalidInput, e))?; + ip_h.dont_fragment = true; + IpHeader::Ipv4(ip_h) + } + (std::net::IpAddr::V6(src), std::net::IpAddr::V6(dst)) => { + let mut ip_h = etherparse::Ipv6Header { + traffic_class: 0, + flow_label: Ipv6FlowLabel::ZERO, + payload_length: 0, + next_header: IpNumber::TCP, + hop_limit: ttl, + source: src.octets(), + destination: dst.octets(), + }; + let payload_len = calculate_payload_max_len(ip_h.header_len(), tcp_header.header_len()); + payload.truncate(payload_len); + let len = payload.len() + tcp_header.header_len(); + ip_h.set_payload_length(len).map_err(|e| std::io::Error::new(InvalidInput, e))?; + + IpHeader::Ipv6(ip_h) + } + _ => return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "IP version mismatch")), + }; + + match ip_header { + IpHeader::Ipv4(ref ip_header) => { + tcp_header.checksum = tcp_header + .calc_checksum_ipv4(ip_header, &payload) + .map_err(|e| std::io::Error::new(InvalidInput, e))?; + } + IpHeader::Ipv6(ref ip_header) => { + tcp_header.checksum = tcp_header + .calc_checksum_ipv6(ip_header, &payload) + .map_err(|e| std::io::Error::new(InvalidInput, e))?; + } + } + Ok(NetworkPacket { + ip: ip_header, + transport: TransportHeader::Tcp(tcp_header), + payload: Some(payload), + }) +} diff --git a/third_party/ipstack/src/stream/udp.rs b/third_party/ipstack/src/stream/udp.rs new file mode 100644 index 00000000..c2f7c4b8 --- /dev/null +++ b/third_party/ipstack/src/stream/udp.rs @@ -0,0 +1,261 @@ +use crate::{ + IpStackError, PacketReceiver, PacketSender, TTL, + packet::{IpHeader, NetworkPacket, TransportHeader}, +}; +use etherparse::{IpNumber, Ipv4Header, Ipv6FlowLabel, Ipv6Header, UdpHeader}; +use std::{future::Future, net::SocketAddr, pin::Pin, time::Duration}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::mpsc, + time::Sleep, +}; + +/// A UDP stream in the IP stack. +/// +/// This type represents a UDP connection and implements `AsyncRead` and `AsyncWrite` +/// for bidirectional data transfer. UDP streams have a configurable timeout and +/// automatically handle packet fragmentation based on MTU. +/// +/// # Examples +/// +/// ```no_run +/// use ipstack::{IpStack, IpStackConfig, IpStackStream}; +/// use tokio::io::{AsyncReadExt, AsyncWriteExt}; +/// +/// # async fn example(mut ip_stack: IpStack) -> Result<(), Box> { +/// if let IpStackStream::Udp(mut udp_stream) = ip_stack.accept().await? { +/// println!("New UDP stream from {}", udp_stream.peer_addr()); +/// +/// // Read data +/// let mut buffer = [0u8; 1024]; +/// let n = udp_stream.read(&mut buffer).await?; +/// +/// // Write data +/// udp_stream.write_all(b"Response").await?; +/// } +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug)] +pub struct IpStackUdpStream { + src_addr: SocketAddr, + dst_addr: SocketAddr, + stream_sender: PacketSender, + stream_receiver: PacketReceiver, + up_pkt_sender: PacketSender, + first_payload: Option>, + timeout: Pin>, + timeout_interval: Duration, + mtu: u16, + destroy_messenger: Option<::tokio::sync::oneshot::Sender<()>>, +} + +impl IpStackUdpStream { + pub fn new( + src_addr: SocketAddr, + dst_addr: SocketAddr, + payload: Vec, + up_pkt_sender: PacketSender, + mtu: u16, + timeout_interval: Duration, + destroy_messenger: Option<::tokio::sync::oneshot::Sender<()>>, + ) -> Self { + let (stream_sender, stream_receiver) = mpsc::unbounded_channel::(); + let deadline = tokio::time::Instant::now() + timeout_interval; + IpStackUdpStream { + src_addr, + dst_addr, + stream_sender, + stream_receiver, + up_pkt_sender, + first_payload: Some(payload), + timeout: Box::pin(tokio::time::sleep_until(deadline)), + timeout_interval, + mtu, + destroy_messenger, + } + } + + pub(crate) fn stream_sender(&self) -> PacketSender { + self.stream_sender.clone() + } + + fn create_rev_packet(&self, ttl: u8, mut payload: Vec) -> std::io::Result { + const UHS: usize = 8; // udp header size is 8 + match (self.dst_addr.ip(), self.src_addr.ip()) { + (std::net::IpAddr::V4(dst), std::net::IpAddr::V4(src)) => { + let mut ip_h = Ipv4Header::new(0, ttl, IpNumber::UDP, dst.octets(), src.octets()).map_err(IpStackError::from)?; + let line_buffer = self.mtu.saturating_sub((ip_h.header_len() + UHS) as u16); + payload.truncate(line_buffer as usize); + ip_h.set_payload_len(payload.len() + UHS).map_err(IpStackError::from)?; + let udp_header = UdpHeader::with_ipv4_checksum(self.dst_addr.port(), self.src_addr.port(), &ip_h, &payload) + .map_err(IpStackError::from)?; + Ok(NetworkPacket { + ip: IpHeader::Ipv4(ip_h), + transport: TransportHeader::Udp(udp_header), + payload: Some(payload), + }) + } + (std::net::IpAddr::V6(dst), std::net::IpAddr::V6(src)) => { + let mut ip_h = Ipv6Header { + traffic_class: 0, + flow_label: Ipv6FlowLabel::ZERO, + payload_length: 0, + next_header: IpNumber::UDP, + hop_limit: ttl, + source: dst.octets(), + destination: src.octets(), + }; + let line_buffer = self.mtu.saturating_sub((ip_h.header_len() + UHS) as u16); + + payload.truncate(line_buffer as usize); + + ip_h.payload_length = (payload.len() + UHS) as u16; + let udp_header = UdpHeader::with_ipv6_checksum(self.dst_addr.port(), self.src_addr.port(), &ip_h, &payload) + .map_err(IpStackError::from)?; + Ok(NetworkPacket { + ip: IpHeader::Ipv6(ip_h), + transport: TransportHeader::Udp(udp_header), + payload: Some(payload), + }) + } + _ => unreachable!(), + } + } + + /// Returns the local socket address of the UDP stream. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackUdpStream; + /// # fn example(udp_stream: &IpStackUdpStream) { + /// let local_addr = udp_stream.local_addr(); + /// println!("Local address: {}", local_addr); + /// # } + /// ``` + pub fn local_addr(&self) -> SocketAddr { + self.src_addr + } + + /// Returns the remote socket address of the UDP stream. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackUdpStream; + /// # fn example(udp_stream: &IpStackUdpStream) { + /// let peer_addr = udp_stream.peer_addr(); + /// println!("Peer address: {}", peer_addr); + /// # } + /// ``` + pub fn peer_addr(&self) -> SocketAddr { + self.dst_addr + } + + fn reset_timeout(&mut self) { + let deadline = tokio::time::Instant::now() + self.timeout_interval; + self.timeout.as_mut().reset(deadline); + } +} + +impl AsyncRead for IpStackUdpStream { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + if let Some(p) = self.first_payload.take() { + // Clamp to `buf.remaining()`: an oversized datagram would otherwise + // panic `put_slice` (hit under `copy_bidirectional` write + // backpressure). Unlike the TCP path the remainder is DROPPED, not buffered matches `recvfrom` without MSG_WAITALL. + let n = p.len().min(buf.remaining()); + buf.put_slice(&p[..n]); + return std::task::Poll::Ready(Ok(())); + } + if matches!(self.timeout.as_mut().poll(cx), std::task::Poll::Ready(_)) { + return std::task::Poll::Ready(Err(std::io::Error::from(std::io::ErrorKind::TimedOut))); + } + + self.reset_timeout(); + + match self.stream_receiver.poll_recv(cx) { + std::task::Poll::Ready(Some(p)) => { + if let Some(payload) = p.payload { + // Clamp like the first_payload branch above (drop the tail). + let n = payload.len().min(buf.remaining()); + buf.put_slice(&payload[..n]); + } + std::task::Poll::Ready(Ok(())) + } + std::task::Poll::Ready(None) => std::task::Poll::Ready(Ok(())), + std::task::Poll::Pending => std::task::Poll::Pending, + } + } +} + +impl AsyncWrite for IpStackUdpStream { + fn poll_write(mut self: Pin<&mut Self>, _cx: &mut std::task::Context<'_>, buf: &[u8]) -> std::task::Poll> { + self.reset_timeout(); + let packet = self.create_rev_packet(TTL, buf.to_vec())?; + let payload_len = packet.payload.as_ref().map(|p| p.len()).unwrap_or(0); + self.up_pkt_sender.send(packet).or(Err(std::io::ErrorKind::UnexpectedEof))?; + std::task::Poll::Ready(Ok(payload_len)) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } +} + +impl Drop for IpStackUdpStream { + fn drop(&mut self) { + if let Some(messenger) = self.destroy_messenger.take() { + let _ = messenger.send(()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::AsyncReadExt; + + fn stream(first_payload: Vec) -> IpStackUdpStream { + let (up_tx, _up_rx) = mpsc::unbounded_channel(); + IpStackUdpStream::new( + "127.0.0.1:1234".parse().unwrap(), + "127.0.0.1:53".parse().unwrap(), + first_payload, + up_tx, + 1500, + Duration::from_secs(30), + None, + ) + } + + // A datagram larger than the caller's buffer used to panic `put_slice`; it + // must now truncate to the buffer instead. Covers both branches of poll_read. + + #[tokio::test] + async fn poll_read_truncates_oversized_first_payload() { + let mut s = stream(vec![7u8; 1250]); + let mut small = [0u8; 502]; + assert_eq!(s.read(&mut small).await.unwrap(), 502); + assert!(small.iter().all(|&b| b == 7)); + } + + #[tokio::test] + async fn poll_read_truncates_oversized_relayed_datagram() { + let mut s = stream(Vec::new()); + s.first_payload = None; // skip to the stream_receiver branch + let pkt = s.create_rev_packet(64, vec![9u8; 1250]).unwrap(); + s.stream_sender().send(pkt).unwrap(); + let mut small = [0u8; 502]; + assert_eq!(s.read(&mut small).await.unwrap(), 502); + } +} diff --git a/third_party/ipstack/src/stream/unknown.rs b/third_party/ipstack/src/stream/unknown.rs new file mode 100644 index 00000000..a61c0bf6 --- /dev/null +++ b/third_party/ipstack/src/stream/unknown.rs @@ -0,0 +1,214 @@ +use crate::{ + IpStackError, PacketSender, TTL, + packet::{IpHeader, NetworkPacket, TransportHeader}, +}; +use etherparse::{IpNumber, Ipv4Header, Ipv6FlowLabel, Ipv6Header}; +use std::net::IpAddr; + +/// A stream for unknown transport layer protocols. +/// +/// This type handles network packets with transport protocols that are not TCP or UDP +/// (e.g., ICMP, IGMP, ESP, etc.). It provides methods to inspect the packet details +/// and send responses. +/// +/// # Examples +/// +/// ```no_run +/// use ipstack::{IpStack, IpStackConfig, IpStackStream}; +/// +/// # async fn example(mut ip_stack: IpStack) -> Result<(), Box> { +/// if let IpStackStream::UnknownTransport(unknown) = ip_stack.accept().await? { +/// println!("Unknown transport protocol: {:?}", unknown.ip_protocol()); +/// println!("Source: {}", unknown.src_addr()); +/// println!("Destination: {}", unknown.dst_addr()); +/// println!("Payload: {} bytes", unknown.payload().len()); +/// +/// // Send a response +/// unknown.send(vec![0x08, 0x00, 0x00, 0x00])?; +/// } +/// # Ok(()) +/// # } +/// ``` +pub struct IpStackUnknownTransport { + src_addr: IpAddr, + dst_addr: IpAddr, + payload: Vec, + protocol: IpNumber, + mtu: u16, + packet_sender: PacketSender, +} + +impl IpStackUnknownTransport { + pub(crate) fn new(src_addr: IpAddr, dst_addr: IpAddr, payload: Vec, ip: &IpHeader, mtu: u16, packet_sender: PacketSender) -> Self { + let protocol = match ip { + IpHeader::Ipv4(ip) => ip.protocol, + IpHeader::Ipv6(ip) => ip.next_header, + }; + IpStackUnknownTransport { + src_addr, + dst_addr, + payload, + protocol, + mtu, + packet_sender, + } + } + + /// Returns the source IP address of the packet. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackUnknownTransport; + /// # fn example(unknown: &IpStackUnknownTransport) { + /// let src = unknown.src_addr(); + /// println!("Source: {}", src); + /// # } + /// ``` + pub fn src_addr(&self) -> IpAddr { + self.src_addr + } + + /// Returns the destination IP address of the packet. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackUnknownTransport; + /// # fn example(unknown: &IpStackUnknownTransport) { + /// let dst = unknown.dst_addr(); + /// println!("Destination: {}", dst); + /// # } + /// ``` + pub fn dst_addr(&self) -> IpAddr { + self.dst_addr + } + + /// Returns the payload of the packet. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackUnknownTransport; + /// # fn example(unknown: &IpStackUnknownTransport) { + /// let payload = unknown.payload(); + /// println!("Payload: {} bytes", payload.len()); + /// # } + /// ``` + pub fn payload(&self) -> &[u8] { + &self.payload + } + + /// Returns the IP protocol number of the packet. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackUnknownTransport; + /// # fn example(unknown: &IpStackUnknownTransport) { + /// let protocol = unknown.ip_protocol(); + /// println!("Protocol: {:?}", protocol); + /// # } + /// ``` + pub fn ip_protocol(&self) -> IpNumber { + self.protocol + } + + /// Send a response packet. + /// + /// This method sends one or more packets with the given payload, automatically + /// fragmenting the data if it exceeds the MTU. + /// + /// # Arguments + /// + /// * `payload` - The payload to send + /// + /// # Errors + /// + /// Returns an error if the packet cannot be sent. + /// + /// # Examples + /// + /// ```no_run + /// # use ipstack::IpStackUnknownTransport; + /// # fn example(unknown: &IpStackUnknownTransport) -> std::io::Result<()> { + /// // Send an ICMP echo reply + /// unknown.send(vec![0x08, 0x00, 0x00, 0x00])?; + /// # Ok(()) + /// # } + /// ``` + pub fn send(&self, mut payload: Vec) -> std::io::Result<()> { + loop { + let packet = self.create_rev_packet(&mut payload)?; + self.packet_sender + .send(packet) + .map_err(|e| std::io::Error::other(format!("send error: {e}")))?; + if payload.is_empty() { + return Ok(()); + } + } + } + + /// Create a reverse packet for sending a response. + /// + /// This method creates a packet with swapped source and destination addresses, + /// suitable for sending responses to received packets. If the payload exceeds + /// the MTU, only a portion of the payload is consumed and included in the packet. + /// + /// # Arguments + /// + /// * `payload` - A mutable reference to the payload vector. If the payload exceeds + /// the MTU, data is drained from the front. Otherwise, the entire vector is taken. + /// + /// # Returns + /// + /// Returns a `NetworkPacket` with the reversed addresses and up to MTU bytes of payload. + /// + /// # Errors + /// + /// Returns an error if the packet cannot be constructed. + pub fn create_rev_packet(&self, payload: &mut Vec) -> std::io::Result { + match (self.dst_addr, self.src_addr) { + (std::net::IpAddr::V4(dst), std::net::IpAddr::V4(src)) => { + let mut ip_h = Ipv4Header::new(0, TTL, self.protocol, dst.octets(), src.octets()).map_err(IpStackError::from)?; + let line_buffer = self.mtu.saturating_sub(ip_h.header_len() as u16); + + let p = if payload.len() > line_buffer as usize { + payload.drain(0..line_buffer as usize).collect::>() + } else { + std::mem::take(payload) + }; + ip_h.set_payload_len(p.len()).map_err(IpStackError::from)?; + Ok(NetworkPacket { + ip: IpHeader::Ipv4(ip_h), + transport: TransportHeader::Unknown, + payload: Some(p), + }) + } + (std::net::IpAddr::V6(dst), std::net::IpAddr::V6(src)) => { + let mut ip_h = Ipv6Header { + traffic_class: 0, + flow_label: Ipv6FlowLabel::ZERO, + payload_length: 0, + next_header: self.protocol, + hop_limit: TTL, + source: dst.octets(), + destination: src.octets(), + }; + let line_buffer = self.mtu.saturating_sub(ip_h.header_len() as u16); + let p = if payload.len() > line_buffer as usize { + payload.drain(0..line_buffer as usize).collect::>() + } else { + std::mem::take(payload) + }; + ip_h.set_payload_length(p.len()).map_err(IpStackError::from)?; + Ok(NetworkPacket { + ip: IpHeader::Ipv6(ip_h), + transport: TransportHeader::Unknown, + payload: Some(p), + }) + } + _ => unreachable!(), + } + } +} diff --git a/third_party/libkrun/VENDOR.md b/third_party/libkrun/VENDOR.md index 7b04d75a..b648233b 100644 --- a/third_party/libkrun/VENDOR.md +++ b/third_party/libkrun/VENDOR.md @@ -261,3 +261,22 @@ same page-cache mapping, and mmap still enforces that a writable mapping needs a The reopen fallback stays for a mapping whose inode has no open fd (a read after close). The lookup keys on the inode and access mode, not on the guest's handle, which is absent for DAX. Covered by the `setupmapping_*` tests. Search for `The guest passes fh = u64::MAX`. + +`src/devices/src/virtio/fs/read_only.rs` + `src/devices/src/virtio/fs/linux/passthrough.rs` — +REMOVEMAPPING on a read-only share keeps the DAX mapping in place (bounds still checked): +tearing a range down was one mmap over the window plus a KVM invalidation, and a guest reading +a source tree reclaims a range for nearly every file once its window is full — 30k files cost +57k host mmaps, 47% of them removals. The next SETUPMAPPING replaces a kept mapping with +MAP_FIXED, and a read-only share cannot be written through it. The read-write path merges the +adjacent ranges of a batch into one mmap (`merge_mappings`). Covered by +`removemapping_keeps_the_mapping_but_checks_bounds` and +`removemapping_batches_merge_adjacent_ranges`. + +`src/libkrun/src/lib.rs` + `src/vmm/src/vmm_config/fs.rs` + `src/devices/src/virtio/fs/` +(`device.rs`, `fuse.rs`, `linux/passthrough.rs`) — per-inode DAX by file size. +`krun_add_virtiofs5(…, dax_inode_min)` (`krun_add_virtiofs4` delegates with 0) carries a size +floor to the passthrough filesystem, which then answers INIT with `HAS_INODE_DAX` when the +guest offers it (a `dax=inode` mount) and sets `ATTR_DAX` on the entries of regular files at or +above the floor, so only those are mapped through the window. Every DAX mapping costs a host +mmap and a guest EPT invalidation per 2 MiB range whatever the file's size; a source tree's +small files never repay it. Covered by `lookup_marks_large_regular_files_for_dax`. diff --git a/third_party/libkrun/src/devices/src/virtio/fs/device.rs b/third_party/libkrun/src/devices/src/virtio/fs/device.rs index 37d2dc4e..8714b008 100644 --- a/third_party/libkrun/src/devices/src/virtio/fs/device.rs +++ b/third_party/libkrun/src/devices/src/virtio/fs/device.rs @@ -70,6 +70,7 @@ impl Fs { gid_map: Vec, virtual_entries: Vec, negative_timeout_ms: u32, + dax_inode_min: Option, ) -> super::Result { let avail_features = (1u64 << VIRTIO_F_VERSION_1) | (1u64 << VIRTIO_RING_F_EVENT_IDX); @@ -81,6 +82,7 @@ impl Fs { let fs_cfg = shared_dir.map(|root_dir| passthrough::Config { root_dir, negative_timeout: std::time::Duration::from_millis(negative_timeout_ms.into()), + dax_inode_min, ..Default::default() }); diff --git a/third_party/libkrun/src/devices/src/virtio/fs/fuse.rs b/third_party/libkrun/src/devices/src/virtio/fs/fuse.rs index 442cdad9..610cb3d2 100644 --- a/third_party/libkrun/src/devices/src/virtio/fs/fuse.rs +++ b/third_party/libkrun/src/devices/src/virtio/fs/fuse.rs @@ -530,6 +530,9 @@ pub const FUSE_COMPAT_22_INIT_OUT_SIZE: u32 = 24; /// Object is a submount root pub const ATTR_SUBMOUNT: u32 = 1; +/// Enable DAX for this file in per-inode DAX mode (the guest mounted `dax=inode` and the +/// server answered INIT with `HAS_INODE_DAX`). +pub const ATTR_DAX: u32 = 2; /// Kill suid and sgid if executable pub const OPEN_KILL_SUIDGID: u32 = 1; diff --git a/third_party/libkrun/src/devices/src/virtio/fs/linux/passthrough.rs b/third_party/libkrun/src/devices/src/virtio/fs/linux/passthrough.rs index 3d3ae862..4a32a9c7 100644 --- a/third_party/libkrun/src/devices/src/virtio/fs/linux/passthrough.rs +++ b/third_party/libkrun/src/devices/src/virtio/fs/linux/passthrough.rs @@ -391,6 +391,11 @@ pub struct Config { pub export_fsid: u64, /// Table of exported FDs to share with other subsystems. pub export_table: Option, + + /// Per-inode DAX: `Some(bytes)` marks regular files at least this large for DAX + /// (`ATTR_DAX` on their entries, `HAS_INODE_DAX` at INIT) so a guest mounted + /// `dax=inode` maps only those; `None` leaves DAX to the mount option alone. + pub dax_inode_min: Option, } impl Default for Config { @@ -406,6 +411,7 @@ impl Default for Config { proc_sfd_rawfd: None, export_fsid: 0, export_table: None, + dax_inode_min: None, } } } @@ -877,6 +883,27 @@ fn forget_one( } } +/// Coalesce the `(moffset, len)` window ranges of a REMOVEMAPPING batch: sorted by offset, +/// touching or overlapping ranges merged, so each run costs one mmap. Zero-length ranges drop +/// out. Callers bounds-check `moffset + len` against the window first; the `saturating_add` +/// only keeps an unchecked caller from overflowing. +pub(crate) fn merge_mappings(requests: &[fuse::RemovemappingOne]) -> Vec<(u64, u64)> { + let mut ranges: Vec<(u64, u64)> = requests + .iter() + .filter(|r| r.len > 0) + .map(|r| (r.moffset, r.moffset.saturating_add(r.len))) + .collect(); + ranges.sort_unstable(); + let mut merged: Vec<(u64, u64)> = Vec::with_capacity(ranges.len()); + for (start, end) in ranges { + match merged.last_mut() { + Some((_, last_end)) if start <= *last_end => *last_end = (*last_end).max(end), + _ => merged.push((start, end)), + } + } + merged.into_iter().map(|(s, e)| (s, e - s)).collect() +} + impl FileSystem for PassthroughFs { type Inode = Inode; type Handle = Handle; @@ -933,6 +960,12 @@ impl FileSystem for PassthroughFs { self.writeback.store(true, Ordering::Relaxed); } + // The guest offers HAS_INODE_DAX when mounted `dax=inode`; taking it is what makes it + // honour ATTR_DAX on the entries below. Without a floor configured, decline it and the + // mount option alone decides (`always` maps every file, `inode` none). + if self.cfg.dax_inode_min.is_some() && capable.contains(FsOptions::HAS_INODE_DAX) { + opts |= FsOptions::HAS_INODE_DAX; + } if capable.contains(FsOptions::SUBMOUNTS) { opts |= FsOptions::SUBMOUNTS; self.announce_submounts.store(true, Ordering::Relaxed); @@ -1018,6 +1051,17 @@ impl FileSystem for PassthroughFs { { attr_flags |= fuse::ATTR_SUBMOUNT; } + // Per-inode DAX by size: mapping a 2 MiB range costs the same for a 2 KiB file as for + // a large one, and only the large one repays it. The kernel decides DAX for an inode + // from the entry that instantiates it, and every entry-creating path (lookup, + // readdirplus, create) comes through here. + if let Some(min) = self.cfg.dax_inode_min { + if st.st_mode & libc::S_IFMT == libc::S_IFREG + && u64::try_from(st.st_size).is_ok_and(|size| size >= min) + { + attr_flags |= fuse::ATTR_DAX; + } + } let altkey = InodeAltKey { ino: st.st_ino, @@ -2191,16 +2235,25 @@ impl FileSystem for PassthroughFs { host_shm_base: u64, shm_size: u64, ) -> io::Result<()> { - for req in requests { - let addr = host_shm_base + req.moffset; - if (req.moffset + req.len) > shm_size { + for req in &requests { + if req + .moffset + .checked_add(req.len) + .is_none_or(|end| end > shm_size) + { return Err(einval()); } - debug!("removemapping: addr={:x} len={:?}", addr, req.len); + } + // The guest reclaims DAX ranges in batches; each range torn down is one mmap over + // the window and one KVM invalidation of that guest-physical span. Adjacent ranges + // in a batch are torn down with a single call. + for (moffset, len) in merge_mappings(&requests) { + let addr = host_shm_base + moffset; + debug!("removemapping: addr={addr:x} len={len}"); let ret = unsafe { libc::mmap( addr as *mut libc::c_void, - req.len as usize, + len as usize, libc::PROT_NONE, libc::MAP_ANONYMOUS | libc::MAP_PRIVATE | libc::MAP_FIXED, -1, @@ -2531,4 +2584,85 @@ mod tests { unsafe { libc::munmap(base, page) }; std::fs::remove_dir_all(&root).ok(); } + + /// Touching and overlapping window ranges of one REMOVEMAPPING batch collapse into single + /// runs, in offset order; zero-length entries drop out. + #[test] + fn removemapping_batches_merge_adjacent_ranges() { + let r = |moffset, len| fuse::RemovemappingOne { moffset, len }; + let two_mib = 2u64 << 20; + let merged = merge_mappings(&[ + r(4 * two_mib, two_mib), + r(0, two_mib), + r(two_mib, two_mib), + r(9 * two_mib, 0), + r(4 * two_mib + 4096, two_mib), + ]); + assert_eq!( + merged, + vec![(0, 2 * two_mib), (4 * two_mib, two_mib + 4096)] + ); + assert!(merge_mappings(&[]).is_empty()); + } + + /// With a size floor, lookup marks the regular files at or above it for DAX and nothing + /// else, and INIT takes HAS_INODE_DAX only from a guest that offers it. + #[test] + fn lookup_marks_large_regular_files_for_dax() { + let root_dir = tmp_root(); + let root = std::path::Path::new(&root_dir); + std::fs::write(root.join("small"), vec![0u8; 4095]).unwrap(); + std::fs::write(root.join("exact"), vec![0u8; 4096]).unwrap(); + std::fs::write(root.join("large"), vec![0u8; 65536]).unwrap(); + std::fs::create_dir(root.join("dir")).unwrap(); + let with_floor = |floor: Option| { + PassthroughFs::new( + Config { + root_dir: root_dir.clone(), + dax_inode_min: floor, + ..Default::default() + }, + Arc::new(InodeAllocator::new()), + ) + .unwrap() + }; + let fs = with_floor(Some(4096)); + let taken = fs + .init(FsOptions::HAS_INODE_DAX | FsOptions::SUBMOUNTS) + .unwrap(); + assert!(taken.contains(FsOptions::HAS_INODE_DAX)); + let flags = |name: &CStr| fs.lookup(ctx(), fuse::ROOT_ID, name).unwrap().attr_flags; + assert_eq!(flags(c"small") & fuse::ATTR_DAX, 0, "under the floor"); + assert_eq!( + flags(c"exact") & fuse::ATTR_DAX, + fuse::ATTR_DAX, + "at the floor" + ); + assert_eq!(flags(c"large") & fuse::ATTR_DAX, fuse::ATTR_DAX); + assert_eq!( + flags(c"dir") & fuse::ATTR_DAX, + 0, + "directories are never DAX" + ); + // A guest mounted `dax=always` offers no HAS_INODE_DAX; the server must not claim it. + let fs2 = with_floor(Some(4096)); + assert!(!fs2 + .init(FsOptions::SUBMOUNTS) + .unwrap() + .contains(FsOptions::HAS_INODE_DAX)); + // Without a floor the flag is neither taken nor set, whatever the guest offers. + let fs3 = with_floor(None); + assert!(!fs3 + .init(FsOptions::HAS_INODE_DAX) + .unwrap() + .contains(FsOptions::HAS_INODE_DAX)); + assert_eq!( + fs3.lookup(ctx(), fuse::ROOT_ID, c"large") + .unwrap() + .attr_flags + & fuse::ATTR_DAX, + 0 + ); + std::fs::remove_dir_all(root).ok(); + } } diff --git a/third_party/libkrun/src/devices/src/virtio/fs/read_only.rs b/third_party/libkrun/src/devices/src/virtio/fs/read_only.rs index 5495db1e..8c853a03 100644 --- a/third_party/libkrun/src/devices/src/virtio/fs/read_only.rs +++ b/third_party/libkrun/src/devices/src/virtio/fs/read_only.rs @@ -36,6 +36,10 @@ fn erofs() -> io::Error { io::Error::from_raw_os_error(libc::EROFS) } +fn einval() -> io::Error { + io::Error::from_raw_os_error(libc::EINVAL) +} + fn read_only_open_flags(flags: u32) -> io::Result { let f = flags as i32; if f & libc::O_ACCMODE != libc::O_RDONLY { @@ -293,14 +297,26 @@ impl FileSystem for PassthroughFsRo { shm_size: u64, #[cfg(target_os = "macos")] map_sender: &Option>, ) -> io::Result<()> { - self.inner.removemapping( - ctx, - requests, - host_shm_base, - shm_size, - #[cfg(target_os = "macos")] - map_sender, - ) + // A read-only share keeps its DAX mappings in place. Tearing one down is an mmap over + // the window and a KVM invalidation of that span, and the guest reclaims a range for + // nearly every small file it reads once its window is full — tens of thousands of + // calls over a source tree, none of which buys anything here: the mapping is a + // read-only view of a file the guest may read anyway, the next SETUPMAPPING replaces + // it in place (MAP_FIXED), and the file cannot be written through it. Only the + // request's bounds are still checked, so a malformed batch is refused as before. + let _ = (ctx, host_shm_base); + #[cfg(target_os = "macos")] + let _ = map_sender; + for req in &requests { + if req + .moffset + .checked_add(req.len) + .is_none_or(|end| end > shm_size) + { + return Err(einval()); + } + } + Ok(()) } fn ioctl( @@ -498,4 +514,76 @@ mod tests { assert_eq!(err.raw_os_error(), Some(libc::EROFS)); } + + /// REMOVEMAPPING on a read-only share leaves the DAX mapping in place — the window still + /// reads the file afterwards — while a request past the window is still refused. + #[test] + fn removemapping_keeps_the_mapping_but_checks_bounds() { + use super::super::filesystem::{Context, FileSystem, FsOptions}; + use super::super::fuse; + use super::super::inode_alloc::InodeAllocator; + use super::super::passthrough::Config; + use super::PassthroughFsRo; + use std::sync::Arc; + + let root = std::env::temp_dir().join(format!("vk-ro-dax-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + let page = 4096usize; + std::fs::write(root.join("f"), vec![0x5Au8; page]).unwrap(); + let cfg = Config { + root_dir: root.to_str().unwrap().to_string(), + ..Default::default() + }; + let fs = PassthroughFsRo::new(cfg, Arc::new(InodeAllocator::new())).unwrap(); + fs.init(FsOptions::empty()).unwrap(); + let ctx = Context { + uid: 0, + gid: 0, + pid: 0, + }; + let entry = fs.lookup(ctx, fuse::ROOT_ID, c"f").unwrap(); + let base = unsafe { + libc::mmap( + std::ptr::null_mut(), + page, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + assert!(!std::ptr::eq(base, libc::MAP_FAILED)); + fs.setupmapping( + ctx, + entry.inode, + u64::MAX, + 0, + page as u64, + 0, + 0, + base as u64, + page as u64, + ) + .unwrap(); + assert_eq!( + unsafe { *(base as *const u8) }, + 0x5A, + "the window maps the file" + ); + let one = |moffset, len| vec![fuse::RemovemappingOne { moffset, len }]; + fs.removemapping(ctx, one(0, page as u64), base as u64, page as u64) + .unwrap(); + assert_eq!( + unsafe { *(base as *const u8) }, + 0x5A, + "the mapping stays: no PROT_NONE remap" + ); + let err = fs + .removemapping(ctx, one(0, page as u64 + 1), base as u64, page as u64) + .unwrap_err(); + assert_eq!(err.raw_os_error(), Some(libc::EINVAL)); + unsafe { libc::munmap(base, page) }; + let _ = std::fs::remove_dir_all(&root); + } } diff --git a/third_party/libkrun/src/libkrun/src/lib.rs b/third_party/libkrun/src/libkrun/src/lib.rs index 77d70c94..2969f91e 100644 --- a/third_party/libkrun/src/libkrun/src/lib.rs +++ b/third_party/libkrun/src/libkrun/src/lib.rs @@ -644,6 +644,7 @@ pub unsafe extern "C" fn krun_set_root(ctx_id: u32, c_root_path: *const c_char) uid_map: Vec::new(), gid_map: Vec::new(), negative_timeout_ms: 0, + dax_inode_min: None, virtual_entries: { #[allow(unused_mut)] let mut v = Vec::new(); @@ -720,6 +721,27 @@ pub unsafe extern "C" fn krun_add_virtiofs4( read_only: bool, c_uid_map: *const c_char, c_gid_map: *const c_char, +) -> i32 { + krun_add_virtiofs5( + ctx_id, c_tag, c_path, shm_size, read_only, c_uid_map, c_gid_map, 0, + ) +} + +/// `krun_add_virtiofs4` plus per-inode DAX: `dax_inode_min` > 0 marks regular files of at +/// least that many bytes for DAX (`ATTR_DAX` on their entries, `HAS_INODE_DAX` at INIT), for +/// a guest that mounts the share `dax=inode`; 0 leaves DAX to the mount option. +#[allow(clippy::missing_safety_doc)] +#[no_mangle] +#[cfg(not(any(feature = "tee", feature = "aws-nitro")))] +pub unsafe extern "C" fn krun_add_virtiofs5( + ctx_id: u32, + c_tag: *const c_char, + c_path: *const c_char, + shm_size: u64, + read_only: bool, + c_uid_map: *const c_char, + c_gid_map: *const c_char, + dax_inode_min: u64, ) -> i32 { if c_tag.is_null() { return -libc::EINVAL; @@ -788,6 +810,7 @@ pub unsafe extern "C" fn krun_add_virtiofs4( gid_map, virtual_entries, negative_timeout_ms: 0, + dax_inode_min: (dax_inode_min > 0).then_some(dax_inode_min), }); } Entry::Vacant(_) => return -libc::ENOENT, @@ -2537,6 +2560,7 @@ pub unsafe extern "C" fn krun_set_root_disk_remount( gid_map: Vec::new(), virtual_entries, negative_timeout_ms: 0, + dax_inode_min: None, }); ctx_cfg.set_block_root(device, fstype, options); diff --git a/third_party/libkrun/src/vmm/src/builder.rs b/third_party/libkrun/src/vmm/src/builder.rs index f7e7b9ab..30a5bb35 100644 --- a/third_party/libkrun/src/vmm/src/builder.rs +++ b/third_party/libkrun/src/vmm/src/builder.rs @@ -2010,6 +2010,7 @@ fn attach_fs_devices( config.gid_map.clone(), config.virtual_entries.clone(), config.negative_timeout_ms, + config.dax_inode_min, ) .unwrap(), )); diff --git a/third_party/libkrun/src/vmm/src/vmm_config/fs.rs b/third_party/libkrun/src/vmm/src/vmm_config/fs.rs index 3b92fdfa..64b28df4 100644 --- a/third_party/libkrun/src/vmm/src/vmm_config/fs.rs +++ b/third_party/libkrun/src/vmm/src/vmm_config/fs.rs @@ -18,4 +18,8 @@ pub struct FsDeviceConfig { /// How long (ms) the guest may cache a failed (ENOENT) lookup. `0` = no caching /// (the previous behavior: every miss round-trips). pub negative_timeout_ms: u32, + /// Per-inode DAX: regular files at least this many bytes are marked for DAX + /// (`ATTR_DAX`), for a guest that mounts the share `dax=inode`. `None` = the mount + /// option alone decides. + pub dax_inode_min: Option, } diff --git a/vk-agent/src/init.rs b/vk-agent/src/init.rs index 0235c095..52bb709e 100644 --- a/vk-agent/src/init.rs +++ b/vk-agent/src/init.rs @@ -30,10 +30,14 @@ //! belongs to eth0 //! VIRTKIT_VIRTIOFS tag:path[,tag:path] virtiofs shares to mount //! VIRTKIT_VIRTIOFS_DAX tag[,tag] — these shares have a DAX window, so mount them -//! `dax=always`: file data is read straight out of the host page +//! through it: file data is read straight out of the host page //! cache through the window instead of being copied into this -//! guest's own. A share whose DAX mount fails is mounted without -//! it (a slower share, not a failed boot) +//! guest's own. `dax=always` unless the tag is also in +//! VIRTKIT_VIRTIOFS_DAX_INODE. A share whose DAX mount fails is +//! mounted without it (a slower share, not a failed boot) +//! VIRTKIT_VIRTIOFS_DAX_INODE tag[,tag] — mount these DAX shares `dax=inode`: the host +//! marks the files worth mapping (regular files above a size +//! floor) and the rest read through this guest's page cache //! VIRTKIT_VIRTIOFS_OVERLAY tag[,tag] — mount these shares as the read-only lower //! layer of a tmpfs-backed overlayfs at their path, so every //! write under the mountpoint runs at guest-native speed. A @@ -1179,6 +1183,17 @@ fn materialize_env(cfg: Option<&RunConfig>) { fn mount_virtiofs(cmdline: &HashMap) -> Result> { let mut overlay = overlay_tags(cmdline)?; let dax = share_tags(cmdline, "VIRTKIT_VIRTIOFS_DAX")?; + let dax_inode = share_tags(cmdline, "VIRTKIT_VIRTIOFS_DAX_INODE")?; + // A share can only be mounted `dax=inode` through a window it has. + let windowless: HashSet = dax_inode.difference(&dax).cloned().collect(); + if !windowless.is_empty() { + warn!( + "vk-agent init: VIRTKIT_VIRTIOFS_DAX_INODE names {}, which has no DAX window; \ + mounting without DAX", + sorted_join(&windowless) + ); + } + let dax_mode = |tag: &str| DaxMount::of(tag, &dax, &dax_inode); let size = overlay_size(cmdline)?; let overlay_disks = overlay_disk_devices(cmdline)?; // Each overlay-upper disk names a tag; that tag must be one of the overlay shares, or the @@ -1215,7 +1230,7 @@ fn mount_virtiofs(cmdline: &HashMap) -> Result> { path, size, overlay_disks.get(tag).map(String::as_str), - dax.contains(tag), + dax_mode(tag), ) .with_context(|| format!("overlay-mounting virtiofs share {tag} at {path}"))?; freeze.extend(upper); @@ -1238,7 +1253,7 @@ fn mount_virtiofs(cmdline: &HashMap) -> Result> { continue; } }; - if let Err(e) = mount_share(tag, path, 0, dax.contains(tag)) { + if let Err(e) = mount_share(tag, path, 0, dax_mode(tag)) { warn!( "vk-agent init: mount virtiofs {tag} at {} failed: {e}", mountpoint.display() @@ -1373,11 +1388,41 @@ fn share_tags(cmdline: &HashMap, key: &str) -> Result Option<&'static str> { - dax.then_some("dax=always") +/// How a virtio-fs share uses its DAX window, from the cmdline's tag sets. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DaxMount { + /// No window. + None, + /// Every file's data goes through the window (`dax=always`). + Always, + /// Only the files the host marks — regular files above its size floor — go through the + /// window; the rest read through this guest's page cache (`dax=inode`). + Inode, +} + +impl DaxMount { + fn of(tag: &str, dax: &HashSet, dax_inode: &HashSet) -> DaxMount { + if !dax.contains(tag) { + DaxMount::None + } else if dax_inode.contains(tag) { + DaxMount::Inode + } else { + DaxMount::Always + } + } +} + +/// A virtio-fs share's mount options: `dax=always` or `dax=inode` where the share has a DAX +/// window, none otherwise. `always` rather than `inode` for the whole-share case, so a +/// kernel or a device that cannot honour it says so (`inode` would quietly mount without the +/// window and leave nothing to fall back from); a per-inode share accepts that silence — a +/// file the host does not mark reads through the page cache either way. +fn virtiofs_data(dax: DaxMount) -> Option<&'static str> { + match dax { + DaxMount::None => None, + DaxMount::Always => Some("dax=always"), + DaxMount::Inode => Some("dax=inode"), + } } /// Mount virtio-fs share `tag` at `path`, through its DAX window when it has one. @@ -1388,7 +1433,7 @@ fn virtiofs_data(dax: bool) -> Option<&'static str> { /// a reason to fail the boot. The fallback is unconditional rather than keyed on the errno: /// a mount that fails for some other reason fails again and reports itself properly, which /// beats guessing which errnos mean "no DAX here". -fn mount_share(tag: &str, path: &str, flags: libc::c_ulong, dax: bool) -> io::Result<()> { +fn mount_share(tag: &str, path: &str, flags: libc::c_ulong, dax: DaxMount) -> io::Result<()> { if let Some(data) = virtiofs_data(dax) { match mount_data(tag, path, "virtiofs", flags, data) { Ok(()) => { @@ -1489,7 +1534,7 @@ fn mount_share_overlay( path: &str, size: Option<&str>, upper_device: Option<&str>, - dax: bool, + dax: DaxMount, ) -> Result> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let OverlayDirs { @@ -3089,10 +3134,19 @@ mod tests { #[test] fn a_dax_share_asks_for_the_window_and_the_others_ask_for_nothing() { - // `always` and not `inode`: a kernel that cannot honour the option must fail the mount - // so mount_share can retry without it, which `inode` would not do. - assert_eq!(virtiofs_data(true), Some("dax=always")); - assert_eq!(virtiofs_data(false), None); + // `always` and not `inode` for a whole-share window: a kernel that cannot honour the + // option must fail the mount so mount_share can retry without it. + assert_eq!(virtiofs_data(DaxMount::Always), Some("dax=always")); + assert_eq!(virtiofs_data(DaxMount::Inode), Some("dax=inode")); + assert_eq!(virtiofs_data(DaxMount::None), None); + // The mode of a tag comes from the two cmdline sets; a tag named for per-inode DAX + // without a window is not a DAX share at all. + let set = |tags: &[&str]| tags.iter().map(|t| t.to_string()).collect::>(); + let (dax, inode) = (set(&["work", "big"]), set(&["work", "stray"])); + assert_eq!(DaxMount::of("work", &dax, &inode), DaxMount::Inode); + assert_eq!(DaxMount::of("big", &dax, &inode), DaxMount::Always); + assert_eq!(DaxMount::of("stray", &dax, &inode), DaxMount::None); + assert_eq!(DaxMount::of("other", &dax, &inode), DaxMount::None); } #[test] diff --git a/vk-core/src/pty.rs b/vk-core/src/pty.rs index 287d1a5a..b0bc25df 100644 --- a/vk-core/src/pty.rs +++ b/vk-core/src/pty.rs @@ -187,7 +187,7 @@ impl Drop for RawModeGuard { #[cfg(test)] mod tests { use super::openpty; - use std::os::fd::AsRawFd; + use std::os::fd::{AsRawFd, RawFd}; use std::process::Stdio; use tokio::io::AsyncReadExt; @@ -207,10 +207,14 @@ mod tests { #[tokio::test] async fn no_child_inherits_the_master() { let (mut master, slave) = openpty(24, 80).unwrap(); + // Check this master's fd: the child may inherit unrelated PTY masters (in vscode terminal for example). + let master_fd = master.as_raw_fd(); + // List each child fd as ` `. + const LIST_FDS: &str = + r#"for f in /proc/self/fd/*; do echo "${f##*/} $(readlink "$f" 2>/dev/null)"; done"#; let mut cmd = tokio::process::Command::new("sh"); - // The child's own fds: its stdio is the slave, and a master would show as ptmx. cmd.arg("-c") - .arg("ls -l /proc/self/fd") + .arg(LIST_FDS) .stdin(Stdio::from(slave.try_clone().unwrap())) .stdout(Stdio::from(slave.try_clone().unwrap())) .stderr(Stdio::from(slave)); @@ -230,7 +234,15 @@ mod tests { .unwrap(); let out = String::from_utf8_lossy(&out); assert!(out.contains("/dev/pts/"), "no listing: {out}"); - assert!(!out.contains("ptmx"), "the master reached the child: {out}"); + let leaked = out.lines().any(|line| { + let mut words = line.split_whitespace(); + let fd = words.next().and_then(|fd| fd.parse::().ok()); + fd == Some(master_fd) && words.next() == Some("/dev/ptmx") + }); + assert!( + !leaked, + "the master (fd {master_fd}) reached the child: {out}" + ); } #[tokio::test] diff --git a/vk-driver/config.example.toml b/vk-driver/config.example.toml index 98d10a70..de668a02 100644 --- a/vk-driver/config.example.toml +++ b/vk-driver/config.example.toml @@ -213,7 +213,10 @@ # # cache into the guest, so several VMs cache a shared tree once. # # Reserves guest address space, not memory. At most 64G per guest: # # eight windows at this size; further shares work without DAX. -# # Guests above 63.25G RAM get none. "off" disables it; ignored by +# # Guests above 63.25G RAM get none. A bare size maps only regular +# # files of 1M and more (each mapping costs a host mmap per 2M range, +# # which a small file never repays); "8G:always" maps every file, +# # "8G:inode=64K" moves the floor. "off" disables it; ignored by # # cloud-hypervisor, which has no DAX path # max_cpus = 32 # ceilings for the per-job MICROVM_CPUS/MICROVM_MEM # max_mem = "128G" # variables (unset = jobs cannot exceed cpus/mem above) diff --git a/vk-driver/src/checkout.rs b/vk-driver/src/checkout.rs index 70768bff..d1f51830 100644 --- a/vk-driver/src/checkout.rs +++ b/vk-driver/src/checkout.rs @@ -408,9 +408,34 @@ pub fn ensure(url: &str, ref_name: &str, sha: &str, dest: &Path) -> Result<()> { git(dest, &["update-ref", "--no-deref", "HEAD", sha], "detach")?; git(dest, &["reset", "--hard", sha], "reset")?; git(dest, &["clean", "-ffdx"], "clean")?; + settle_index(dest)?; Ok(()) } +/// Leave the checkout with no racily-clean index entries. +/// +/// `reset --hard` writes the index right after the files, so every file written in the index's +/// own second has an mtime equal to the index timestamp. Git cannot tell from stat data whether +/// such a file changed after it was indexed (racy-git), so it re-reads and re-hashes those +/// entries at the next refresh. Here that refresh would run in the job guest, over virtio-fs, +/// on every job; a fast checkout (a small tree, or a tmpfs host) can leave the whole tree racy. +/// Waiting out the index's second and refreshing once on the host rewrites the index with a +/// later timestamp, so the guest's first git command is a stat pass. The one-second wait +/// assumes ≤1s mtime granularity (tmpfs/ext4, as `checkout_dir` realistically is); on a coarser +/// filesystem it may not clear the second, harmlessly leaving the refresh a no-op. +fn settle_index(dest: &Path) -> Result<()> { + let index = dest.join(".git").join("index"); + // A missing stat or a backward clock step (Err from duration_since) skips the wait: the + // refresh below still runs, degrading to the pre-settle slow path — never wrong, only slower. + if let Ok(written) = std::fs::metadata(&index).and_then(|m| m.modified()) + && let Ok(age) = SystemTime::now().duration_since(written) + && age < Duration::from_secs(1) + { + std::thread::sleep(Duration::from_secs(1) - age); + } + git(dest, &["update-index", "-q", "--refresh"], "refresh") +} + /// A git ref name safe to pass as a fetch argument: git's own rules already forbid these in a /// branch/tag, so rejecting them cannot turn away a legitimate ref. `/` stays allowed /// (`feature/x`); a leading `-`/`+` and refspec/glob metacharacters are refused. @@ -812,4 +837,53 @@ mod tests { assert!(!s.id.exists()); let _ = std::fs::remove_dir_all(&root); } + + /// After `settle_index` no entry is racily clean: the index is strictly newer than every + /// file `reset --hard` wrote, so a refresh in another mount (the job guest) is a stat pass. + #[test] + fn settle_index_leaves_the_index_newer_than_the_tree() { + let repo = root("settle"); + std::fs::create_dir_all(&repo).unwrap(); + let repo = repo.as_path(); + let sh = |args: &[&str]| { + let st = Command::new("git") + .arg("-C") + .arg(repo) + .args(["-c", "user.email=t@t", "-c", "user.name=t"]) + .args(args) + .status() + .unwrap(); + assert!(st.success(), "git {args:?}"); + }; + sh(&["init", "-q"]); + for i in 0..50 { + std::fs::write(repo.join(format!("f{i}")), format!("{i}")).unwrap(); + } + sh(&["add", "-A"]); + sh(&["commit", "-qm", "init"]); + for i in 0..50 { + std::fs::remove_file(repo.join(format!("f{i}"))).unwrap(); + } + sh(&["reset", "-q", "--hard"]); + settle_index(repo).unwrap(); + let index = std::fs::metadata(repo.join(".git/index")) + .unwrap() + .modified() + .unwrap(); + let newest = (0..50) + .map(|i| { + std::fs::metadata(repo.join(format!("f{i}"))) + .unwrap() + .modified() + .unwrap() + }) + .max() + .unwrap(); + assert!( + index > newest, + "index {index:?} not newer than newest file {newest:?}" + ); + // git's own view: refreshing again touches nothing (no racy entries left to re-hash). + sh(&["update-index", "--refresh"]); + } } diff --git a/vk-driver/src/compose.rs b/vk-driver/src/compose.rs index 9630dd78..7582931b 100644 --- a/vk-driver/src/compose.rs +++ b/vk-driver/src/compose.rs @@ -3254,7 +3254,13 @@ mod tests { let u = one("services:\n s:\n image: x\n x-virtkit: { dax: off }\n"); assert_eq!(u.dax, Some(crate::vmm::Dax::Off)); let u = one("services:\n s:\n image: x\n x-virtkit: { dax: 2G }\n"); - assert_eq!(u.dax, Some(crate::vmm::Dax::Window(2 << 30))); + assert_eq!( + u.dax, + Some(crate::vmm::Dax::Inode { + window: 2 << 30, + min: crate::vmm::DAX_INODE_MIN_DEFAULT + }) + ); for bad in ["lots", "1M", "1023K"] { assert!( parse( diff --git a/vk-driver/src/config.rs b/vk-driver/src/config.rs index 4f0e2f21..a3192d1d 100644 --- a/vk-driver/src/config.rs +++ b/vk-driver/src/config.rs @@ -426,10 +426,14 @@ pub struct Vm { /// a minute or two (multi-gen LRU); `"512M"`/`"2G"` or `"5%"` keep that much as a fixed /// floor; `"off"` keeps everything. What goes returns to the host through the balloon. pub reclaim: String, - /// DAX window per virtio-fs share in job and service VMs: a size (default `"8G"`) or - /// `"off"`. Maps the host page cache into guest address space, avoiding a copy per VM. - /// Reserves address space, not memory; libkrun only. Unset by default so cloud-hypervisor - /// can distinguish an explicit request for unsupported DAX from the default. + /// DAX window per virtio-fs share in job and service VMs: a size (default `"8G"`), + /// `"off"`, `":always"` or `":inode="`. Maps the host page cache into + /// guest address space, avoiding a copy per VM. A bare size maps only regular files of + /// 1M and more (`dax=inode`): a mapping costs a host mmap per 2M range whatever the + /// file's size, which a source tree's small files never repay; `:always` maps every + /// file, `:inode=` sets the floor. Reserves address space, not memory; libkrun only. + /// Unset by default so cloud-hypervisor can distinguish an explicit request for + /// unsupported DAX from the default. pub dax: Option, /// Ceilings for the per-job MICROVM_CPUS/MICROVM_MEM variables; unset = /// jobs cannot request more than the cpus/mem defaults above diff --git a/vk-driver/src/libkrun_sys.rs b/vk-driver/src/libkrun_sys.rs index 8b4eefe1..cb1afb08 100644 --- a/vk-driver/src/libkrun_sys.rs +++ b/vk-driver/src/libkrun_sys.rs @@ -32,7 +32,7 @@ use anyhow::{Context, Result, bail}; // >= 0 on success, a negative errno on failure. use krun::{ KRUN_EXIT_GUEST_RESET, krun_add_disk2, krun_add_net_tap, krun_add_net_unixstream, - krun_add_virtiofs4, krun_add_vsock_port2, krun_create_ctx, krun_disable_balloon, + krun_add_virtiofs5, krun_add_vsock_port2, krun_create_ctx, krun_disable_balloon, krun_disable_implicit_init, krun_get_shutdown_eventfd, krun_init_log, krun_set_block_dirty_socket, krun_set_console_output, krun_set_kernel, krun_set_nested_virt, krun_set_pmu, krun_set_vm_config, krun_start_enter, @@ -234,17 +234,20 @@ pub fn boot(spec: &VmSpec) -> Result<()> { for share in &spec.shares { let tag = cstr(&share.tag); let dir = cstr(&share.host_dir.to_string_lossy()); - // The id-map rules for this share, joined by ',' as krun_add_virtiofs4 expects; + // The id-map rules for this share, joined by ',' as krun_add_virtiofs5 expects; // an empty map yields an empty string, which the FFI treats as an identity map. let uid_map = cstr(&share.uid_map.join(",")); let gid_map = cstr(&share.gid_map.join(",")); // shm_size is the share's DAX window, guest address space reserved above RAM // (0 = none). `vmm::apply_dax_budget` has already dropped the windows that do - // not fit the guest's span, so whatever is here is placeable. - let shm_size = share.dax.unwrap_or(0); + // not fit the guest's span, so whatever is here is placeable. dax_inode_min is + // the smallest regular file the server marks for DAX (0 = every file): the + // guest mounts such a share `dax=inode` and maps only the files so marked. + let shm_size = share.dax.map_or(0, |d| d.window); + let dax_inode_min = share.dax.and_then(|d| d.inode_min).unwrap_or(0); ck( - "krun_add_virtiofs4", - krun_add_virtiofs4( + "krun_add_virtiofs5", + krun_add_virtiofs5( ctx, tag.as_ptr(), dir.as_ptr(), @@ -252,6 +255,7 @@ pub fn boot(spec: &VmSpec) -> Result<()> { share.read_only, uid_map.as_ptr(), gid_map.as_ptr(), + dax_inode_min, ), )?; } diff --git a/vk-driver/src/main.rs b/vk-driver/src/main.rs index 8f4a2e47..b74fc8ed 100644 --- a/vk-driver/src/main.rs +++ b/vk-driver/src/main.rs @@ -1624,13 +1624,17 @@ enum Cmd { /// declares its own `x-virtkit.reclaim`, to every service. #[arg(long, value_name = "POLICY")] reclaim: Option, - /// DAX window for virtio-fs shares: a size, or off + /// DAX window for virtio-fs shares: a size, off, :always or :inode= /// /// With a window the guest maps the host's page cache for the shared files it /// reads, so a tree is not cached twice and a host-side edit is visible at once; - /// the window is guest address space, not memory. The default is 8G per share, and - /// a guest holds 64G of windows in total. The built-in VMM only. Applies to the - /// primary and, unless a service declares its own `x-virtkit.dax`, to every service. + /// the window is guest address space, not memory. A bare size maps only regular + /// files of 1M and more (`dax=inode`): each mapping costs a host mmap per 2M range + /// whatever the file's size, which a source tree's small files never repay. + /// `:always` maps every file; `:inode=64K` moves the floor. The default is 8G per + /// share, and a guest holds 64G of windows in total. The built-in VMM only. Applies + /// to the primary and, unless a service declares its own `x-virtkit.dax`, to every + /// service. #[arg(long, value_name = "SIZE")] dax: Option, /// Forward the host SSH agent ($SSH_AUTH_SOCK) into the guest diff --git a/vk-driver/src/run.rs b/vk-driver/src/run.rs index aec7cc3a..434288da 100644 --- a/vk-driver/src/run.rs +++ b/vk-driver/src/run.rs @@ -1769,7 +1769,7 @@ async fn build_and_boot( let mut shares: Vec = Vec::new(); // The DAX window each directory share gets: the guest maps the host page cache through // it instead of copying file data into its own, so a tree read twice is read once. - let dax = dax_window(marker_dax, args.dax, crate::vmm::libkrun_selected()); + let dax = dax_share(marker_dax, args.dax, crate::vmm::libkrun_selected()); // Host-side helpers killed by `teardown_run`: virtiofsd and socket forwarders. let mut aux_children: Vec = Vec::new(); let mut virtiofs = String::new(); @@ -2012,6 +2012,12 @@ async fn build_and_boot( if !dax_tags.is_empty() { cmdline.push_str(&format!(" VIRTKIT_VIRTIOFS_DAX={dax_tags}")); } + // The subset served `dax=inode`: the agent mounts those with a file-size floor rather + // than mapping every file. + let dax_inode_tags = dax_inode_tags(&shares); + if !dax_inode_tags.is_empty() { + cmdline.push_str(&format!(" VIRTKIT_VIRTIOFS_DAX_INODE={dax_inode_tags}")); + } if !overlay_tags.is_empty() { cmdline.push_str(&format!( " VIRTKIT_VIRTIOFS_OVERLAY={}", @@ -2579,22 +2585,23 @@ pub(crate) fn effective_dax( declared.or(fallback).unwrap_or(crate::vmm::DAX_DEFAULT) } -/// DAX window per guest directory share, in bytes; `None` for no window. +/// What each guest directory share is served with: its DAX window and file-size floor; +/// `None` for no window. /// /// Only libkrun supports DAX; cloud-hypervisor serves shares the ordinary way. Warn once /// for an explicit window request so the unsupported setting is visible; defaults stay /// silent. /// Pass `libkrun` explicitly because tests cannot set the process-global backend selection. -pub(crate) fn dax_window( +pub(crate) fn dax_share( declared: Option, fallback: Option, libkrun: bool, -) -> Option { - let window = effective_dax(declared, fallback).window(); +) -> Option { + let share = effective_dax(declared, fallback).share(); if libkrun { - return window; + return share; } - if window.is_some() && declared.or(fallback).is_some() { + if share.is_some() && declared.or(fallback).is_some() { static SAID: std::sync::Once = std::sync::Once::new(); SAID.call_once(|| { eprintln!( @@ -2607,7 +2614,7 @@ pub(crate) fn dax_window( } /// The `VIRTKIT_VIRTIOFS_DAX` value for these shares: the tags that got a window, which is -/// what tells the agent to mount them `dax=always`. Empty when none did. +/// what tells the agent to mount them through it. Empty when none did. pub(crate) fn dax_tags(shares: &[crate::vmm::FsShare]) -> String { shares .iter() @@ -2617,6 +2624,17 @@ pub(crate) fn dax_tags(shares: &[crate::vmm::FsShare]) -> String { .join(",") } +/// The `VIRTKIT_VIRTIOFS_DAX_INODE` value: the subset of [`dax_tags`] whose window serves +/// only files above a size floor, which the agent mounts `dax=inode` rather than `always`. +pub(crate) fn dax_inode_tags(shares: &[crate::vmm::FsShare]) -> String { + shares + .iter() + .filter(|s| s.dax.is_some_and(|d| d.inode_min.is_some())) + .map(|s| s.tag.as_str()) + .collect::>() + .join(",") +} + /// Enable trimming only with agent PID 1 and a balloon. `run_init` reads the knob; /// `image`/`entrypoint` init execs the image's PID 1, leaving an unused `VIRTKIT_RECLAIM` /// environment variable and `psi=1`. An image *kernel* keeps the agent and remains eligible. @@ -4927,39 +4945,40 @@ mod tests { // The agent is told the tags that actually got a window, and nothing else — a // share mounted `dax=always` without one would fall back on every boot. - let share = |tag: &str, dax| crate::vmm::FsShare { + let share = |tag: &str, dax: Option| crate::vmm::FsShare { tag: tag.into(), socket: PathBuf::new(), host_dir: PathBuf::new(), read_only: false, - dax, + dax: dax.and_then(Dax::share), uid_map: Vec::new(), gid_map: Vec::new(), }; - assert_eq!( - dax_tags(&[ - share("work", Some(8 << 30)), - share("atop", None), - share("vol1", Some(8 << 30)), - ]), - "work,vol1" - ); + let shares = [ + share("work", Some(crate::vmm::DAX_DEFAULT)), + share("atop", None), + share("vol1", Some(Dax::Window(8 << 30))), + ]; + assert_eq!(dax_tags(&shares), "work,vol1"); + // Only the size-floored window is mounted `dax=inode`. + assert_eq!(dax_inode_tags(&shares), "work"); assert_eq!(dax_tags(&[share("atop", None)]), ""); assert_eq!(dax_tags(&[]), ""); + assert_eq!( + dax_inode_tags(&[share("vol1", Some(Dax::Window(1 << 30)))]), + "" + ); // The window exists only under the built-in VMM; cloud-hypervisor's virtio-fs has // no DAX path, so every share there is served the ordinary way whatever was asked. + assert_eq!(dax_share(None, None, true), crate::vmm::DAX_DEFAULT.share()); + assert_eq!(dax_share(Some(Dax::Off), None, true), None); assert_eq!( - dax_window(None, None, true), - crate::vmm::DAX_DEFAULT.window() - ); - assert_eq!(dax_window(Some(Dax::Off), None, true), None); - assert_eq!( - dax_window(Some(Dax::Window(1 << 30)), None, true), + dax_share(Some(Dax::Window(1 << 30)), None, true).map(|d| d.window), Some(1 << 30) ); - assert_eq!(dax_window(None, None, false), None); - assert_eq!(dax_window(Some(Dax::Window(1 << 30)), None, false), None); + assert_eq!(dax_share(None, None, false), None); + assert_eq!(dax_share(Some(Dax::Window(1 << 30)), None, false), None); } #[test] diff --git a/vk-driver/src/switch.rs b/vk-driver/src/switch.rs index 33d6d4a8..f1ef83ff 100644 --- a/vk-driver/src/switch.rs +++ b/vk-driver/src/switch.rs @@ -61,6 +61,9 @@ const FIRST_LEASE: u32 = 2; /// ServerHello). Bounding the dial fails the flow in seconds — we drop the guest stream /// and ipstack RSTs it — so a dead backend degrades to a fast connection error, not a hang. const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// Retransmissions of a guest-bound TCP segment before the flow is reset (see `run`'s +/// `TcpConfig`): 1+2+4+…+32s of tolerance for a switch the host did not schedule. +const TCP_MAX_RETRANSMITS: usize = 6; #[derive(Clone, Copy)] struct Cfg { @@ -796,6 +799,14 @@ pub async fn run( let (ret_tx, mut ret_rx) = unbounded_channel::>(); let mut config = IpStackConfig::default(); config.mtu_unchecked(MTU); + // A guest-bound segment is retransmitted on a doubling timeout from 1s; the default gives + // up after 3 tries — 7s. A switch a busy host fails to schedule for that long abandons the + // segment; unpatched, ipstack then left the flow Established with a permanent hole and the + // guest's transfer stuck at 0 bytes. Six tries hold the segment for 63s; past that the + // patched stack resets the connection, so the guest application fails fast and reconnects. + let mut tcp = ipstack::TcpConfig::default(); + tcp.max_retransmit_count = TCP_MAX_RETRANSMITS; + config.with_tcp_config(tcp); let ip_stack = IpStack::new( config, ChannelDevice { diff --git a/vk-driver/src/units.rs b/vk-driver/src/units.rs index c40aacb6..7bc442e3 100644 --- a/vk-driver/src/units.rs +++ b/vk-driver/src/units.rs @@ -527,7 +527,7 @@ pub fn boot_unit( let mut shares: Vec = Vec::new(); // The DAX window each of this service's shares gets (its own `x-virtkit.dax`, else the // run-wide default already folded in before provisioning). - let dax = crate::run::dax_window(svc.dax, None, crate::vmm::libkrun_selected()); + let dax = crate::run::dax_share(svc.dax, None, crate::vmm::libkrun_selected()); let mut virtiofs = String::new(); // Tags the agent should mount behind a tmpfs-backed overlay (`host:guest:overlay`), // exactly as the primary's own `-v`/compose volumes do in `run::build_and_boot` — a @@ -706,6 +706,12 @@ pub fn boot_unit( if !dax_tags.is_empty() { cmdline.push_str(&format!(" VIRTKIT_VIRTIOFS_DAX={dax_tags}")); } + // The subset served `dax=inode`: the agent mounts those with a file-size floor rather + // than mapping every file. + let dax_inode_tags = crate::run::dax_inode_tags(&shares); + if !dax_inode_tags.is_empty() { + cmdline.push_str(&format!(" VIRTKIT_VIRTIOFS_DAX_INODE={dax_inode_tags}")); + } // Idle page-cache trimming: a service that idles between requests gives the file // cache it piled up back to the host, not just the pages its processes freed. A // service always gets a balloon of its own (below), so only its init axis can take diff --git a/vk-driver/src/vm.rs b/vk-driver/src/vm.rs index 44ebe045..9724362a 100644 --- a/vk-driver/src/vm.rs +++ b/vk-driver/src/vm.rs @@ -1118,7 +1118,7 @@ pub async fn supervise(ctx: &JobCtx, job_dir_arg: &Path) -> Result<()> { // `[executor.vm] dax`: the window each directory share gets, so the guest reads a shared tree // out of the host page cache rather than copying it into its own. Same window for every // share here — the tools tree is the one several job VMs read at once. - let dax = crate::run::dax_window(vm_dax(cfg)?, None, crate::vmm::libkrun_selected()); + let dax = crate::run::dax_share(vm_dax(cfg)?, None, crate::vmm::libkrun_selected()); if let Some(share) = &cfg.executor.share { let vfsd_sock = ctx.vfsd_sock(); // libkrun mounts the host dir directly (built-in virtio-fs); only @@ -1299,6 +1299,12 @@ pub async fn supervise(ctx: &JobCtx, job_dir_arg: &Path) -> Result<()> { if !dax_tags.is_empty() { cmdline.push_str(&format!(" VIRTKIT_VIRTIOFS_DAX={dax_tags}")); } + // The subset served `dax=inode`: the agent mounts those with a file-size floor rather + // than mapping every file. + let dax_inode_tags = crate::run::dax_inode_tags(&shares); + if !dax_inode_tags.is_empty() { + cmdline.push_str(&format!(" VIRTKIT_VIRTIOFS_DAX_INODE={dax_inode_tags}")); + } // Idle page-cache trimming (`[executor.vm] reclaim`): the job guest gives file cache it stopped // using back to the host whenever it is not under memory pressure, so a job's read-once @@ -2466,7 +2472,10 @@ mod tests { cfg.executor.vm.dax = Some("4G".into()); assert_eq!( vm_dax(&cfg).unwrap(), - Some(crate::vmm::Dax::Window(4 << 30)) + Some(crate::vmm::Dax::Inode { + window: 4 << 30, + min: crate::vmm::DAX_INODE_MIN_DEFAULT + }) ); cfg.executor.vm.dax = Some("off".into()); assert_eq!(vm_dax(&cfg).unwrap(), Some(crate::vmm::Dax::Off)); diff --git a/vk-driver/src/vmm.rs b/vk-driver/src/vmm.rs index 03d3d050..d4c8c60c 100644 --- a/vk-driver/src/vmm.rs +++ b/vk-driver/src/vmm.rs @@ -165,13 +165,41 @@ impl Disk { pub enum Dax { /// No window: file data is copied into the guest's page cache on every read. Off, - /// A window this many bytes wide, per share. + /// A window this many bytes wide, per share, used for every file (`dax=always`). Window(u64), + /// A window this many bytes wide, per share, used only for regular files of at least + /// `min` bytes (`dax=inode`): a mapping costs the host an mmap and the guest an EPT + /// invalidation per 2 MiB range whatever the file's size, which a small file never + /// repays — so a source tree reads through the guest page cache and only the large + /// files (images, archives, build products) share the host's. + Inode { window: u64, min: u64 }, +} + +/// The smallest file `Dax::Inode` maps by default. Half a range is the point where mapping +/// beats copying on the trees measured (30k files of 2K each never repaid the mmap; an +/// archive or a disk image does at once). +pub const DAX_INODE_MIN_DEFAULT: u64 = 1 << 20; + +/// One host page. A DAX floor below this maps every regular file — that is `dax=always` +/// with a slower spelling — so `inode=` rejects anything smaller. +const DAX_INODE_MIN_FLOOR: u64 = 4096; + +/// What one share is served with: its window and, for `dax=inode`, the size floor. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct DaxShare { + /// Bytes of guest address space for the window. + pub window: u64, + /// `Some(bytes)`: DAX only for regular files at least this large (`dax=inode`); + /// `None`: every file (`dax=always`). + pub inode_min: Option, } /// Default per-share window. Reserves address space, not memory: the host maps and unmaps /// file ranges on demand, so it costs nothing until used. Sized for a working tree, not RAM. -pub const DAX_DEFAULT: Dax = Dax::Window(8 << 30); +pub const DAX_DEFAULT: Dax = Dax::Inode { + window: 8 << 30, + min: DAX_INODE_MIN_DEFAULT, +}; /// Smallest useful window: the guest's FUSE DAX layer hands out 2 MiB ranges. const DAX_MIN: u64 = 2 << 20; @@ -191,19 +219,57 @@ pub const DAX_TOTAL_MAX: u64 = 64 << 30; pub const DAX_MAX_GUEST_MIB: u64 = 64768; impl Dax { - /// The window in bytes, or `None` when off. - pub fn window(self) -> Option { + /// What a share gets under this policy, or `None` when off. + pub fn share(self) -> Option { match self { Dax::Off => None, - Dax::Window(bytes) => Some(bytes), + Dax::Window(window) => Some(DaxShare { + window, + inode_min: None, + }), + Dax::Inode { window, min } => Some(DaxShare { + window, + inode_min: Some(min), + }), } } } +/// A size with an optional binary suffix: `G`, `M`, `K`, or bare MiB. +fn parse_size(s: &str) -> Option { + let (digits, scale) = if let Some(d) = s.strip_suffix(['G', 'g']) { + (d, 1u64 << 30) + } else if let Some(d) = s.strip_suffix(['M', 'm']) { + (d, 1 << 20) + } else if let Some(d) = s.strip_suffix(['K', 'k']) { + (d, 1 << 10) + } else { + (s, 1 << 20) + }; + digits + .parse::() + .ok() + .and_then(|n| n.checked_mul(scale)) +} + +/// `8G`, `512M`, `1K` — whole units where the value has them, else the largest that divides it. +fn fmt_size(bytes: u64) -> String { + if bytes.is_multiple_of(1 << 30) { + format!("{}G", bytes >> 30) + } else if bytes.is_multiple_of(1 << 20) { + format!("{}M", bytes >> 20) + } else { + format!("{}K", bytes >> 10) + } +} + impl std::str::FromStr for Dax { type Err = String; - /// `off`, or a window size: `G`, `M`, or a bare MiB count. + /// `off`; a window size (`G`, `M`, or a bare MiB count), which maps files of at + /// least [`DAX_INODE_MIN_DEFAULT`]; `:always` for every file; or + /// `:inode=` for another floor. The window may be left out before the + /// colon (`always`, `inode=4M`) to take the default 8G. fn from_str(s: &str) -> Result { let s = s.trim(); // The spellings a YAML or TOML scalar turns "no" into; they are not sizes, so `0M` @@ -211,23 +277,47 @@ impl std::str::FromStr for Dax { if matches!(s, "off" | "false" | "0" | "no") { return Ok(Dax::Off); } - let (digits, scale) = match s.strip_suffix(['G', 'g']) { - Some(d) => (d, 1 << 30), - None => (s.strip_suffix(['M', 'm']).unwrap_or(s), 1 << 20), + let usage = || { + format!( + "expected off; a window of 2M..{}G written G, M or a MiB count; or \ + :always / :inode=, got {s:?}", + DAX_TOTAL_MAX >> 30 + ) }; - digits - .parse::() - .ok() - .and_then(|n| n.checked_mul(scale)) - .filter(|bytes| (DAX_MIN..=DAX_TOTAL_MAX).contains(bytes)) - .map(Dax::Window) - .ok_or_else(|| { - format!( - "expected off, or a window of 2M..{}G written G, M or a MiB count, \ - got {s:?}", - DAX_TOTAL_MAX >> 30 - ) - }) + let (size, mode) = match s.split_once(':') { + Some((size, mode)) => (size.trim(), Some(mode.trim())), + // `always` / `inode[=…]` alone: the default window under that mode. + None if s == "always" || s.starts_with("inode") => ("", Some(s)), + None => (s, None), + }; + let window = if size.is_empty() { + 8 << 30 + } else { + parse_size(size) + .filter(|bytes| (DAX_MIN..=DAX_TOTAL_MAX).contains(bytes)) + .ok_or_else(usage)? + }; + match mode { + None => Ok(Dax::Inode { + window, + min: DAX_INODE_MIN_DEFAULT, + }), + Some("always") => Ok(Dax::Window(window)), + Some("inode") => Ok(Dax::Inode { + window, + min: DAX_INODE_MIN_DEFAULT, + }), + Some(m) => { + let min = m + .strip_prefix("inode=") + .and_then(parse_size) + // Below one page is `always` with a slower spelling; above the window + // nothing would ever map. + .filter(|min| (DAX_INODE_MIN_FLOOR..=window).contains(min)) + .ok_or_else(usage)?; + Ok(Dax::Inode { window, min }) + } + } } } @@ -235,10 +325,14 @@ impl std::fmt::Display for Dax { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Dax::Off => f.write_str("off"), - // Whole gibibytes as `8G`, the spelling the docs and the CLI use; anything - // else in MiB, which every size this parser takes can be written in. - Dax::Window(bytes) if bytes.is_multiple_of(1 << 30) => write!(f, "{}G", bytes >> 30), - Dax::Window(bytes) => write!(f, "{}M", bytes >> 20), + Dax::Window(bytes) => write!(f, "{}:always", fmt_size(*bytes)), + // The default floor is implied by the bare window, the spelling the docs use. + Dax::Inode { window, min } if *min == DAX_INODE_MIN_DEFAULT => { + f.write_str(&fmt_size(*window)) + } + Dax::Inode { window, min } => { + write!(f, "{}:inode={}", fmt_size(*window), fmt_size(*min)) + } } } } @@ -253,14 +347,14 @@ pub struct FsShare { pub socket: PathBuf, pub host_dir: PathBuf, pub read_only: bool, - /// Bytes of guest address space for this share's DAX window; `None` = no window. - /// libkrun passes it to `krun_add_virtiofs4` as its `shm_size`. + /// This share's DAX window and file-size floor; `None` = no window. libkrun takes the + /// window as `krun_add_virtiofs5`'s `shm_size` and the floor as its `dax_inode_min`. #[serde(default)] - pub dax: Option, + pub dax: Option, /// virtiofsd-style UID id-map spec strings (`type:from:to[:count]`) applied at the /// guest↔host boundary; empty = identity. Under cloud-hypervisor these become /// `--uid-map` args to the bundled virtiofsd; under libkrun they go to - /// `krun_add_virtiofs4`. `gid_map` is the same for GIDs. + /// `krun_add_virtiofs5`. `gid_map` is the same for GIDs. #[serde(default)] pub uid_map: Vec, #[serde(default)] @@ -292,7 +386,9 @@ pub fn apply_dax_budget(shares: &mut [FsShare], mem: &str) { } let mut next = 0u64; for share in shares.iter_mut() { - let Some(window) = share.dax else { continue }; + let Some(DaxShare { window, .. }) = share.dax else { + continue; + }; let placed = window .checked_next_power_of_two() .map(|size| size.max(DAX_MIN)) @@ -962,10 +1058,37 @@ mod tests { #[test] fn dax_policy_parses_sizes_and_the_spellings_of_off() { use std::str::FromStr; - assert_eq!(Dax::from_str("8G").unwrap(), Dax::Window(8 << 30)); - assert_eq!(Dax::from_str("512M").unwrap(), Dax::Window(512 << 20)); + let inode = |window| Dax::Inode { + window, + min: DAX_INODE_MIN_DEFAULT, + }; + // A bare window maps files from the default floor up. + assert_eq!(Dax::from_str("8G").unwrap(), inode(8 << 30)); + assert_eq!(Dax::from_str("512M").unwrap(), inode(512 << 20)); // A bare count is MiB, like every other size this CLI takes. - assert_eq!(Dax::from_str("64").unwrap(), Dax::Window(64 << 20)); + assert_eq!(Dax::from_str("64").unwrap(), inode(64 << 20)); + // `:always` maps every file; `:inode=` picks the floor; the window may be left out. + assert_eq!(Dax::from_str("8G:always").unwrap(), Dax::Window(8 << 30)); + assert_eq!(Dax::from_str("always").unwrap(), Dax::Window(8 << 30)); + assert_eq!(Dax::from_str("inode").unwrap(), inode(8 << 30)); + assert_eq!( + Dax::from_str("4G:inode=64K").unwrap(), + Dax::Inode { + window: 4 << 30, + min: 64 << 10 + } + ); + assert_eq!( + Dax::from_str("inode=4M").unwrap(), + Dax::Inode { + window: 8 << 30, + min: 4 << 20 + } + ); + // A floor under a page or above the window, or an unknown mode, is refused. + assert!(Dax::from_str("8G:inode=1K").is_err()); + assert!(Dax::from_str("2M:inode=4M").is_err()); + assert!(Dax::from_str("8G:sometimes").is_err()); // What a YAML or TOML scalar turns "no" into all mean off. for off in ["off", "false", "0", "no", " off "] { assert_eq!(Dax::from_str(off).unwrap(), Dax::Off, "{off}"); @@ -978,18 +1101,46 @@ mod tests { assert!(Dax::from_str("128G").is_err()); assert_eq!( Dax::from_str(&format!("{}G", DAX_TOTAL_MAX >> 30)).unwrap(), - Dax::Window(DAX_TOTAL_MAX) + inode(DAX_TOTAL_MAX) ); // Round-trips through the config file, which stores the policy as a string. assert_eq!(DAX_DEFAULT.to_string(), "8G"); - assert_eq!(Dax::Window(512 << 20).to_string(), "512M"); + assert_eq!(Dax::Window(512 << 20).to_string(), "512M:always"); + for spelled in ["8G:always", "4G:inode=64K", "inode=4M", "2M"] { + let policy = Dax::from_str(spelled).unwrap(); + assert_eq!( + Dax::from_str(&policy.to_string()).unwrap(), + policy, + "{spelled}" + ); + } + assert_eq!( + Dax::from_str("8G").unwrap().share(), + Some(DaxShare { + window: 8 << 30, + inode_min: Some(DAX_INODE_MIN_DEFAULT) + }) + ); + assert_eq!( + Dax::Window(8 << 30).share(), + Some(DaxShare { + window: 8 << 30, + inode_min: None + }) + ); + assert_eq!(Dax::Off.share(), None); assert_eq!( Dax::from_str(&DAX_DEFAULT.to_string()).unwrap(), DAX_DEFAULT ); assert_eq!(Dax::Off.to_string(), "off"); - assert_eq!(Dax::Off.window(), None); - assert_eq!(DAX_DEFAULT.window(), Some(8 << 30)); + assert_eq!(window(Dax::Off), None); + assert_eq!(window(DAX_DEFAULT), Some(8 << 30)); + } + + /// The policy's window, as the tests compare it: `share()` without the floor. + fn window(policy: Dax) -> Option { + policy.share().map(|d| d.window) } fn dax_share(tag: &str, dax: Option) -> FsShare { @@ -998,7 +1149,10 @@ mod tests { socket: PathBuf::new(), host_dir: PathBuf::new(), read_only: false, - dax, + dax: dax.map(|window| DaxShare { + window, + inode_min: None, + }), uid_map: Vec::new(), gid_map: Vec::new(), } @@ -1009,11 +1163,15 @@ mod tests { // Eight default windows fill the span exactly; the ninth gets none, and a share // that asked for nothing is charged nothing. let mut shares: Vec = (0..9) - .map(|i| dax_share(&format!("s{i}"), DAX_DEFAULT.window())) + .map(|i| dax_share(&format!("s{i}"), window(DAX_DEFAULT))) .collect(); shares.push(dax_share("atop", None)); apply_dax_budget(&mut shares, "4G"); - assert!(shares[..8].iter().all(|s| s.dax == DAX_DEFAULT.window())); + assert!( + shares[..8] + .iter() + .all(|s| s.dax.map(|d| d.window) == window(DAX_DEFAULT)) + ); assert_eq!(shares[8].dax, None); assert_eq!(shares[9].dax, None); } @@ -1041,10 +1199,10 @@ mod tests { /// about a window would earn a refused mount on every boot. #[test] fn a_guest_too_large_for_the_span_gets_no_windows_at_all() { - let shares = || vec![dax_share("work", DAX_DEFAULT.window())]; + let shares = || vec![dax_share("work", window(DAX_DEFAULT))]; let mut s = shares(); apply_dax_budget(&mut s, &format!("{DAX_MAX_GUEST_MIB}M")); - assert_eq!(s[0].dax, DAX_DEFAULT.window()); + assert_eq!(s[0].dax.map(|d| d.window), window(DAX_DEFAULT)); let mut s = shares(); apply_dax_budget(&mut s, &format!("{}M", DAX_MAX_GUEST_MIB + 1)); assert_eq!(s[0].dax, None); @@ -1052,7 +1210,7 @@ mod tests { // window to lose anyway. let mut s = shares(); apply_dax_budget(&mut s, "64G@0"); - assert_eq!(s[0].dax, DAX_DEFAULT.window()); + assert_eq!(s[0].dax.map(|d| d.window), window(DAX_DEFAULT)); } /// The CI path: API socket (graceful shutdown), a rw qcow2 overlay root,