From 1e82b87f23b3a5c2189313c61fd5055f514e0389 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Wed, 7 Oct 2026 17:14:16 +0200 Subject: [PATCH 01/71] CI: build and test DCE with the Linux/FreeBSD kernel stacks Run the GitHub Actions workflow on pushes to master and on every pull request, and drop the obsolete bake-based workflow (Ubuntu 20.04 runners and upload-artifact@v1 no longer exist). Add utils/build_kernel_deps.sh, which builds what bake used to provide: net-next-nuse (Linux 4.4 libos), freebsd-sim, DCE builds of ip, iperf, thttpd, wget, ping/ping6 and quagga, and the ns-3-dce-quagga module, patched for current ns-3 and toolchains (GCC 11-14, Make 4.4). The workflow caches its output and configures DCE with --enable-kernel-stack. Fix DCE for ns-3.46 and Linux semantics: - set the node on Ipv4Interfaces (Linux and FreeBSD glue) and give kernel-stack nodes an inert ArpL3Protocol for CSMA devices - notify connection success for connectionless kernel sockets - deliver multicast locally in Ipv4DceRouting (raw sockets, OSPF) - implement sigwait() - ns-3 stream sockets: connect() succeeds when the peer accepts and closes at once, and reports ECONNREFUSED when refused - ns-3 stack: IPv6 UDP sockets, IPV6_JOIN/LEAVE_GROUP, IPV6_RECVPKTINFO, ICMP6_FILTER, IPV6_PKTINFO on send, no IPv6 header on raw receive, delivery for raw-only IPv6 protocols such as OSPFv3 - netlink and pktinfo use Linux-style interface indexes consistently - reset glibc's getopt state for each simulated process - drop the NSC stacks from dce-tcp-ns3-nsc-comparison; use 802.11a in the MPTCP handoff examples Enable the disabled select/poll/tcp-socket/signal process-manager tests, all quagga test cases and the dce-xfrm example. 79/79 pass on Ubuntu 22.04, Ubuntu 24.04 and Debian trixie with ns-3.46. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-and-test.yml | 65 +++- .github/workflows/dce-gaction-test.yml | 69 ---- example/dce-mptcp-handoff-v4v6.cc | 1 + example/dce-mptcp-handoff-v6.cc | 1 + example/dce-tcp-ns3-nsc-comparison.cc | 19 - example/examples-to-run.py | 3 +- model/dce-manager.cc | 2 + model/dce-signal.cc | 28 +- model/dce.cc | 19 + model/freebsd/ipv4-freebsd.cc | 10 + model/ipv4-dce-routing.cc | 23 ++ model/ipv4-dce-routing.h | 4 + model/kernel-socket-fd-factory.cc | 2 +- model/linux/ipv4-linux.cc | 10 + model/linux/linux-socket-impl.cc | 7 + model/ns3-socket-fd-factory.cc | 47 +++ model/process.h | 2 + model/unix-datagram-socket-fd.cc | 45 ++- model/unix-datagram-socket-fd.h | 4 +- model/unix-socket-fd.cc | 70 ++++ model/unix-stream-socket-fd.cc | 12 +- model/unix-stream-socket-fd.h | 1 + model/utils.cc | 5 +- netlink/netlink-socket.cc | 82 ++++- netlink/netlink-socket.h | 4 + test/dce-manager-test.cc | 8 +- utils/build_kernel_deps.sh | 200 ++++++++++ utils/freebsd-sim-dce.patch | 52 +++ utils/ns-3-dce-quagga-ns3.patch | 491 +++++++++++++++++++++++++ 29 files changed, 1154 insertions(+), 132 deletions(-) delete mode 100644 .github/workflows/dce-gaction-test.yml create mode 100755 utils/build_kernel_deps.sh create mode 100644 utils/freebsd-sim-dce.patch create mode 100644 utils/ns-3-dce-quagga-ns3.patch diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 67fa9f5a..863c2a79 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -2,11 +2,16 @@ name: Build and Test DCE on: push: - branches: [ sikabo_master ] + branches: [ master ] + # Run for every pull request (merge-request pipeline), whatever the target branch pull_request: - branches: [ sikabo_master ] workflow_dispatch: +# Cancel superseded runs of the same branch/PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: NS3_VER: "3.46" @@ -44,7 +49,17 @@ jobs: libsqlite3-dev \ libxml2-dev \ wget \ - make + make \ + bc \ + flex \ + bison \ + byacc \ + gawk \ + file \ + xz-utils \ + libssl-dev \ + libdb-dev \ + libsctp-dev - name: Configure git safe directory run: git config --global --add safe.directory "$GITHUB_WORKSPACE/ns-3-dce" @@ -88,24 +103,62 @@ jobs: working-directory: ns-${{ env.NS3_VER }} run: ./ns3 install + # Linux/FreeBSD kernel libraries, DCE builds of ip, iperf, thttpd, + # wget, ping and quagga, and the ns-3-dce-quagga module. + - name: Cache kernel stack dependencies + id: cache-kernel-deps + uses: actions/cache@v4 + with: + path: | + deps/bin_dce + deps/kernel + deps/ns-3-dce-quagga + key: kernel-deps-${{ matrix.distro }}-${{ hashFiles('ns-3-dce/utils/build_kernel_deps.sh', 'ns-3-dce/utils/*.patch') }} + + - name: Build kernel stack dependencies + if: steps.cache-kernel-deps.outputs.cache-hit != 'true' + working-directory: ns-3-dce + run: | + ./utils/build_kernel_deps.sh "$GITHUB_WORKSPACE/deps" + rm -rf "$GITHUB_WORKSPACE/deps/src" + + - name: Add Quagga module + run: cp -a deps/ns-3-dce-quagga ns-3-dce/myscripts/ + - name: Configure DCE working-directory: ns-3-dce run: | ./waf configure \ --with-ns3="$GITHUB_WORKSPACE/build" \ --prefix="$GITHUB_WORKSPACE/build" \ + --enable-kernel-stack="$GITHUB_WORKSPACE/deps/kernel" \ --enable-opt - name: Build DCE working-directory: ns-3-dce run: ./waf build - - name: Build bash for DCE + - name: Install DCE binaries working-directory: ns-3-dce - run: ./utils/build_bash.sh build/bin_dce + run: | + cp -a "$GITHUB_WORKSPACE/deps/bin_dce/." build/bin_dce/ + ./utils/build_bash.sh build/bin_dce - name: Run DCE tests working-directory: ns-3-dce env: LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib:${{ github.workspace }}/build/lib64 - run: python3 test.py + run: python3 test.py -r + + - name: Remove sockets from test output + if: always() + working-directory: ns-3-dce + run: if [ -d testpy-output ]; then find testpy-output -type s -delete; fi + + - name: Upload test output + if: always() + uses: actions/upload-artifact@v4 + with: + name: testpy-output-${{ matrix.distro }} + path: ns-3-dce/testpy-output + if-no-files-found: ignore diff --git a/.github/workflows/dce-gaction-test.yml b/.github/workflows/dce-gaction-test.yml deleted file mode 100644 index 14983f28..00000000 --- a/.github/workflows/dce-gaction-test.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: DCE Build -on: [push, pull_request] - -# Anchors are not supported, so could not make use of modular workflow - -jobs : - build: - strategy: - matrix: - job: - # Used a personal Ubuntu-16.04 docker image since the previous once defaulted to root user, - # so had to prefix all commands with sudo, and Github actions like checkout and caching throws permission error - # [ 'job-name' , 'docker-image-name' ] - # [ 'ubuntu-16.04', 'parthpratim27/ns-3-dce:ubuntu-16.04' ] - - [ 'ubuntu-20.04', 'parthpratim27/ns-3-dce:ubuntu-20.04' ] - - name : ${{ matrix.job[0] }} - runs-on : ubuntu-20.04 # This is ignored, but is a required field so is being set to this - container: ${{ matrix.job[1] }} - - env: - BAKE_GIT: 'https://gitlab.com/nsnam/bake.git' - BAKE_CONFIG: 'dce-linux-dev' # list-type, for each entry, -e {bake_config_entry} ... , will be added - GIT_HEAD_TRACK : '.git/HEAD' - HG_CHANGE_LOG : '.hg/store/00changelog.i' - - steps: - - name: Pull Source Tree - uses: actions/checkout@v2 - with: - path : target-dce-repo - - - name: Download Bake - run: git clone $BAKE_GIT bake - - - name: Configure Bake - run: ./bake/bake.py configure -e $BAKE_CONFIG - - - name: Download all required repos - run: ./bake/bake.py download -vvv - - - name: Copy Current ${{ github.event_name }} to source/ns-3-dce - run: cp -a target-dce-repo/. source/ns-3-dce - - # Below Caching saves 16 mins of build time (if an ns-3-dev build with same HEAD already exists) - # Couldn't add the same for net-next-nuse-4.4.0, because it would anyways still build the entire project after defconfig - - name: Load Build Cache for ns-3-dev - uses: actions/cache@v2 - with: - id : cache-ns-3-dev - path: source/ns-3-dev - key: ${{ matrix.job[0] }}-src-build-ns-3-dev-${{ hashFiles('source/ns-3-dev/.git/HEAD') }} - - - name: Build Bake - run: ./bake/bake.py build -vvv - - - name: Run Tests - run: | - cd source/ns-3-dce && ./test.py -r - find "testpy-output/" -type s -exec rm -f {} \; - - - name: Save Test Output Artifacts - uses: actions/upload-artifact@v1 - with: - name: ${{ github.job }}-test-py-output - path: source/ns-3-dce/testpy-output - - - diff --git a/example/dce-mptcp-handoff-v4v6.cc b/example/dce-mptcp-handoff-v4v6.cc index c113b8e3..a555d0fe 100644 --- a/example/dce-mptcp-handoff-v4v6.cc +++ b/example/dce-mptcp-handoff-v4v6.cc @@ -111,6 +111,7 @@ int main (int argc, char *argv[]) WifiMacHelper mac; YansWifiPhyHelper phy; YansWifiChannelHelper phyChannel = YansWifiChannelHelper::Default (); + wifi.SetStandard (WIFI_STANDARD_80211a); wifi.SetRemoteStationManager ("ns3::ArfWifiManager"); // setup Wifi sta. diff --git a/example/dce-mptcp-handoff-v6.cc b/example/dce-mptcp-handoff-v6.cc index 4e88b601..0a590493 100644 --- a/example/dce-mptcp-handoff-v6.cc +++ b/example/dce-mptcp-handoff-v6.cc @@ -110,6 +110,7 @@ int main (int argc, char *argv[]) WifiMacHelper mac; YansWifiPhyHelper phy; YansWifiChannelHelper phyChannel = YansWifiChannelHelper::Default (); + wifi.SetStandard (WIFI_STANDARD_80211a); wifi.SetRemoteStationManager ("ns3::ArfWifiManager"); // setup Wifi sta. diff --git a/example/dce-tcp-ns3-nsc-comparison.cc b/example/dce-tcp-ns3-nsc-comparison.cc index 6586e060..70b8a085 100644 --- a/example/dce-tcp-ns3-nsc-comparison.cc +++ b/example/dce-tcp-ns3-nsc-comparison.cc @@ -118,25 +118,6 @@ main (int argc, char *argv[]) sock_factory = "ns3::TcpSocketFactory"; internetStack.Install (nodes); } - else if (m_stack == "nsc-linux") - { - internetStack.Install (routers); - sock_factory = "ns3::TcpSocketFactory"; - internetStack.SetTcp ("ns3::NscTcpL4Protocol", - "Library", StringValue ("liblinux2.6.26.so")); - internetStack.Install (lefts); - internetStack.Install (rights); - // Config::Set ("/NodeList/*/$ns3::Ns3NscStack/net.ipv4.tcp_congestion_control", - // StringValue ("reno")); - } - else if (m_stack == "nsc-freebsd") - { - // didn't work under 64bit (12/11/13) - sock_factory = "ns3::TcpSocketFactory"; - internetStack.SetTcp ("ns3::NscTcpL4Protocol", - "Library", StringValue ("libfreebsd5.3.so")); - internetStack.Install (nodes); - } else { NS_ASSERT_MSG (0, "no stack " << m_stack << " is available"); diff --git a/example/examples-to-run.py b/example/examples-to-run.py index c9e18111..78f27339 100644 --- a/example/examples-to-run.py +++ b/example/examples-to-run.py @@ -54,7 +54,8 @@ ("dce-ping-mt1 --kernel=1", "True", "True"), ("dce-mt2 --kernel=1", "True", "True"), ("dce-mt3 --kernel=1", "True", "True"), - ("dce-xfrm", "False", "False"), + # Runs, but libos 4.4 cannot install IPsec states (ENOSYS) + ("dce-xfrm", "True", "True"), # ("dce-ltp", "True", "True"), ("dce-cradle-mptcp", "True", "True"), ("dce-iperf-mptcp", "IPERF_DCE_FOUND == True", "IPERF_DCE_FOUND == True"), diff --git a/model/dce-manager.cc b/model/dce-manager.cc index 0a4227f1..ce57f0e6 100644 --- a/model/dce-manager.cc +++ b/model/dce-manager.cc @@ -338,6 +338,7 @@ DceManager::CreateProcess (std::string name, std::string stdinfilename, std::vec process->pgid = 0; process->pid = pid ? pid : AllocatePid (); process->manager = this; + process->getoptStarted = false; sigemptyset (&process->pendingSignals); SetDefaultSigHandler (process->signalHandlers); @@ -544,6 +545,7 @@ DceManager::CreateThread (struct Process *process) thread->pollTable = 0; thread->ioWait = std::make_pair ((UnixFd*)0,(WaitQueueEntry*)0); sigemptyset (&thread->signalMask); + sigemptyset (&thread->sigwaitSet); if (!process->threads.empty ()) { // copy all key values. diff --git a/model/dce-signal.cc b/model/dce-signal.cc index e8317aff..b7bfe277 100644 --- a/model/dce-signal.cc +++ b/model/dce-signal.cc @@ -106,13 +106,27 @@ int dce_sigwait (const sigset_t *set, int *sig) NS_LOG_FUNCTION (current << UtilsGetNodeId () << set << sig); NS_ASSERT (current != 0); - int ret = 0; - // NEED TO WORK!! - // XXX: we need to add signal num notiifcation - current->process->manager->Wait (); - // sigdelset (¤t->pendingSignals, numnum); - - return ret; + while (true) + { + for (int s = 1; s < NSIG; s++) + { + if (sigismember (set, s) != 1) + { + continue; + } + if (sigismember (¤t->pendingSignals, s) == 1 + || sigismember (¤t->process->pendingSignals, s) == 1) + { + sigdelset (¤t->pendingSignals, s); + sigdelset (¤t->process->pendingSignals, s); + sigemptyset (¤t->sigwaitSet); + *sig = s; + return 0; + } + } + current->sigwaitSet = *set; + current->process->manager->Wait (); + } } int dce_sigprocmask (int how, const sigset_t *set, sigset_t *oldset) { diff --git a/model/dce.cc b/model/dce.cc index d404d08b..62c8ed2a 100644 --- a/model/dce.cc +++ b/model/dce.cc @@ -446,6 +446,23 @@ const char * dce_inet_ntop (int af, const void *src, } return retval; } +// glibc keeps getopt parsing state (e.g. the position inside a cluster of +// short options) in a hidden static shared by all simulated processes. +// optind = 0 makes glibc reinitialize it: always on a process' first call, +// and when another process used getopt last and this one is at the start +// of its arguments. +static void +ResetGetoptIfOtherProcess (Process *process) +{ + static Process *lastProcess = 0; + if (!process->getoptStarted || (lastProcess != process && optind <= 1)) + { + optind = 0; + } + process->getoptStarted = true; + lastProcess = process; +} + int dce_getopt (int argc, char * const argv[], const char *optstring) { NS_LOG_FUNCTION (Current () << UtilsGetNodeId () << argc << argv << optstring); @@ -464,6 +481,7 @@ int dce_getopt (int argc, char * const argv[], const char *optstring) optind = *process->poptind; opterr = *process->popterr; optopt = *process->poptopt; + ResetGetoptIfOtherProcess (process); int retval = getopt (argc, argv, optstring); *process->poptarg = optarg; *process->poptind = optind; @@ -495,6 +513,7 @@ int dce_getopt_long (int argc, char * const argv[], const char *optstring, optind = *process->poptind; opterr = *process->popterr; optopt = *process->poptopt; + ResetGetoptIfOtherProcess (process); int retval = getopt_long (argc, argv, optstring, longopts, longindex); *process->poptarg = optarg; *process->poptind = optind; diff --git a/model/freebsd/ipv4-freebsd.cc b/model/freebsd/ipv4-freebsd.cc index eeeea438..6ccc6416 100644 --- a/model/freebsd/ipv4-freebsd.cc +++ b/model/freebsd/ipv4-freebsd.cc @@ -25,6 +25,7 @@ #include "ns3/ipv4-static-routing-helper.h" #include "ns3/ipv4-global-routing-helper.h" #include "ns3/ipv4-interface.h" +#include "ns3/arp-l3-protocol.h" #include "ns3/ipv4-global-routing.h" #include "ns3/ipv4-routing-table-entry.h" #include "dce-application-helper.h" @@ -98,7 +99,9 @@ Ipv4FreeBSD::AddInterface (Ptr device) { NS_LOG_FUNCTION (this << &device); + Ptr node = GetObject (); Ptr interface = CreateObject (); + interface->SetNode (node); interface->SetDevice (device); interface->SetForwarding (m_ipForward); return AddIpv4Interface (interface); @@ -561,6 +564,13 @@ Ipv4FreeBSD::InstallNode (Ptr node) factory.SetTypeId ("ns3::Ipv4FreeBSD"); Ptr protocol = factory.Create (); node->AggregateObject (protocol); + // ns3::Ipv4Interface creates an ARP cache for devices that need ARP once it + // knows its node. The FreeBSD kernel does ARP itself, so this ArpL3Protocol + // is never registered as a protocol handler and stays inert. + if (!node->GetObject ()) + { + node->AggregateObject (CreateObject ()); + } Ipv4GlobalRoutingHelper globalRouting; // Set routing Ptr ipv4 = node->GetObject (); diff --git a/model/ipv4-dce-routing.cc b/model/ipv4-dce-routing.cc index e497b0f1..f26206e4 100644 --- a/model/ipv4-dce-routing.cc +++ b/model/ipv4-dce-routing.cc @@ -63,6 +63,29 @@ Ipv4DceRouting::~Ipv4DceRouting () { } +bool +Ipv4DceRouting::RouteInput (Ptr p, const Ipv4Header &header, Ptr idev, + const UnicastForwardCallback &ucb, const MulticastForwardCallback &mcb, + const LocalDeliverCallback &lcb, const ErrorCallback &ecb) +{ + NS_LOG_FUNCTION (this << p << header << idev); + + if (Ipv4StaticRouting::RouteInput (p, header, idev, ucb, mcb, lcb, ecb)) + { + return true; + } + // ns-3 only hands packets to raw sockets on local delivery, and + // Ipv4StaticRouting does not locally deliver multicast without a multicast + // route. Deliver it so that applications such as routing daemons, which + // listen on raw sockets for groups like 224.0.0.5, receive it. + if (header.GetDestination ().IsMulticast () && !lcb.IsNull ()) + { + lcb (p, header, m_ipv4->GetInterfaceForDevice (idev)); + return true; + } + return false; +} + void Ipv4DceRouting::NotifyInterfaceUp (uint32_t i) { diff --git a/model/ipv4-dce-routing.h b/model/ipv4-dce-routing.h index 9ae033ea..65683ed1 100644 --- a/model/ipv4-dce-routing.h +++ b/model/ipv4-dce-routing.h @@ -61,6 +61,10 @@ class Ipv4DceRouting : public Ipv4StaticRouting virtual void PrintRoutingTable (Ptr stream) const; + bool RouteInput (Ptr p, const Ipv4Header &header, Ptr idev, + const UnicastForwardCallback &ucb, const MulticastForwardCallback &mcb, + const LocalDeliverCallback &lcb, const ErrorCallback &ecb) override; + virtual void SetIpv4 (Ptr ipv4); template diff --git a/model/kernel-socket-fd-factory.cc b/model/kernel-socket-fd-factory.cc index 0eaba47e..5346cb4b 100644 --- a/model/kernel-socket-fd-factory.cc +++ b/model/kernel-socket-fd-factory.cc @@ -211,7 +211,7 @@ KernelSocketFdFactory::Open (struct SimKernel *kernel, const char *pathname, int int KernelSocketFdFactory::__Fxstat (struct SimKernel *kernel, int ver, int fd, void *buf) { - return dce___fxstat (ver, fd, (struct stat *)buf); + return dce_fstat (fd, (struct stat *)buf); } int KernelSocketFdFactory::Fseek (struct SimKernel *kernel, FILE *stream, long offset, int whence) diff --git a/model/linux/ipv4-linux.cc b/model/linux/ipv4-linux.cc index 3f61f0d7..5e4d157e 100644 --- a/model/linux/ipv4-linux.cc +++ b/model/linux/ipv4-linux.cc @@ -26,6 +26,7 @@ #include "ns3/ipv4-static-routing-helper.h" #include "ns3/ipv4-global-routing-helper.h" #include "ns3/ipv4-interface.h" +#include "ns3/arp-l3-protocol.h" #include "ns3/ipv4-global-routing.h" #include "ns3/ipv4-routing-table-entry.h" #include "linux-stack-helper.h" @@ -82,7 +83,9 @@ Ipv4Linux::AddInterface (Ptr device) { NS_LOG_FUNCTION (this << &device); + Ptr node = GetObject (); Ptr interface = CreateObject (); + interface->SetNode (node); interface->SetDevice (device); interface->SetForwarding (m_ipForward); return AddIpv4Interface (interface); @@ -541,6 +544,13 @@ Ipv4Linux::InstallNode (Ptr node) factory.SetTypeId ("ns3::Ipv4Linux"); Ptr protocol = factory.Create (); node->AggregateObject (protocol); + // ns3::Ipv4Interface creates an ARP cache for devices that need ARP once it + // knows its node. The Linux kernel does ARP itself, so this ArpL3Protocol is + // never registered as a protocol handler and stays inert. + if (!node->GetObject ()) + { + node->AggregateObject (CreateObject ()); + } Ipv4GlobalRoutingHelper globalRouting; // Set routing Ptr ipv4 = node->GetObject (); diff --git a/model/linux/linux-socket-impl.cc b/model/linux/linux-socket-impl.cc index 543d8f48..e4c1cf31 100644 --- a/model/linux/linux-socket-impl.cc +++ b/model/linux/linux-socket-impl.cc @@ -319,6 +319,13 @@ LinuxSocketImpl::Connect (const Address & address) m_conn_inprogress = true; } LeaveFakeTask (pid); + if (ret == 0) + { + // Connectionless sockets (UDP, raw) connect immediately; notify like + // ns-3's own sockets do, since applications such as OnOffApplication + // only start sending from the connection-succeeded callback. + NotifyConnectionSucceeded (); + } return ret; } diff --git a/model/ns3-socket-fd-factory.cc b/model/ns3-socket-fd-factory.cc index d91b4882..94dced0a 100644 --- a/model/ns3-socket-fd-factory.cc +++ b/model/ns3-socket-fd-factory.cc @@ -29,6 +29,8 @@ #include "ns3/node.h" #include "ns3/uinteger.h" #include "ns3/packet-socket-address.h" +#include "ns3/ip-l4-protocol.h" +#include "ns3/ipv6-l3-protocol.h" #include #include @@ -38,6 +40,38 @@ namespace ns3 { NS_OBJECT_ENSURE_REGISTERED (Ns3SocketFdFactory); +/** + * ns3::Ipv6L3Protocol only hands packets to raw sockets when an L4 protocol + * is registered for their next header; otherwise it drops them and answers + * with an ICMPv6 parameter problem. This protocol accepts packets for a + * protocol number that only raw sockets handle (e.g. OSPFv3). + */ +class DceIpv6RawOnlyProtocol : public IpL4Protocol +{ +public: + static TypeId GetTypeId (void) + { + static TypeId tid = TypeId ("ns3::DceIpv6RawOnlyProtocol") + .SetParent () + .SetGroupName ("Dce"); + return tid; + } + DceIpv6RawOnlyProtocol (int protocol = 0) : m_protocol (protocol) {} + int GetProtocolNumber (void) const override { return m_protocol; } + RxStatus Receive (Ptr p, const Ipv4Header &header, + Ptr incomingInterface) override { return RX_OK; } + RxStatus Receive (Ptr p, const Ipv6Header &header, + Ptr incomingInterface) override { return RX_OK; } + void SetDownTarget (DownTargetCallback cb) override { m_downTarget = cb; } + void SetDownTarget6 (DownTargetCallback6 cb) override { m_downTarget6 = cb; } + DownTargetCallback GetDownTarget (void) const override { return m_downTarget; } + DownTargetCallback6 GetDownTarget6 (void) const override { return m_downTarget6; } +private: + int m_protocol; + DownTargetCallback m_downTarget; + DownTargetCallback6 m_downTarget6; +}; + TypeId Ns3SocketFdFactory::GetTypeId (void) { @@ -121,9 +155,22 @@ Ns3SocketFdFactory::CreateSocket (int domain, int type, int protocol) sock = factory->CreateSocket (); sock->SetAttribute ("Protocol", UintegerValue (protocol)); socket = new UnixDatagramSocketFd (sock); + Ptr ipv6 = GetObject (); + if (ipv6 && protocol > 0 && !ipv6->GetProtocol (protocol)) + { + ipv6->Insert (CreateObject (protocol)); + } } break; case SOCK_DGRAM: { + // ns-3 UDP sockets are dual-stack: binding an IPv6 address makes + // them IPv6 sockets. + TypeId tid = TypeId::LookupByName ("ns3::UdpSocketFactory"); + Ptr factory = GetObject (tid); + NS_ASSERT_MSG (factory, "InternetStackHelper (v6) is not installed. " + "Install it before using Ns3SocketFdFactory."); + sock = factory->CreateSocket (); + socket = new UnixDatagramSocketFd (sock); } break; case SOCK_STREAM: { diff --git a/model/process.h b/model/process.h index 8969776b..feb59bcb 100644 --- a/model/process.h +++ b/model/process.h @@ -148,6 +148,7 @@ struct Process std::vector conditions; std::vector atExitHandlers; std::set children; + bool getoptStarted; // getopt has been called by this process sigset_t pendingSignals; Time itimerInterval; EventId itimer; @@ -222,6 +223,7 @@ struct Thread std::list keyValues; sigset_t signalMask; sigset_t pendingSignals; + sigset_t sigwaitSet; // Signals this thread waits for in sigwait, empty otherwise Time lastTime; // Last time of a possible infinite loop checkpoint. Waiter *childWaiter; // Not zero if thread waiting for a child in wait or waitall ... PollTable *pollTable; // No 0 if a poll is running on this thread diff --git a/model/unix-datagram-socket-fd.cc b/model/unix-datagram-socket-fd.cc index 24f3f4ca..5dce831c 100644 --- a/model/unix-datagram-socket-fd.cc +++ b/model/unix-datagram-socket-fd.cc @@ -36,6 +36,7 @@ #include "ns3/ipv6.h" #include "ns3/ipv4-packet-info-tag.h" #include "ns3/ipv6-packet-info-tag.h" +#include "ns3/ipv6-header.h" #include "cmsg.h" #include #include @@ -183,7 +184,9 @@ UnixDatagramSocketFd::DoRecvmsg (struct msghdr *msg, int flags) // we ignore the secondary items of the iovec buffers. // because we implement a datagram-only socket for now. Address from; - Ptr packet = m_socket->RecvFrom (count, flags, from); + // ns-3 IPv6 raw sockets return the IPv6 header too; Linux does not. + bool ipv6Raw = m_socket->GetInstanceTypeId () == TypeId::LookupByName ("ns3::Ipv6RawSocketImpl"); + Ptr packet = m_socket->RecvFrom (ipv6Raw ? count + 40 : count, flags, from); uint32_t l = 0; if (!packet) @@ -191,6 +194,11 @@ UnixDatagramSocketFd::DoRecvmsg (struct msghdr *msg, int flags) current->err = ErrnoToSimuErrno (); return -1; } + if (ipv6Raw) + { + Ipv6Header ipv6Header; + packet->RemoveHeader (ipv6Header); + } if ((PacketSocketAddress::IsMatchingType (from))) { if (msg->msg_namelen < sizeof (sockaddr_ll)) @@ -235,7 +243,8 @@ UnixDatagramSocketFd::DoRecvmsg (struct msghdr *msg, int flags) { struct in_pktinfo pkt; Ptr ipv4 = node->GetObject (); - pkt.ipi_ifindex = ipv4->GetInterfaceForDevice (node->GetDevice (ipv4Tag.GetRecvIf ())); + // Interface index as used by netlink: NetDevice index + 1. + pkt.ipi_ifindex = ipv4Tag.GetRecvIf () + 1; if (msg->msg_name) { memcpy (&pkt.ipi_addr, msg->msg_name, sizeof (pkt.ipi_addr)); @@ -251,7 +260,7 @@ UnixDatagramSocketFd::DoRecvmsg (struct msghdr *msg, int flags) { struct in6_pktinfo pkt6; Ptr ipv6 = node->GetObject (); - pkt6.ipi6_ifindex = ipv6->GetInterfaceForDevice (node->GetDevice (ipv6Tag.GetRecvIf ())); + pkt6.ipi6_ifindex = ipv6Tag.GetRecvIf () + 1; if (msg->msg_name) { memcpy (&pkt6.ipi6_addr, msg->msg_name, sizeof (pkt6.ipi6_addr)); @@ -282,6 +291,24 @@ UnixDatagramSocketFd::DoSendmsg (const struct msghdr *msg, int flags) BooleanValue isIpHeaderIncluded (false); m_socket->GetAttributeFailSafe ("IpHeaderInclude", isIpHeaderIncluded); + // IPV6_PKTINFO selects the output interface, as needed to send to + // link-local multicast groups. Its index is the interface index that + // netlink link messages report, i.e. the NetDevice index + 1. + Ptr outDev = 0; + for (struct cmsghdr *c = CMSG_FIRSTHDR (msg); c != 0; c = CMSG_NXTHDR ((struct msghdr *)msg, c)) + { + if (c->cmsg_level == SOL_IPV6 && c->cmsg_type == IPV6_PKTINFO + && c->cmsg_len >= CMSG_LEN (sizeof (struct in6_pktinfo))) + { + struct in6_pktinfo *pkt6 = (struct in6_pktinfo *)CMSG_DATA (c); + Ptr node = current->process->manager->GetObject (); + if (pkt6->ipi6_ifindex >= 1 && pkt6->ipi6_ifindex <= node->GetNDevices ()) + { + outDev = node->GetDevice (pkt6->ipi6_ifindex - 1); + } + } + } + ssize_t retval = 0; Ipv4Header ipHeader; for (uint32_t i = 0; i < msg->msg_iovlen; ++i) @@ -356,7 +383,7 @@ UnixDatagramSocketFd::DoSendmsg (const struct msghdr *msg, int flags) result = -1; manager->ExecOnMain (MakeEvent (&UnixDatagramSocketFd::MainSendTo, - this, &result, packet, flags, ad)); + this, &result, packet, flags, ad, outDev)); } else { @@ -443,9 +470,17 @@ UnixDatagramSocketFd::Poll (PollTable* ptable) return ret; } void -UnixDatagramSocketFd::MainSendTo (int *r, Ptr p, uint32_t f, Address ad) +UnixDatagramSocketFd::MainSendTo (int *r, Ptr p, uint32_t f, Address ad, Ptr dev) { + if (!dev) + { + *r = m_socket->SendTo (p, f, ad); + return; + } + Ptr bound = m_socket->GetBoundNetDevice (); + m_socket->BindToNetDevice (dev); *r = m_socket->SendTo (p, f, ad); + m_socket->BindToNetDevice (bound); } void UnixDatagramSocketFd::MainSend (int *r, Ptr p) diff --git a/model/unix-datagram-socket-fd.h b/model/unix-datagram-socket-fd.h index ebda7a2a..d6aa2400 100644 --- a/model/unix-datagram-socket-fd.h +++ b/model/unix-datagram-socket-fd.h @@ -8,6 +8,8 @@ namespace ns3 { +class NetDevice; + class Address; class UnixDatagramSocketFd : public UnixSocketFd @@ -30,7 +32,7 @@ class UnixDatagramSocketFd : public UnixSocketFd uint32_t icmpInfo); void QueueErr (sock_extended_err ee, struct sockaddr_in offender, uint8_t ttl); void CopyMacAddress (const Address &a, uint8_t* const buf); - void MainSendTo (int *r, Ptr p, uint32_t f, Address ad); + void MainSendTo (int *r, Ptr p, uint32_t f, Address ad, Ptr dev); void MainSend (int *r, Ptr p); struct Error diff --git a/model/unix-socket-fd.cc b/model/unix-socket-fd.cc index 749a878d..18a604d4 100644 --- a/model/unix-socket-fd.cc +++ b/model/unix-socket-fd.cc @@ -34,9 +34,12 @@ #include "ns3/simulator.h" #include "ns3/netlink-socket-address.h" #include "ns3/inet6-socket-address.h" +#include "ns3/ipv6-l3-protocol.h" +#include "ns3/node.h" #include #include #include // need ICMP_FILTER +#include // need ICMP6_FILTER #include #include #include @@ -395,6 +398,7 @@ UnixSocketFd::Setsockopt (int level, int optname, case SOL_IPV6: switch (optname) { + case IPV6_RECVPKTINFO: case IPV6_PKTINFO: { if (optlen != sizeof (int)) @@ -405,6 +409,52 @@ UnixSocketFd::Setsockopt (int level, int optname, int *v = (int*)optval; m_socket->SetRecvPktInfo (*v ? true : false); } break; + case IPV6_JOIN_GROUP: + case IPV6_LEAVE_GROUP: + { + if (optlen < (socklen_t) sizeof (struct ipv6_mreq)) + { + current->err = EINVAL; + return -1; + } + const struct ipv6_mreq *mreq = (const struct ipv6_mreq *)optval; + Ptr node = current->process->manager->GetObject (); + Ptr ipv6 = node->GetObject (); + // The interface index is NetDevice index + 1, as reported by + // netlink; 0 means all interfaces. + if (!ipv6 || mreq->ipv6mr_interface > node->GetNDevices ()) + { + current->err = ENODEV; + return -1; + } + Ipv6Address group ((uint8_t *)mreq->ipv6mr_multiaddr.s6_addr); + if (mreq->ipv6mr_interface == 0) + { + if (optname == IPV6_JOIN_GROUP) + { + ipv6->AddMulticastAddress (group); + } + else + { + ipv6->RemoveMulticastAddress (group); + } + break; + } + int32_t ifIndex = ipv6->GetInterfaceForDevice (node->GetDevice (mreq->ipv6mr_interface - 1)); + if (ifIndex < 0) + { + current->err = ENODEV; + return -1; + } + if (optname == IPV6_JOIN_GROUP) + { + ipv6->AddMulticastAddress (group, ifIndex); + } + else + { + ipv6->RemoveMulticastAddress (group, ifIndex); + } + } break; // case IPV6_RECVPKTINFO: { // if (optlen != sizeof (int)) // { @@ -464,6 +514,26 @@ UnixSocketFd::Setsockopt (int level, int optname, break; } break; + case IPPROTO_ICMPV6: + switch (optname) + { + case ICMP6_FILTER: + { + if (optlen != sizeof (struct icmp6_filter)) + { + current->err = EINVAL; + return -1; + } + // ns-3 does not expose its ICMPv6 raw socket filter; accept the + // option and deliver all ICMPv6 types (applications such as + // zebra check the type of every message they read). + NS_LOG_WARN ("ICMP6_FILTER accepted but not applied"); + } break; + default: + NS_LOG_WARN ("Unsupported setsockopt requested. level: IPPROTO_ICMPV6, optname: " << optname); + break; + } + break; default: { NS_LOG_WARN ("Unsupported sockopt: level = " << level); diff --git a/model/unix-stream-socket-fd.cc b/model/unix-stream-socket-fd.cc index 06141425..561604c4 100644 --- a/model/unix-stream-socket-fd.cc +++ b/model/unix-stream-socket-fd.cc @@ -19,6 +19,7 @@ namespace ns3 { UnixStreamSocketFd::UnixStreamSocketFd (Ptr sock, bool connected) : UnixSocketFd (sock), m_backlog (0), + m_connectEstablished (false), m_peerAddress (0), m_shutWrite (0) { @@ -464,6 +465,7 @@ UnixStreamSocketFd::ConnectionSuccess (Ptr sock) if (CONNECTING == m_state) { m_state = CONNECTED; + m_connectEstablished = true; Address ad; if (0 == sock->GetSockName (ad)) { @@ -540,10 +542,12 @@ UnixStreamSocketFd::Connect (const struct sockaddr *my_addr, socklen_t addrlen) } m_state = CONNECTING; + m_connectEstablished = false; int sup = UnixSocketFd::Connect (my_addr, addrlen); + bool handshakeStarted = (0 == sup); - if (0 == sup) + if (handshakeStarted) { sup = -1; WaitQueueEntryTimeout *wq = new WaitQueueEntryTimeout (POLLIN | POLLHUP, GetRecvTimeout ()); @@ -576,7 +580,9 @@ UnixStreamSocketFd::Connect (const struct sockaddr *my_addr, socklen_t addrlen) delete wq; wq = 0; } - if (CONNECTED == m_state) + // The peer may accept and close the connection before this thread runs + // again; like Linux, connect () still succeeds and later reads see EOF. + if (CONNECTED == m_state || (REMOTECLOSED == m_state && m_connectEstablished)) { sup = 0; Address ad = PosixAddressToNs3Address (my_addr, addrlen); @@ -587,7 +593,7 @@ UnixStreamSocketFd::Connect (const struct sockaddr *my_addr, socklen_t addrlen) { sup = -1; - if ((m_state == CLOSED)||(REMOTECLOSED == m_state)) + if (handshakeStarted || (m_state == CLOSED) || (REMOTECLOSED == m_state)) { Current ()->err = ECONNREFUSED; } diff --git a/model/unix-stream-socket-fd.h b/model/unix-stream-socket-fd.h index 5c8b0923..111147d7 100644 --- a/model/unix-stream-socket-fd.h +++ b/model/unix-stream-socket-fd.h @@ -52,6 +52,7 @@ class UnixStreamSocketFd : public UnixSocketFd std::list,Address> > m_connectionQueue; int m_backlog; + bool m_connectEstablished; // the current Connect () completed its handshake State m_state; Address* m_peerAddress; bool m_shutWrite; diff --git a/model/utils.cc b/model/utils.cc index 7fe1fe5c..79bd85fd 100644 --- a/model/utils.cc +++ b/model/utils.cc @@ -201,9 +201,10 @@ UtilsSendSignal (Process *process, int signum) i != process->threads.end (); ++i) { Thread *thread = *i; - if (sigismember (&thread->signalMask, signum) == 0) + if (sigismember (&thread->signalMask, signum) == 0 + || sigismember (&thread->sigwaitSet, signum) == 1) { - // signal not blocked by thread. + // signal not blocked by thread, or the thread waits for it in sigwait. if (thread->task->IsBlocked ()) { process->manager->Wakeup (thread); diff --git a/netlink/netlink-socket.cc b/netlink/netlink-socket.cc index e566eb27..ec216a0f 100644 --- a/netlink/netlink-socket.cc +++ b/netlink/netlink-socket.cc @@ -798,7 +798,7 @@ NetlinkSocket::BuildInterfaceAddressDumpMessages (uint32_t received_seq) ); InterfaceAddressMessage ifamsg; - ifamsg.SetInterfaceIndex (i); + ifamsg.SetInterfaceIndex (Ipv4InterfaceToDevice (i)); ifamsg.SetFamily (AF_INET); //default AF_INET ifamsg.SetLength (mask_len); ifamsg.SetFlags (0); @@ -848,7 +848,7 @@ NetlinkSocket::BuildInterfaceAddressDumpMessages (uint32_t received_seq) NETLINK_RTM_NEWADDR, NETLINK_MSG_F_MULTI, received_seq, m_Pid); InterfaceAddressMessage ifamsg; - ifamsg.SetInterfaceIndex (i); + ifamsg.SetInterfaceIndex (Ipv6InterfaceToDevice (i)); ifamsg.SetFamily (AF_INET6); ifamsg.SetFlags (0); @@ -879,6 +879,18 @@ NetlinkSocket::BuildInterfaceAddressDumpMessages (uint32_t received_seq) return nlmsg_dump; } +uint32_t +NetlinkSocket::Ipv4InterfaceToDevice (uint32_t interface) const +{ + return m_node->GetObject ()->GetNetDevice (interface)->GetIfIndex () + 1; +} + +uint32_t +NetlinkSocket::Ipv6InterfaceToDevice (uint32_t interface) const +{ + return m_node->GetObject ()->GetNetDevice (interface)->GetIfIndex () + 1; +} + NetlinkMessage NetlinkSocket::BuildInterfaceInfoDumpMessage (uint32_t interface_num, uint32_t seq) { @@ -922,7 +934,8 @@ NetlinkSocket::BuildInterfaceInfoDumpMessage (uint32_t interface_num, uint32_t s ifinfomsg.SetFamily (0); // AF_UNSPEC ifinfomsg.SetDeviceType (0); // not clear - ifinfomsg.SetInterfaceIndex (interface_num); + // Linux interface indexes start at 1: NetDevice index + 1. + ifinfomsg.SetInterfaceIndex (interface_num + 1); ifinfomsg.SetDeviceFlags (flags); // not clear ifinfomsg.SetChangeMask (0xffffffff); @@ -997,8 +1010,8 @@ NetlinkSocket::BuildRouteDumpMessages (uint32_t seq) // ns3 use local address as the route src address // rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_SRC, ADDRESS, route.GetSource())); // rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_PREFSRC, ADDRESS, route.GetSource()));//not used in ns3 - rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_IIF, U32, route.GetInterface ())); - rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_OIF, U32, route.GetInterface ())); + rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_IIF, U32, Ipv4InterfaceToDevice (route.GetInterface ()))); + rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_OIF, U32, Ipv4InterfaceToDevice (route.GetInterface ()))); rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_GATEWAY, ADDRESS, route.GetGateway ())); nlmsg_rt.SetHeader (nhr); @@ -1032,8 +1045,8 @@ NetlinkSocket::BuildRouteDumpMessages (uint32_t seq) // ipv6->GetSourceAddress(route.GetDest ()))); // rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_PREFSRC, ADDRESS, // ipv6->GetSourceAddress(route.GetDest ()))); - rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_IIF, U32, route.GetInterface ())); - rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_OIF, U32, route.GetInterface ())); + rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_IIF, U32, Ipv6InterfaceToDevice (route.GetInterface ()))); + rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_OIF, U32, Ipv6InterfaceToDevice (route.GetInterface ()))); rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_GATEWAY, ADDRESS, route.GetGateway ())); nlmsg_rt.SetHeader (nhr); @@ -1281,6 +1294,32 @@ NetlinkSocket::DoRouteMessage (const NetlinkMessage &nlmsg, uint16_t type, uint8 Ipv6StaticRoutingHelper routingHelper6; Ptr ipv6Static = routingHelper6.GetStaticRouting (ipv6); + // Netlink interface indexes are NetDevice indexes + 1, as in link + // messages; routing tables are indexed by IP interface. + if (attr_flags[RouteMessage::RT_A_OIF]) + { + int32_t ifIndex = -1; + if (index >= 1 && index <= m_node->GetNDevices ()) + { + Ptr dev = m_node->GetDevice (index - 1); + if (family == AF_INET && ipv4) + { + ifIndex = ipv4->GetInterfaceForDevice (dev); + } + else if (family == AF_INET6 && ipv6) + { + ifIndex = ipv6->GetInterfaceForDevice (dev); + } + } + if (ifIndex < 0) + { + NS_LOG_DEBUG ("No IP interface for output device " << index); + m_errno = ERROR_ADDRNOTAVAIL; + return -1; + } + index = ifIndex; + } + NS_LOG_DEBUG (Simulator::Now ().GetSeconds () << " Route message, type: " << type << "; from " << m_node->GetObject ()->GetAddress (1, 0).GetLocal () << " to " << dest << " through " << gateway); @@ -1429,14 +1468,29 @@ NetlinkSocket::DoRouteMessage (const NetlinkMessage &nlmsg, uint16_t type, uint8 { if (!attr_flags[RouteMessage::RT_A_OIF]) { -#ifdef FIXME - if (ipv6->GetIfIndexForDestination (gateway6, index) == false) + // Find the interface on which the gateway is on-link. + bool found = false; + for (uint32_t i = 0; i < ipv6->GetNInterfaces () && !found; i++) + { + for (uint32_t j = 0; j < ipv6->GetNAddresses (i); j++) + { + Ipv6InterfaceAddress ifAddr = ipv6->GetAddress (i, j); + if (attr_flags[RouteMessage::RT_A_GATEWAY] + && !gateway6.IsLinkLocal () + && ifAddr.GetPrefix ().IsMatch (ifAddr.GetAddress (), gateway6)) + { + index = i; + found = true; + break; + } + } + } + if (!found) { NS_LOG_INFO ("No suitable interface to add an route entry"); m_errno = ERROR_ADDRNOTAVAIL; return -1; } -#endif } Ipv6Prefix pref (dstlen); @@ -1543,8 +1597,8 @@ NetlinkSocket::DoRouteMessage (const NetlinkMessage &nlmsg, uint16_t type, uint8 //ns3 use local address as the route src address // rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_SRC, ADDRESS, ipv4->GetSourceAddress(route.GetDest ()))); // rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_PREFSRC, ADDRESS, ipv4->GetSourceAddress(route.GetDest ()))); - rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_IIF, U32, route.GetInterface ())); - rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_OIF, U32, route.GetInterface ())); + rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_IIF, U32, Ipv4InterfaceToDevice (route.GetInterface ()))); + rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_OIF, U32, Ipv4InterfaceToDevice (route.GetInterface ()))); rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_GATEWAY, ADDRESS, route.GetGateway ())); //fill an netlink message body @@ -1583,8 +1637,8 @@ NetlinkSocket::DoRouteMessage (const NetlinkMessage &nlmsg, uint16_t type, uint8 //ns3 use local address as the route src address // rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_SRC, ADDRESS, ipv6->GetSourceAddress(route.GetDest ()))); // rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_PREFSRC, ADDRESS, ipv6->GetSourceAddress(route.GetDest ()))); - rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_IIF, U32, route.GetInterface ())); - rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_OIF, U32, route.GetInterface ())); + rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_IIF, U32, Ipv6InterfaceToDevice (route.GetInterface ()))); + rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_OIF, U32, Ipv6InterfaceToDevice (route.GetInterface ()))); rtmsg.AppendAttribute (NetlinkAttribute (RouteMessage::RT_A_GATEWAY, ADDRESS, route.GetGateway ())); //fill an netlink message body diff --git a/netlink/netlink-socket.h b/netlink/netlink-socket.h index 165cfdd5..3d607b5d 100644 --- a/netlink/netlink-socket.h +++ b/netlink/netlink-socket.h @@ -193,6 +193,10 @@ class NetlinkSocket : public Socket NetlinkMessage BuildInterfaceInfoDumpMessage (uint32_t interface_id, uint32_t seq); + // Netlink interface index (NetDevice index + 1) of an IP interface. + uint32_t Ipv4InterfaceToDevice (uint32_t interface) const; + uint32_t Ipv6InterfaceToDevice (uint32_t interface) const; + /** * \brief Build a multipart netlink message consisting of several * (possibly zero) InterfaceInfo dump messages diff --git a/test/dce-manager-test.cc b/test/dce-manager-test.cc index b4f7eb40..9cc4d001 100644 --- a/test/dce-manager-test.cc +++ b/test/dce-manager-test.cc @@ -199,12 +199,12 @@ DceManagerTestSuite::DceManagerTestSuite () { "test-timer-fd", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, { "test-stdlib", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, { "test-fork", 0, "", false, true, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-select", 3600, "", true, false, 0 /*LINUX_STACK*/}, + { "test-select", 3600, "", true, false, NS3_STACK|LINUX_STACK}, // freebsd-sim has no sock_poll { "test-nanosleep", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, { "test-random", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, { "test-local-socket", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-poll", 3200, "", true, false, 0 /*NS3_STACK|LINUX_STACK*/}, - { "test-tcp-socket", 320, "", true, false, 0/*LINUX_STACK*/}, + { "test-poll", 3200, "", true, false, NS3_STACK|LINUX_STACK}, // freebsd-sim has no sock_poll + { "test-tcp-socket", 320, "", true, false, NS3_STACK|LINUX_STACK}, { "test-exec", 0, "", false, true, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, { "test-raw-socket", 320, "", true, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, { "test-iperf", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, @@ -216,7 +216,7 @@ DceManagerTestSuite::DceManagerTestSuite () { "test-tsearch", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, { "test-clock-gettime", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, { "test-gcc-builtin-apply", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - // XXX: not completely tested { "test-signal", 30, "" , false}, + { "test-signal", 30, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, }; // Prepare directories and files for test-stdio diff --git a/utils/build_kernel_deps.sh b/utils/build_kernel_deps.sh new file mode 100755 index 00000000..d01df69d --- /dev/null +++ b/utils/build_kernel_deps.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# +# Build everything DCE needs to run its kernel-stack tests and examples: +# - net-next-nuse (Linux 4.4 libos) -> liblinux.so, and the kernel tree +# used by "waf configure --enable-kernel-stack" +# - freebsd-sim -> libfreebsd.so +# - DCE-compatible (PIE) builds of ip, iperf, thttpd, wget, ping/ping6 +# and the quagga routing daemons +# - the ns-3-dce-quagga module sources, patched for current ns-3 +# +# Usage: ./utils/build_kernel_deps.sh [deps_dir] +# deps_dir: output directory (default: ../dce-kernel-deps) +# JOBS: parallel make jobs (default: nproc) +# +# Outputs (sources and build trees stay in /src, which can be +# deleted afterwards): +# /bin_dce/ binaries and libraries for DCE_PATH +# /kernel/ argument for --enable-kernel-stack +# /ns-3-dce-quagga/ module to copy into myscripts/ +# + +set -e + +DCE_DIR="$(cd "$(dirname "$0")/.." && pwd)" +DEPS_DIR="$(mkdir -p "${1:-${DCE_DIR}/../dce-kernel-deps}" && cd "${1:-${DCE_DIR}/../dce-kernel-deps}" && pwd)" +BIN_DCE="${DEPS_DIR}/bin_dce" +SRC="${DEPS_DIR}/src" +JOBS="${JOBS:-$(nproc)}" + +NUSE_REV="libos-v4.4-fix1" +FREEBSD_REV="sim-ns3-dev-branch" +IPROUTE2_REV="v4.4.0" +IPUTILS_REV="s20101006" +QUAGGA_MODULE_REV="b57e0f3184e34107c7e452b3c41807e0aff5b5ce" +IPERF_VERSION="2.0.5" +THTTPD_VERSION="2.25b" +WGET_VERSION="1.15" +QUAGGA_VERSION="0.99.20" + +# Old C code: keep building with GCC >= 10 (-fcommon) and GCC >= 14, which +# turned these warnings into errors. -U_FORTIFY_SOURCE: DCE does not provide +# the *_chk fortified libc entry points. +LEGACY_CFLAGS="-fcommon -U_FORTIFY_SOURCE \ + -Wno-error=implicit-function-declaration -Wno-error=implicit-int \ + -Wno-error=int-conversion -Wno-error=incompatible-pointer-types" + +mkdir -p "${BIN_DCE}" "${SRC}" + +fetch_git () { + # fetch_git + # Shallow clone of a branch or tag; full clone only for a commit hash. + if [ ! -d "${SRC}/$3" ]; then + if ! git -c advice.detachedHead=false clone -q --depth 1 --branch "$2" "$1" "${SRC}/$3" 2> /dev/null; then + git clone -q "$1" "${SRC}/$3" + git -C "${SRC}/$3" -c advice.detachedHead=false checkout -q "$2" + fi + fi +} + +fetch_tar () { + # fetch_tar + if [ ! -d "${SRC}/$2" ]; then + wget -q -O "${SRC}/$2.tar" "$1" + tar xf "${SRC}/$2.tar" -C "${SRC}" + rm -f "${SRC}/$2.tar" + fi +} + +echo "== net-next-nuse (Linux 4.4 libos)" +fetch_git https://github.com/libos-nuse/net-next-nuse.git "${NUSE_REV}" net-next-nuse +( + cd "${SRC}/net-next-nuse" + make defconfig ARCH=lib > /dev/null + # Generate objs.mk serially: under -j, GNU Make >= 4.4 runs this rule + # twice concurrently and the output file gets interleaved. + make ARCH=lib arch/lib/objs.mk > /dev/null + # The NUSE/rump tools fail to build and are not used by DCE; only the + # libsim-linux library from arch/lib/tools is required. + make library ARCH=lib -j"${JOBS}" -k > build.log 2>&1 || true + test -f arch/lib/tools/libsim-linux-4.4.0.so || { tail -50 build.log; exit 1; } +) +cp "${SRC}/net-next-nuse/arch/lib/tools/libsim-linux-4.4.0.so" "${BIN_DCE}/" +ln -sf libsim-linux-4.4.0.so "${BIN_DCE}/liblinux.so" +# DCE only needs the libos API headers from the kernel tree. +rm -rf "${DEPS_DIR}/kernel" +mkdir -p "${DEPS_DIR}/kernel/lib" +cp -a "${SRC}/net-next-nuse/arch/lib/include" "${DEPS_DIR}/kernel/lib/" + +echo "== freebsd-sim" +fetch_git https://github.com/direct-code-execution/freebsd-sim.git "${FREEBSD_REV}" freebsd-sim +( + cd "${SRC}/freebsd-sim" + git apply --check "${DCE_DIR}/utils/freebsd-sim-dce.patch" 2> /dev/null \ + && git apply "${DCE_DIR}/utils/freebsd-sim-dce.patch" + cd sys/sim + # The kernel config tool and generated headers must exist before the + # parallel kernel build; the Makefile does not order this correctly. + make -f Makefile.config clean > /dev/null + make -f Makefile.config CC="gcc ${LEGACY_CFLAGS}" > /dev/null + rm -rf compile && mkdir -p compile + (cd conf && ../config -C DCE > /dev/null) + ( + cd compile/DCE + for m in device_if cpufreq_if bus_if linker_if clock_if; do + awk -f ../../../tools/makeobjops.awk ../../../kern/$m.m -h + done + awk -f ../../../tools/vnode_if.awk ../../../kern/vnode_if.src -q -p -h -c + ) + touch config + make buildkernel -j"${JOBS}" CC="gcc ${LEGACY_CFLAGS}" > build.log 2>&1 || { tail -50 build.log; exit 1; } +) +cp "${SRC}/freebsd-sim/libsim-freebsd.git.so" "${BIN_DCE}/" +ln -sf libsim-freebsd.git.so "${BIN_DCE}/libfreebsd.so" + +echo "== iproute2 ${IPROUTE2_REV}" +fetch_git https://git.kernel.org/pub/scm/network/iproute2/iproute2.git "${IPROUTE2_REV}" iproute2 +( + cd "${SRC}/iproute2" + ./configure > /dev/null + # glibc has setns(); the configure probe for it fails under GCC 14. + make -j"${JOBS}" "CCOPTS=-fpic -D_GNU_SOURCE -O0 -g -DUINT16_MAX=65535 -DHAVE_SETNS ${LEGACY_CFLAGS}" \ + "LDFLAGS=-pie -rdynamic" > /dev/null 2>&1 +) +cp "${SRC}/iproute2/ip/ip" "${BIN_DCE}/" + +echo "== iperf ${IPERF_VERSION}" +fetch_tar "https://sourceforge.net/projects/iperf/files/iperf-${IPERF_VERSION}.tar.gz/download" "iperf-${IPERF_VERSION}" +( + cd "${SRC}/iperf-${IPERF_VERSION}" + patch -p1 -N -s < "${DCE_DIR}/utils/iperf_4_dce.patch" || true + ./configure -q CFLAGS="-g -fPIC ${LEGACY_CFLAGS}" CXXFLAGS="-g -fPIC -U_FORTIFY_SOURCE" \ + LDFLAGS="-pie -rdynamic" > /dev/null + make -j"${JOBS}" > /dev/null +) +cp "${SRC}/iperf-${IPERF_VERSION}/src/iperf" "${BIN_DCE}/" + +echo "== thttpd ${THTTPD_VERSION}" +fetch_tar "http://www.acme.com/software/thttpd/thttpd-${THTTPD_VERSION}.tar.gz" "thttpd-${THTTPD_VERSION}" +( + cd "${SRC}/thttpd-${THTTPD_VERSION}" + sed -i "s/rm conftest.c/rm -f conftests.c/" configure + CC="gcc ${LEGACY_CFLAGS}" ./configure > /dev/null 2>&1 + # Same changes as utils/dce-thttpd.patch, which no longer applies to the + # generated Makefile. No -fpie: it would make stdin/stdout/stderr copy + # relocations, which DCE's per-process stdio cannot reach. + sed -i -e "s/ -DHAVE_SIGSET=1//" \ + -e "s/^CFLAGS =\t\(.*\)/CFLAGS =\t\1 -fPIC -g ${LEGACY_CFLAGS}/" \ + -e "s/^LDFLAGS =.*/LDFLAGS =\t-pie -rdynamic/" Makefile + make thttpd > /dev/null 2>&1 +) +cp "${SRC}/thttpd-${THTTPD_VERSION}/thttpd" "${BIN_DCE}/" + +echo "== wget ${WGET_VERSION}" +fetch_tar "https://ftp.gnu.org/gnu/wget/wget-${WGET_VERSION}.tar.gz" "wget-${WGET_VERSION}" +( + cd "${SRC}/wget-${WGET_VERSION}" + CFLAGS="-fPIC -g ${LEGACY_CFLAGS}" LDFLAGS="-pie -rdynamic" ./configure -q \ + --disable-opie --disable-digest --disable-ntlm --disable-largefile --disable-threads \ + --disable-nls --disable-rpath --disable-iri --without-ssl --without-zlib \ + --without-libiconv-prefix --without-libintl-prefix --without-libpth-prefix \ + --without-included-regex > /dev/null + make -j"${JOBS}" > /dev/null 2>&1 +) +cp "${SRC}/wget-${WGET_VERSION}/src/wget" "${BIN_DCE}/" + +echo "== iputils ${IPUTILS_REV}" +fetch_git https://github.com/iputils/iputils.git "${IPUTILS_REV}" iputils +( + cd "${SRC}/iputils" + patch -p1 -N -s < "${DCE_DIR}/utils/iputils-ping6.patch" || true + make CFLAGS="-fpic -D_GNU_SOURCE -g ${LEGACY_CFLAGS}" LDFLAGS="-pie -rdynamic" ping ping6 > /dev/null +) +cp "${SRC}/iputils/ping" "${SRC}/iputils/ping6" "${BIN_DCE}/" + +echo "== quagga ${QUAGGA_VERSION}" +fetch_tar "https://src.fedoraproject.org/repo/pkgs/quagga/quagga-${QUAGGA_VERSION}.tar.gz/64cc29394eb8a4e24649d19dac868f64/quagga-${QUAGGA_VERSION}.tar.gz" "quagga-${QUAGGA_VERSION}" +( + cd "${SRC}/quagga-${QUAGGA_VERSION}" + CFLAGS="-fPIC -g ${LEGACY_CFLAGS}" LDFLAGS="-pie -rdynamic" ./configure -q \ + --disable-shared --enable-static --disable-user --disable-group \ + --disable-capabilities > /dev/null + make -j"${JOBS}" > /dev/null 2>&1 +) +for d in zebra ripd ripngd ospfd ospf6d bgpd; do + cp "${SRC}/quagga-${QUAGGA_VERSION}/${d}/${d}" "${BIN_DCE}/" +done + +echo "== ns-3-dce-quagga module" +fetch_git https://github.com/direct-code-execution/ns-3-dce-quagga.git "${QUAGGA_MODULE_REV}" ns-3-dce-quagga +( + cd "${SRC}/ns-3-dce-quagga" + git apply --check "${DCE_DIR}/utils/ns-3-dce-quagga-ns3.patch" 2> /dev/null \ + && git apply "${DCE_DIR}/utils/ns-3-dce-quagga-ns3.patch" +) +rm -rf "${DEPS_DIR}/ns-3-dce-quagga" +cp -a "${SRC}/ns-3-dce-quagga" "${DEPS_DIR}/ns-3-dce-quagga" +rm -rf "${DEPS_DIR}/ns-3-dce-quagga/.git" + +echo "Done. Contents of ${BIN_DCE}:" +ls "${BIN_DCE}" diff --git a/utils/freebsd-sim-dce.patch b/utils/freebsd-sim-dce.patch new file mode 100644 index 00000000..5a655935 --- /dev/null +++ b/utils/freebsd-sim-dce.patch @@ -0,0 +1,52 @@ +--- a/sys/sim/generate-linker-script.py ++++ b/sys/sim/generate-linker-script.py +@@ -1,4 +1,4 @@ +-#!/usr/bin/env python ++#!/usr/bin/env python3 + + import re + +--- a/usr.sbin/config/main.c ++++ b/usr.sbin/config/main.c +@@ -121,7 +121,7 @@ + int * + __error(void) + { +- int test = 0; ++ static int test = 0; + return (&test); + } + +--- a/sys/sim/sim-device.c ++++ b/sys/sim/sim-device.c +@@ -525,7 +525,7 @@ + + static const u_char etherbroadcastaddr[ETHER_ADDR_LEN] = + { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +-struct SimDevice *sim_dev_create (void *priv, enum SimDevFlags flags) ++struct SimDevice *sim_dev_create (const char *ifname, void *priv, enum SimDevFlags flags) + { + /* FIXME!! */ + static uint32_t devnum = 0; +--- a/sys/sim/sim.c ++++ b/sys/sim/sim.c +@@ -79,7 +79,7 @@ + extern void sim_task_destroy (struct SimTask *task); + extern void *sim_task_get_private (struct SimTask *task); + +-FORWARDER2(sim_dev_create, nvoid, struct SimDevice *, void *, enum SimDevFlags); ++FORWARDER3(sim_dev_create, nvoid, struct SimDevice *, const char *, void *, enum SimDevFlags); + FORWARDER1(sim_dev_destroy, void, void, struct SimDevice *); + FORWARDER2(sim_dev_set_address, void, void, struct SimDevice *, unsigned char *); + FORWARDER2(sim_dev_set_mtu, void, void, struct SimDevice *, int); +--- a/sys/sim/include/sim-init.h ++++ b/sys/sim/include/sim-init.h +@@ -38,7 +38,7 @@ + void (*sock_poll) (struct SimSocket *socket, void *ret); + void (*sock_pollfreewait) (void *polltable); + +- struct SimDevice *(*dev_create) (void *priv, enum SimDevFlags flags); ++ struct SimDevice *(*dev_create) (const char *ifname, void *priv, enum SimDevFlags flags); + void (*dev_destroy) (struct SimDevice *dev); + void *(*dev_get_private) (struct SimDevice *task); + void (*dev_set_address) (struct SimDevice *dev, unsigned char buffer[6]); diff --git a/utils/ns-3-dce-quagga-ns3.patch b/utils/ns-3-dce-quagga-ns3.patch new file mode 100644 index 00000000..c050bced --- /dev/null +++ b/utils/ns-3-dce-quagga-ns3.patch @@ -0,0 +1,491 @@ +diff -ruN a/example/dce-quagga-bgpd-caida.cc b/example/dce-quagga-bgpd-caida.cc +--- a/example/dce-quagga-bgpd-caida.cc ++++ b/example/dce-quagga-bgpd-caida.cc +@@ -99,7 +99,7 @@ + topoHelp.SetFileType (format); + inFile = topoHelp.GetTopologyReader (); + +- if (inFile != 0) ++ if (inFile) + { + nodes = inFile->Read (); + } +diff -ruN a/example/dce-quagga-bgpd.cc b/example/dce-quagga-bgpd.cc +--- a/example/dce-quagga-bgpd.cc ++++ b/example/dce-quagga-bgpd.cc +@@ -111,7 +111,7 @@ + { + // processManager.SetLoader ("ns3::DlmLoaderFactory"); + processManager.SetTaskManagerAttribute ("FiberManagerType", +- EnumValue (0)); ++ StringValue ("UcontextFiberManager")); + processManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", + "Library", StringValue ("liblinux.so")); + processManager.Install (nodes); +diff -ruN a/example/dce-quagga-ospf6d.cc b/example/dce-quagga-ospf6d.cc +--- a/example/dce-quagga-ospf6d.cc ++++ b/example/dce-quagga-ospf6d.cc +@@ -110,7 +110,7 @@ + { + // processManager.SetLoader ("ns3::DlmLoaderFactory"); + processManager.SetTaskManagerAttribute ("FiberManagerType", +- EnumValue (0)); ++ StringValue ("UcontextFiberManager")); + processManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", + "Library", StringValue ("liblinux.so")); + processManager.Install (nodes); +diff -ruN a/example/dce-quagga-ospfd-rocketfuel.cc b/example/dce-quagga-ospfd-rocketfuel.cc +--- a/example/dce-quagga-ospfd-rocketfuel.cc ++++ b/example/dce-quagga-ospfd-rocketfuel.cc +@@ -26,7 +26,7 @@ + #include "ns3/point-to-point-helper.h" + #include "ns3/topology-read-module.h" + +-#include "ns3/v4ping.h" ++#include "ns3/ping.h" + + #include + #undef NS3_MPI +@@ -100,7 +100,7 @@ + topoHelp.SetFileType (format); + inFile = topoHelp.GetTopologyReader (); + +- if (inFile != 0) ++ if (inFile) + { + nodes = inFile->Read (); + } +@@ -198,11 +198,11 @@ + if (systemId == 0) + #endif + { +- Ptr app = CreateObject (); ++ Ptr app = CreateObject (); + //Ptr ipv4Server = nodes.Get (25)->GetObject (); + Ptr ipv4Server = nodes.Get (nodes.GetN () - 1)->GetObject (); +- app->SetAttribute ("Remote", Ipv4AddressValue (ipv4Server->GetAddress (1, 0).GetLocal ())); +- app->SetAttribute ("Verbose", BooleanValue (true)); ++ app->SetAttribute ("Destination", AddressValue (Ipv4Address (ipv4Server->GetAddress (1, 0).GetLocal ()))); ++ app->SetAttribute ("VerboseMode", EnumValue (Ping::VerboseMode::VERBOSE)); + nodes.Get (0)->AddApplication (app); + app->SetStartTime (Seconds (20.0)); + app->SetStopTime (Seconds (stopTime)); +diff -ruN a/example/dce-quagga-ospfd.cc b/example/dce-quagga-ospfd.cc +--- a/example/dce-quagga-ospfd.cc ++++ b/example/dce-quagga-ospfd.cc +@@ -109,7 +109,7 @@ + { + // processManager.SetLoader ("ns3::DlmLoaderFactory"); + processManager.SetTaskManagerAttribute ("FiberManagerType", +- EnumValue (0)); ++ StringValue ("UcontextFiberManager")); + processManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", + "Library", StringValue ("liblinux.so")); + processManager.Install (nodes); +diff -ruN a/example/dce-quagga-radvd.cc b/example/dce-quagga-radvd.cc +--- a/example/dce-quagga-radvd.cc ++++ b/example/dce-quagga-radvd.cc +@@ -110,7 +110,7 @@ + { + // processManager.SetLoader ("ns3::DlmLoaderFactory"); + processManager.SetTaskManagerAttribute ("FiberManagerType", +- EnumValue (0)); ++ StringValue ("UcontextFiberManager")); + processManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", + "Library", StringValue ("liblinux.so")); + processManager.Install (nodes); +diff -ruN a/example/dce-quagga-ripd.cc b/example/dce-quagga-ripd.cc +--- a/example/dce-quagga-ripd.cc ++++ b/example/dce-quagga-ripd.cc +@@ -110,7 +110,7 @@ + { + // processManager.SetLoader ("ns3::DlmLoaderFactory"); + processManager.SetTaskManagerAttribute ("FiberManagerType", +- EnumValue (0)); ++ StringValue ("UcontextFiberManager")); + processManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", + "Library", StringValue ("liblinux.so")); + processManager.Install (nodes); +diff -ruN a/example/dce-quagga-ripngd.cc b/example/dce-quagga-ripngd.cc +--- a/example/dce-quagga-ripngd.cc ++++ b/example/dce-quagga-ripngd.cc +@@ -110,7 +110,7 @@ + { + // processManager.SetLoader ("ns3::DlmLoaderFactory"); + processManager.SetTaskManagerAttribute ("FiberManagerType", +- EnumValue (0)); ++ StringValue ("UcontextFiberManager")); + processManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", + "Library", StringValue ("liblinux.so")); + processManager.Install (nodes); +diff -ruN a/example/dce-zebra-simple.cc b/example/dce-zebra-simple.cc +--- a/example/dce-zebra-simple.cc ++++ b/example/dce-zebra-simple.cc +@@ -47,7 +47,7 @@ + NS_LOG_FUNCTION (node->GetId ()); + + Ptr ipv4 = node->GetObject (); +- NS_ASSERT_MSG (ipv4 != 0, "ipv4 should not be null"); ++ NS_ASSERT_MSG (ipv4, "ipv4 should not be null"); + + // The thing I didn't know is that interface status (isUp/isDown) and device status (isLinkUp) are two totally different things. It is possible to set interface up and down, but there is not an easy way to alter status of device. This similar to reality: it is possible to programmatically disable interface, but to actually disable a link one needs to physically cut the cable + ipv4->SetDown (2); +diff -ruN a/helper/quagga-helper.cc b/helper/quagga-helper.cc +--- a/helper/quagga-helper.cc ++++ b/helper/quagga-helper.cc +@@ -58,12 +58,6 @@ + ; + return tid; + } +- TypeId +- GetInstanceTypeId (void) const +- { +- return GetTypeId (); +- } +- + void + SetFilename (const std::string &filename) + { +@@ -128,12 +122,6 @@ + ; + return tid; + } +- TypeId +- GetInstanceTypeId (void) const +- { +- return GetTypeId (); +- } +- + void + addNetwork (std::string prefix, uint32_t area) + { +@@ -235,12 +223,6 @@ + ; + return tid; + } +- TypeId +- GetInstanceTypeId (void) const +- { +- return GetTypeId (); +- } +- + void + SetAsn (uint32_t lasn) + { +@@ -405,12 +387,6 @@ + ; + return tid; + } +- TypeId +- GetInstanceTypeId (void) const +- { +- return GetTypeId (); +- } +- + + void + SetFilename (const std::string &filename) +@@ -495,12 +471,6 @@ + ; + return tid; + } +- TypeId +- GetInstanceTypeId (void) const +- { +- return GetTypeId (); +- } +- + + void + SetFilename (const std::string &filename) +@@ -575,12 +545,6 @@ + ; + return tid; + } +- TypeId +- GetInstanceTypeId (void) const +- { +- return GetTypeId (); +- } +- + + void + SetFilename (const std::string &filename) +@@ -647,7 +611,7 @@ + Ptr ospf_conf = nodes.Get (i)->GetObject (); + if (!ospf_conf) + { +- ospf_conf = new OspfConfig (); ++ ospf_conf = CreateObject (); + nodes.Get (i)->AggregateObject (ospf_conf); + } + +@@ -662,7 +626,7 @@ + Ptr ospf_conf = node->GetObject (); + if (!ospf_conf) + { +- ospf_conf = new OspfConfig (); ++ ospf_conf = CreateObject (); + node->AggregateObject (ospf_conf); + } + ospf_conf->SetRouterId (routerid); +@@ -678,7 +642,7 @@ + Ptr ospf_conf = nodes.Get (i)->GetObject (); + if (!ospf_conf) + { +- ospf_conf = new OspfConfig (); ++ ospf_conf = CreateObject (); + nodes.Get (i)->AggregateObject (ospf_conf); + } + ospf_conf->m_ospfdebug = true; +@@ -695,7 +659,7 @@ + Ptr zebra_conf = nodes.Get (i)->GetObject (); + if (!zebra_conf) + { +- zebra_conf = new QuaggaConfig (); ++ zebra_conf = CreateObject (); + nodes.Get (i)->AggregateObject (zebra_conf); + } + zebra_conf->m_zebradebug = true; +@@ -709,7 +673,7 @@ + Ptr zebra_conf = node->GetObject (); + if (!zebra_conf) + { +- zebra_conf = new QuaggaConfig (); ++ zebra_conf = CreateObject (); + node->AggregateObject (zebra_conf); + } + +@@ -726,7 +690,7 @@ + Ptr zebra_conf = node->GetObject (); + if (!zebra_conf) + { +- zebra_conf = new QuaggaConfig (); ++ zebra_conf = CreateObject (); + node->AggregateObject (zebra_conf); + } + +@@ -743,7 +707,7 @@ + Ptr zebra_conf = nodes.Get (i)->GetObject (); + if (!zebra_conf) + { +- zebra_conf = new QuaggaConfig (); ++ zebra_conf = CreateObject (); + nodes.Get (i)->AggregateObject (zebra_conf); + } + zebra_conf->m_usemanualconf = true; +@@ -818,7 +782,7 @@ + Ptr ospf6_conf = nodes.Get (i)->GetObject (); + if (!ospf6_conf) + { +- ospf6_conf = new Ospf6Config (); ++ ospf6_conf = CreateObject (); + nodes.Get (i)->AggregateObject (ospf6_conf); + } + +@@ -837,7 +801,7 @@ + Ptr ospf6_conf = nodes.Get (i)->GetObject (); + if (!ospf6_conf) + { +- ospf6_conf = new Ospf6Config (); ++ ospf6_conf = CreateObject (); + nodes.Get (i)->AggregateObject (ospf6_conf); + } + ospf6_conf->m_ospf6debug = true; +@@ -854,7 +818,7 @@ + Ptr rip_conf = nodes.Get (i)->GetObject (); + if (!rip_conf) + { +- rip_conf = new RipConfig (); ++ rip_conf = CreateObject (); + nodes.Get (i)->AggregateObject (rip_conf); + } + +@@ -872,7 +836,7 @@ + Ptr rip_conf = nodes.Get (i)->GetObject (); + if (!rip_conf) + { +- rip_conf = new RipConfig (); ++ rip_conf = CreateObject (); + nodes.Get (i)->AggregateObject (rip_conf); + } + rip_conf->m_ripdebug = true; +@@ -889,7 +853,7 @@ + Ptr ripng_conf = nodes.Get (i)->GetObject (); + if (!ripng_conf) + { +- ripng_conf = new RipngConfig (); ++ ripng_conf = CreateObject (); + nodes.Get (i)->AggregateObject (ripng_conf); + } + +@@ -907,7 +871,7 @@ + Ptr ripng_conf = nodes.Get (i)->GetObject (); + if (!ripng_conf) + { +- ripng_conf = new RipngConfig (); ++ ripng_conf = CreateObject (); + nodes.Get (i)->AggregateObject (ripng_conf); + } + ripng_conf->m_ripngdebug = true; +@@ -1149,7 +1113,7 @@ + Ptr zebra_conf = node->GetObject (); + if (!zebra_conf) + { +- zebra_conf = new QuaggaConfig (); ++ zebra_conf = CreateObject (); + node->AggregateObject (zebra_conf); + } + GenerateConfigZebra (node); +diff -ruN a/test/dce-quagga-test.cc b/test/dce-quagga-test.cc +--- a/test/dce-quagga-test.cc ++++ b/test/dce-quagga-test.cc +@@ -24,8 +24,7 @@ + #include "ns3/dce-module.h" + #include "ns3/quagga-helper.h" + #include "ns3/csma-helper.h" +-#include "ns3/v4ping.h" +-#include "ns3/ping6.h" ++#include "ns3/ping.h" + #include "ns3/ethernet-header.h" + #include "ns3/icmpv4.h" + +@@ -156,6 +155,14 @@ + // + std::string routerPort; + ++ // Routers answer for any of their addresses, like Linux does. ++ Config::SetDefault ("ns3::Ipv6::StrongEndSystemModel", BooleanValue (false)); ++ ++ // IPv6 ping target: node 0's address on its second interface. ++ Ipv6Address target6 ("2001:db8:0:2::1"); ++ // IPv6 addresses of node 0 and node 1 on the shared link. ++ std::string link6[2] = {"2001:db8:0:1::1", "2001:db8:0:1::2"}; ++ + NodeContainer nodes; + nodes.Create (2); + +@@ -169,7 +176,7 @@ + + // processManager.SetLoader ("ns3::DlmLoaderFactory"); + processManager.SetTaskManagerAttribute ("FiberManagerType", +- EnumValue (0)); ++ StringValue ("UcontextFiberManager")); + // + // Step 2 + // Address Configuration +@@ -188,12 +195,19 @@ + ipv4AddrHelper.SetBase ("10.0.0.0", "255.255.255.0"); + ipv4AddrHelper.Assign (devices); + ipv6AddrHelper.SetBase ("2001:db8:0:1::", Ipv6Prefix (64)); +- ipv6AddrHelper.Assign (devices); ++ Ipv6InterfaceContainer linkIfs = ipv6AddrHelper.Assign (devices); ++ for (uint32_t n = 0; n < 2; n++) ++ { ++ std::ostringstream oss; ++ oss << linkIfs.GetAddress (n, 1); ++ link6[n] = oss.str (); ++ } + + ipv4AddrHelper.SetBase ("11.0.0.0", "255.255.255.0"); + ipv4AddrHelper.Assign (dev1); + ipv6AddrHelper.NewNetwork (); +- ipv6AddrHelper.Assign (dev1); ++ // ns-3 derives the interface identifier from the MAC address. ++ target6 = ipv6AddrHelper.Assign (dev1).GetAddress (0, 1); + + ipv4AddrHelper.SetBase ("12.0.0.0", "255.255.255.0"); + ipv4AddrHelper.Assign (dev2); +@@ -284,8 +298,8 @@ + else if (m_testname == "bgpd_v6") + { + quagga.EnableBgp (nodes); +- quagga.BgpAddNeighbor (nodes.Get (0), "2001:db8:0:1::2", quagga.GetAsn (nodes.Get (1))); +- quagga.BgpAddNeighbor (nodes.Get (1), "2001:db8:0:1::1", quagga.GetAsn (nodes.Get (0))); ++ quagga.BgpAddNeighbor (nodes.Get (0), link6[1], quagga.GetAsn (nodes.Get (1))); ++ quagga.BgpAddNeighbor (nodes.Get (1), link6[0], quagga.GetAsn (nodes.Get (0))); + } + else if (m_testname == "radvd") + { +@@ -326,11 +340,11 @@ + } + else + { +- Ptr app = CreateObject (); +- app->SetAttribute ("Remote", Ipv4AddressValue ("11.0.0.1")); ++ Ptr app = CreateObject (); ++ app->SetAttribute ("Destination", AddressValue (Ipv4Address ("11.0.0.1"))); + if (m_debug) + { +- app->SetAttribute ("Verbose", BooleanValue (true)); ++ app->SetAttribute ("VerboseMode", EnumValue (Ping::VerboseMode::VERBOSE)); + } + nodes.Get (1)->AddApplication (app); + app->SetStartTime (Seconds (60.0)); +@@ -352,9 +366,8 @@ + } + else + { +- Ptr app = CreateObject (); +- app->SetAttribute ("RemoteIpv6", Ipv6AddressValue ("2001:db8:0:2::1")); +- app->SetIfIndex (0); ++ Ptr app = CreateObject (); ++ app->SetAttribute ("Destination", AddressValue (target6)); + nodes.Get (1)->AddApplication (app); + app->SetStartTime (Seconds (60.0)); + app->SetStopTime (m_maxDuration); +@@ -407,7 +420,7 @@ + + + DceQuaggaTestSuite::DceQuaggaTestSuite () +- : TestSuite ("dce-quagga", UNIT) ++ : TestSuite ("dce-quagga", Type::UNIT) + { + typedef struct + { +@@ -417,24 +430,20 @@ + } testPair; + + const testPair tests[] = { +-#ifdef FIXME + { "radvd", 120, false}, + { "ripd", 120, false}, + { "ripngd", 120, false}, + { "ospf6d", 120, false}, + { "bgpd_v6", 120, false}, +-#endif + { "ospfd", 120, false}, + { "bgpd", 120, false}, + { "radvd", 120, true}, + { "ripd", 120, true}, + { "ripngd", 120, true}, + { "ospfd", 120, true}, +-#ifdef FIXME + { "ospf6d", 120, true}, + { "bgpd", 120, true}, + { "bgpd_v6", 120, true}, +-#endif + }; + + ::system ("/bin/rm -rf files-*/usr/local/etc/*.pid"); +@@ -452,7 +461,7 @@ + AddTestCase (new DceQuaggaTestCase (std::string (tests[i].name), + Seconds (tests[i].duration), tests[i].useKernel, + (!kern && tests[i].useKernel)), +- TestCase::QUICK); ++ Duration::QUICK); + } + } + +diff -ruN a/wscript b/wscript +--- a/wscript ++++ b/wscript +@@ -30,7 +30,7 @@ + + + def build_dce_tests(module, bld): +- module.add_runner_test(needed=['core', 'dce-quagga', 'internet', 'csma'], ++ module.add_runner_test(needed=['core', 'dce-quagga', 'internet', 'internet-apps', 'csma'], + source=['test/dce-quagga-test.cc']) + + def build_dce_examples(module): From 10a124901a97c9f91ce2574a28273f079df998c6 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Wed, 7 Oct 2026 18:26:52 +0200 Subject: [PATCH 02/71] Port to ns-3.49 and drop Ubuntu 22.04 from CI - Ipv4InterfaceAddress no longer has secondary addresses: drop the IsSecondary() checks in the Linux and FreeBSD IPv4 glue - Ipv4GlobalRouting moved to ns3/global-routing.h - Address::Register() now takes the address kind and its length - CI: build against ns-3.49; drop Ubuntu 22.04, whose CMake (3.22) is older than ns-3.49 requires 79/79 tests and examples pass on Ubuntu 24.04 and Debian trixie. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-and-test.yml | 4 +--- helper/dce-manager-helper.cc | 2 +- model/freebsd/ipv4-freebsd.cc | 10 +--------- model/linux/ipv4-linux.cc | 10 +--------- netlink/netlink-socket-address.cc | 2 +- 5 files changed, 5 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 863c2a79..57480e4b 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -13,7 +13,7 @@ concurrency: cancel-in-progress: true env: - NS3_VER: "3.46" + NS3_VER: "3.49" jobs: build: @@ -23,8 +23,6 @@ jobs: fail-fast: false matrix: include: - - distro: ubuntu-22.04 - image: ubuntu:22.04 - distro: ubuntu-24.04 image: ubuntu:24.04 - distro: debian-trixie diff --git a/helper/dce-manager-helper.cc b/helper/dce-manager-helper.cc index f76bafa2..3a51eb9a 100644 --- a/helper/dce-manager-helper.cc +++ b/helper/dce-manager-helper.cc @@ -14,7 +14,7 @@ #include "ipv4-linux.h" #include "dce-application-helper.h" #include "ns3/ipv4-routing-protocol.h" -#include "ns3/ipv4-global-routing.h" +#include "ns3/global-routing.h" #include #include #include diff --git a/model/freebsd/ipv4-freebsd.cc b/model/freebsd/ipv4-freebsd.cc index 6ccc6416..1574878a 100644 --- a/model/freebsd/ipv4-freebsd.cc +++ b/model/freebsd/ipv4-freebsd.cc @@ -26,7 +26,7 @@ #include "ns3/ipv4-global-routing-helper.h" #include "ns3/ipv4-interface.h" #include "ns3/arp-l3-protocol.h" -#include "ns3/ipv4-global-routing.h" +#include "ns3/global-routing.h" #include "ns3/ipv4-routing-table-entry.h" #include "dce-application-helper.h" #include "freebsd-socket-fd-factory.h" @@ -398,10 +398,6 @@ Ipv4FreeBSD::SelectSourceAddress (Ptr device, for (uint32_t j = 0; j < GetNAddresses (i); j++) { iaddr = GetAddress (i, j); - if (iaddr.IsSecondary ()) - { - continue; - } if (iaddr.GetScope () > scope) { continue; @@ -428,10 +424,6 @@ Ipv4FreeBSD::SelectSourceAddress (Ptr device, for (uint32_t j = 0; j < GetNAddresses (i); j++) { iaddr = GetAddress (i, j); - if (iaddr.IsSecondary ()) - { - continue; - } if (iaddr.GetScope () != Ipv4InterfaceAddress::LINK && iaddr.GetScope () <= scope) { diff --git a/model/linux/ipv4-linux.cc b/model/linux/ipv4-linux.cc index 5e4d157e..b3d27fdb 100644 --- a/model/linux/ipv4-linux.cc +++ b/model/linux/ipv4-linux.cc @@ -27,7 +27,7 @@ #include "ns3/ipv4-global-routing-helper.h" #include "ns3/ipv4-interface.h" #include "ns3/arp-l3-protocol.h" -#include "ns3/ipv4-global-routing.h" +#include "ns3/global-routing.h" #include "ns3/ipv4-routing-table-entry.h" #include "linux-stack-helper.h" #include "linux-ipv4-raw-socket-factory-impl.h" @@ -378,10 +378,6 @@ Ipv4Linux::SelectSourceAddress (Ptr device, for (uint32_t j = 0; j < GetNAddresses (i); j++) { iaddr = GetAddress (i, j); - if (iaddr.IsSecondary ()) - { - continue; - } if (iaddr.GetScope () > scope) { continue; @@ -408,10 +404,6 @@ Ipv4Linux::SelectSourceAddress (Ptr device, for (uint32_t j = 0; j < GetNAddresses (i); j++) { iaddr = GetAddress (i, j); - if (iaddr.IsSecondary ()) - { - continue; - } if (iaddr.GetScope () != Ipv4InterfaceAddress::LINK && iaddr.GetScope () <= scope) { diff --git a/netlink/netlink-socket-address.cc b/netlink/netlink-socket-address.cc index 895cbfc0..4adb4caf 100644 --- a/netlink/netlink-socket-address.cc +++ b/netlink/netlink-socket-address.cc @@ -117,7 +117,7 @@ bool NetlinkSocketAddress::IsMatchingType (const Address &address) uint8_t NetlinkSocketAddress::GetType (void) { - static uint8_t type = Address::Register (); + static uint8_t type = Address::Register ("NetlinkSocketAddress", 8); return type; } From 72994bed7aca9e910781d7f49a5b3fb356d2631e Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Thu, 8 Oct 2026 13:58:45 +0200 Subject: [PATCH 03/71] DCE: core fixes found while porting the kernel stack to LKL - TaskManager: - RunningTask(): the task whose fiber runs, ignoring the hi task of DceManager temporary tasks (only a fiber can block). - RunNow(): from the main fiber, run the ready tasks until a condition holds, without advancing time. - SetTaskEndNotifier(): called when a task exits or is stopped. - LinuxSocketImpl (ns-3 sockets backed by the kernel): - Uses the generic UnixFd, not the libos KernelSocketFd. - Delivers the data that arrived with the end of a connection before treating the socket as closed. - After a send failed with EAGAIN, notifies the application again only once time went on: DCCP reports sockets writable while its sends fail, and the failed send wakes the socket again (zero-time livelock). - getaddrinfo: ignore AI_ADDRCONFIG (also the default without hints), which checks the host's addresses, not the node's: without IPv6 on the host, IPv6 addresses did not resolve. - if_nametoindex/if_indextoname ask the kernel of LKL nodes too, and close the socket they open. - ns3waf: the headers copied to build/include/ns3 are outputs of the copy task, so that waf rebuilds what includes them when they change (stale example objects crashed after a class layout change). Co-Authored-By: Claude Opus 5.5 --- model/dce-netdb.cc | 17 ++++++++++++ model/dce.cc | 16 ++++++++---- model/linux/linux-socket-impl.cc | 45 +++++++++++++++++++++++--------- model/linux/linux-socket-impl.h | 6 +++-- model/task-manager.cc | 39 ++++++++++++++++++++++++++- model/task-manager.h | 18 +++++++++++++ ns3waf/__init__.py | 14 +++++----- 7 files changed, 129 insertions(+), 26 deletions(-) diff --git a/model/dce-netdb.cc b/model/dce-netdb.cc index 587789cb..7219061d 100644 --- a/model/dce-netdb.cc +++ b/model/dce-netdb.cc @@ -59,6 +59,23 @@ int dce_getaddrinfo (const char *node, const char *service, NS_LOG_FUNCTION (Current () << UtilsGetNodeId () << ((NULL == node) ? "" : node) << ((NULL == service) ? "" : service) << hints << res); NS_ASSERT (Current () != 0); struct addrinfo *tmp = 0; + // AI_ADDRCONFIG (also the default without hints) would check the + // addresses of the host, not of the node: without IPv6 on the host, IPv6 + // addresses would not resolve. + struct addrinfo nodeHints; + memset (&nodeHints, 0, sizeof (nodeHints)); + if (hints == 0) + { + nodeHints.ai_family = AF_UNSPEC; + nodeHints.ai_flags = AI_V4MAPPED; + hints = &nodeHints; + } + else if (hints->ai_flags & AI_ADDRCONFIG) + { + nodeHints = *hints; + nodeHints.ai_flags &= ~AI_ADDRCONFIG; + hints = &nodeHints; + } int status = ::getaddrinfo (node, service, hints, &tmp); // copy outgoing data structure so that the memory is allocated from the calling process memory pool. struct addrinfo *cur, *prev, *head; diff --git a/model/dce.cc b/model/dce.cc index 62c8ed2a..54810624 100644 --- a/model/dce.cc +++ b/model/dce.cc @@ -694,7 +694,10 @@ unsigned dce_if_nametoindex (const char *ifname) Ptr factory = 0; factory = current->process->manager->GetObject (); - if (factory->GetInstanceTypeId () == TypeId::LookupByName ("ns3::LinuxSocketFdFactory")) + // Kernel stacks know their interfaces; ask them. + TypeId tid = factory->GetInstanceTypeId (); + if (tid == TypeId::LookupByName ("ns3::LinuxSocketFdFactory") + || tid.GetName () == "ns3::LklSocketFdFactory") { struct ifreq ifr; int fd = dce_socket (AF_INET, SOCK_DGRAM, 0); @@ -704,10 +707,11 @@ unsigned dce_if_nametoindex (const char *ifname) } strncpy (ifr.ifr_name, ifname, sizeof (ifr.ifr_name)); - if (dce_ioctl (fd, SIOCGIFINDEX, (char *)&ifr) < 0) + int ret = dce_ioctl (fd, SIOCGIFINDEX, (char *)&ifr); + dce_close (fd); + if (ret < 0) { - current->err = errno; - return -1; + return 0; } return ifr.ifr_ifindex; } @@ -739,7 +743,9 @@ char * dce_if_indextoname (unsigned ifindex, char *ifname) } ifr.ifr_ifindex = ifindex; - if (dce_ioctl (fd, SIOCGIFNAME, (char *)&ifr) < 0) + int ret = dce_ioctl (fd, SIOCGIFNAME, (char *)&ifr); + dce_close (fd); + if (ret < 0) { return 0; } diff --git a/model/linux/linux-socket-impl.cc b/model/linux/linux-socket-impl.cc index e4c1cf31..54f30094 100644 --- a/model/linux/linux-socket-impl.cc +++ b/model/linux/linux-socket-impl.cc @@ -24,7 +24,7 @@ #include "ns3/uinteger.h" #include "ns3/event-id.h" #include "linux-socket-fd-factory.h" -#include "kernel-socket-fd.h" +#include "unix-fd.h" #include "linux-socket-impl.h" #include "dce-manager.h" #include "process.h" @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -143,6 +144,7 @@ LinuxSocketImpl::LinuxSocketImpl () NS_LOG_FUNCTION_NOARGS (); m_listening = false; m_conn_inprogress = false; + m_sendBlocked = false; m_pid = -1; SetNs3ToPosixConverter (MakeCallback (&LinuxSocketImpl::Ns3AddressToPosixAddress, this)); SetPosixToNs3Converter (MakeCallback (&LinuxSocketImpl::PosixAddressToNs3Address, this)); @@ -357,6 +359,11 @@ LinuxSocketImpl::Send (Ptr p, uint32_t flags) if (ret < 0) { NS_LOG_INFO ("send returns " << ret << " errno " << Current ()->err); + if (Current ()->err == EAGAIN) + { + m_sendBlocked = true; + m_sendBlockedAt = Simulator::Now (); + } } delete[] buf; LeaveFakeTask (pid); @@ -585,10 +592,6 @@ LinuxSocketImpl::Poll () mask &= (POLLIN | POLLOUT | POLLERR | POLLHUP | POLLRDHUP); if (mask) { - Ptr factory = 0; - Ptr node = GetNode (); - Ptr manager = node->GetObject (); - factory = manager->GetObject (); if (m_listening) { struct sockaddr_storage my_addr; @@ -608,9 +611,9 @@ LinuxSocketImpl::Poll () { NS_LOG_INFO ("accept error"); } - KernelSocketFd *kern_sock; + UnixFd *kern_sock; FileUsage *fu = Current ()->process->openFiles[sock]; - kern_sock = (KernelSocketFd *)fu->GetFileInc (); + kern_sock = fu->GetFileInc (); kern_sock->IncFdCount (); kern_sock->Fcntl (F_SETFL, O_NONBLOCK); @@ -635,8 +638,20 @@ LinuxSocketImpl::Poll () } else { + // Data that arrived with the end of the connection is + // still to be read. + int pending = 0; + if (mask & POLLIN) + { + pid = EnterFakeTask (); + if (this->m_kernsock->Ioctl (FIONREAD, (char *)&pending) < 0) + { + pending = 0; + } + LeaveFakeTask (pid); + } // FIXME: handle closed socket - if (mask & POLLRDHUP || mask & POLLHUP || mask & POLLERR) + if ((mask & POLLRDHUP || mask & POLLHUP || mask & POLLERR) && pending == 0) { NS_LOG_FUNCTION ("socket has closed ?" << mask); // FIXME: may need m_closed flag @@ -652,8 +667,15 @@ LinuxSocketImpl::Poll () } else if (mask & POLLOUT) { - Simulator::ScheduleWithContext (m_node->GetId (), Seconds (0.0), - MakeEvent (&LinuxSocketImpl::NotifySend, this, 0)); + // Some protocols (e.g. DCCP) report the socket writable + // while sends fail with EAGAIN, and a failed send wakes + // the socket again: notify again only once time went on. + if (!m_sendBlocked || Simulator::Now () > m_sendBlockedAt) + { + m_sendBlocked = false; + Simulator::ScheduleWithContext (m_node->GetId (), Seconds (0.0), + MakeEvent (&LinuxSocketImpl::NotifySend, this, 0)); + } NS_LOG_INFO ("wait send for next poll event"); table->Wait (Seconds (0)); NS_LOG_INFO ("awaken"); @@ -709,8 +731,7 @@ LinuxSocketImpl::CreateSocket () NS_LOG_FUNCTION_NOARGS (); uint16_t pid = EnterFakeTask (); Ptr factory = m_node->GetObject (); - KernelSocketFd *kern_sock = (KernelSocketFd *) - factory->CreateSocket (m_family, m_socktype, m_protocol); + UnixFd *kern_sock = factory->CreateSocket (m_family, m_socktype, m_protocol); LeaveFakeTask (pid); if (!kern_sock) { diff --git a/model/linux/linux-socket-impl.h b/model/linux/linux-socket-impl.h index 55b99bad..1d86ed7a 100644 --- a/model/linux/linux-socket-impl.h +++ b/model/linux/linux-socket-impl.h @@ -29,7 +29,7 @@ namespace ns3 { class Node; class Packet; class Task; -class KernelSocketFd; +class UnixFd; class LinuxSocketImpl : public Socket { @@ -85,7 +85,7 @@ class LinuxSocketImpl : public Socket void SetNs3ToPosixConverter (LinuxSocketImpl::Ns3ToPosixConverter cb); void SetPosixToNs3Converter (LinuxSocketImpl::PosixToNs3Converter cb); - KernelSocketFd *m_kernsock; + UnixFd *m_kernsock; Task *m_task; Ns3ToPosixConverter m_ns3toposix; PosixToNs3Converter m_posixtons3; @@ -117,6 +117,8 @@ class LinuxSocketImpl : public Socket uint16_t m_protocol; bool m_listening; bool m_conn_inprogress; + bool m_sendBlocked; // the last send failed with EAGAIN + Time m_sendBlockedAt; uint16_t m_pid; EventId m_poll; diff --git a/model/task-manager.cc b/model/task-manager.cc index 6019a3ef..0edc8e53 100644 --- a/model/task-manager.cc +++ b/model/task-manager.cc @@ -107,7 +107,9 @@ TaskManager::TaskManager () m_disposing (0), m_todoOnMain (0), m_noSignal (0), - m_hightask (0) + m_hightask (0), + m_taskEndNotifier (0), + m_taskEndNotifierContext (0) { NS_LOG_FUNCTION (this); } @@ -286,6 +288,10 @@ TaskManager::Stop (Task *task) return; } + if (m_taskEndNotifier) + { + m_taskEndNotifier (task, false, m_taskEndNotifierContext); + } // we can delete the task immediately. NS_LOG_DEBUG ("delete " << task << " fiber=" << task->m_fiber); m_scheduler->Dequeue (task); @@ -367,12 +373,23 @@ TaskManager::Exit (void) NS_ASSERT (m_current != 0); NS_ASSERT (m_current->m_state == Task::RUNNING); Task *current = m_current; + if (m_taskEndNotifier) + { + m_taskEndNotifier (current, true, m_taskEndNotifierContext); + } current->m_state = Task::DEAD; current->m_waitTimer.Cancel (); m_deadTasks.push_back (current); Schedule (); } +void +TaskManager::SetTaskEndNotifier (void (*fn)(Task *task, bool running, void *context), void *context) +{ + m_taskEndNotifier = fn; + m_taskEndNotifierContext = context; +} + void TaskManager::EnterHiTask (Task *task) { @@ -397,6 +414,26 @@ TaskManager::CurrentTask (void) } return m_current; } +Task * +TaskManager::RunningTask (void) +{ + return m_current; +} +bool +TaskManager::RunNow (const bool *done) +{ + NS_LOG_FUNCTION (this); + NS_ASSERT (m_current == 0); + // The tasks run here are not the hi task. + Task *hightask = m_hightask; + m_hightask = 0; + while (!*done && m_scheduler->PeekNext () != 0) + { + Schedule (); + } + m_hightask = hightask; + return *done; +} TaskManager * TaskManager::Current (void) { diff --git a/model/task-manager.h b/model/task-manager.h index 0021e68e..5a75135f 100644 --- a/model/task-manager.h +++ b/model/task-manager.h @@ -125,6 +125,17 @@ class TaskManager : public Object * running. */ Task * CurrentTask (void); + /** + * Returns the task whose fiber is executing, ignoring the task entered + * with EnterHiTask, or 0 on the main fiber. + */ + Task * RunningTask (void); + /** + * From the main fiber: run the ready tasks, without advancing the + * simulation time, until *done is true. Returns false if no task is + * ready while *done is still false. + */ + bool RunNow (const bool *done); /** * Returns the 'current' task manager. * Note that this function does not return a reference (i.e., the caller does not @@ -145,6 +156,11 @@ class TaskManager : public Object void LeaveHiTask (Task *task); void SetSwitchNotify (void (*fn)(void)); + /** + * fn is called when a task ends: by the task itself when it calls Exit + * (running is true), or when it is stopped (running is false). + */ + void SetTaskEndNotifier (void (*fn)(Task *task, bool running, void *context), void *context); uint32_t GetStackSize (Task *task) const; /** @@ -193,6 +209,8 @@ class TaskManager : public Object EventImpl *m_todoOnMain; bool m_noSignal; // I am not come back from a real thread interruption do not run signal .... bool m_disposing; // In order to never loop while disposing me. + void (*m_taskEndNotifier)(Task *task, bool running, void *context); + void *m_taskEndNotifierContext; }; } // namespace diff --git a/ns3waf/__init__.py b/ns3waf/__init__.py index a24574ef..43386009 100644 --- a/ns3waf/__init__.py +++ b/ns3waf/__init__.py @@ -345,11 +345,9 @@ def _build_headers(bld, name, headers): import os import shutil def run(task): - out_dir = os.path.dirname(task.outputs[0].abspath()) - for header in task.inputs: - dst = os.path.join(out_dir, os.path.basename(header.abspath())) - src = header.abspath() - shutil.copyfile(src, dst) + # outputs: the module header, then a copy of each input header + for header, copy in zip(task.inputs, task.outputs[1:]): + shutil.copyfile(header.abspath(), copy.abspath()) with open(task.outputs[0].abspath(), "w") as outfile: @@ -370,7 +368,11 @@ def run(task): out_relpath = os.path.relpath(bld.out_dir, str(bld.out_dir) + "/" + bld.path.path_from(bld.srcnode)) target = os.path.join(out_relpath, 'include', 'ns3', '%s-module.h' % name) - bld(rule=run, source=headers, target=target) + # The copies are outputs too, so that waf rebuilds what includes them + # when a header changes. + copies = [os.path.join(out_relpath, 'include', 'ns3', os.path.basename(h)) + for h in headers] + bld(rule=run, source=headers, target=[target] + copies) bld(use=[target], target='NS3_HEADERS_%s' % name.upper(), export_includes=['include']) bld.install_files(os.path.join('${PREFIX}', 'include', 'ns3'), headers + [target]) From 5efde39cdfed18149207d9644b165dc6f9476239 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Thu, 8 Oct 2026 13:58:45 +0200 Subject: [PATCH 04/71] LKL: build script and kernel configuration utils/build_lkl.sh builds LKL (Linux 6.12, pinned commit, shallow fetch) for DCE: it applies utils/lkl-patches, configures LKL's defconfig plus utils/lkl-dce.config, and relinks liblkl.so without default libraries (its libc references bind to the simulator's libc; DCE loads a copy per kernel instance) and stripped, with the debug information in liblkl.so.debug. It also copies LKL's headers: configure DCE with --with-lkl=. utils/lkl-dce.config adds what the examples and tests use, as the libos kernel had it: SCTP, DCCP (CCID 3), MPTCP, AF_PACKET, IPsec and its crypto, Mobile IPv6, tunnels, multicast routing, more TCP congestion controls. Patch 0001 backports the upstream fix for the 6.12 deadline server counted in nr_running (52d15521eb75), which kept an idle kernel busy. Co-Authored-By: Claude Opus 5.5 --- utils/build_lkl.sh | 74 +++++++++++++++++++ utils/lkl-dce.config | 38 ++++++++++ ...on-t-count-nr_running-for-dl_server-.patch | 45 +++++++++++ 3 files changed, 157 insertions(+) create mode 100755 utils/build_lkl.sh create mode 100644 utils/lkl-dce.config create mode 100644 utils/lkl-patches/0001-sched-deadline-Don-t-count-nr_running-for-dl_server-.patch diff --git a/utils/build_lkl.sh b/utils/build_lkl.sh new file mode 100755 index 00000000..10e2f3f9 --- /dev/null +++ b/utils/build_lkl.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Build LKL (the Linux Kernel Library) for DCE. +# +# DCE loads a separate copy of the kernel library for each node. liblkl.so +# is relinked without default libraries, so that its few libc references +# bind to the simulator's libc, and stripped, since each node gets a copy; +# the debug information is kept in liblkl.so.debug, which gdb finds through +# the library's debug link. +# +# Usage: ./utils/build_lkl.sh [lkl_dir] [out_dir] +# lkl_dir: LKL source tree, cloned if missing (default: ../lkl) +# out_dir: where liblkl.so and the LKL headers (include/) are written +# (default: /dce) +# JOBS: parallel make jobs (default: nproc) +# +# Configure DCE with --with-lkl= and add to DCE_PATH. +# +# The patches in utils/lkl-patches are applied first: see the commit +# message in each of them. The kernel is configured with LKL's defconfig +# plus utils/lkl-dce.config. +# + +set -e + +DCE_DIR="$(cd "$(dirname "$0")/.." && pwd)" +LKL_DIR="${1:-${DCE_DIR}/../lkl}" +LKL_REV="d0f76a77e8c2ca68bd7d96190b8da09040eab072" # LKL master, Linux 6.12, 2026-09-28 +JOBS="${JOBS:-$(nproc)}" + +if [ ! -d "${LKL_DIR}" ]; then + # Shallow fetch of the pinned commit: the full history is several GB. + git init -q "${LKL_DIR}" + git -C "${LKL_DIR}" fetch -q --depth 1 https://github.com/lkl/linux "${LKL_REV}" + git -C "${LKL_DIR}" -c advice.detachedHead=false checkout -q FETCH_HEAD +fi +LKL_DIR="$(cd "${LKL_DIR}" && pwd)" + +for patch in "${DCE_DIR}"/utils/lkl-patches/*.patch; do + if git -C "${LKL_DIR}" apply --reverse --check "${patch}" 2> /dev/null; then + continue # already applied + fi + git -C "${LKL_DIR}" apply "${patch}" +done +OUT_DIR="${2:-${LKL_DIR}/dce}" + +DEFCONFIG="${LKL_DIR}/arch/lkl/configs/dce_defconfig" +cat "${LKL_DIR}/arch/lkl/configs/defconfig" "${DCE_DIR}/utils/lkl-dce.config" > "${DEFCONFIG}.new" +if ! cmp -s "${DEFCONFIG}.new" "${DEFCONFIG}"; then + mv "${DEFCONFIG}.new" "${DEFCONFIG}" + rm -f "${LKL_DIR}/.config" # reconfigure +else + rm "${DEFCONFIG}.new" +fi + +make -C "${LKL_DIR}/tools/lkl" -j"${JOBS}" KCONFIG=dce_defconfig > "${LKL_DIR}/dce-build.log" 2>&1 \ + || { tail -50 "${LKL_DIR}/dce-build.log"; exit 1; } + +mkdir -p "${OUT_DIR}" +gcc -shared -nostdlib -o "${OUT_DIR}/liblkl.so" \ + -Wl,--whole-archive "${LKL_DIR}/tools/lkl/liblkl.a" -Wl,--no-whole-archive \ + -Wl,-z,noexecstack +( + cd "${OUT_DIR}" + objcopy --only-keep-debug liblkl.so liblkl.so.debug + objcopy --strip-debug --add-gnu-debuglink=liblkl.so.debug liblkl.so +) + +# The headers DCE is built with (some are generated by the build). +rm -rf "${OUT_DIR}/include" +cp -a "${LKL_DIR}/tools/lkl/include" "${OUT_DIR}/include" + +echo "Built ${OUT_DIR}/liblkl.so" +echo "Configure DCE with --with-lkl=${OUT_DIR} and add ${OUT_DIR} to DCE_PATH." diff --git a/utils/lkl-dce.config b/utils/lkl-dce.config new file mode 100644 index 00000000..510db091 --- /dev/null +++ b/utils/lkl-dce.config @@ -0,0 +1,38 @@ +# Kernel options DCE needs on top of LKL's defconfig: the protocols and +# features the DCE examples and tests use, as the libos kernel had them. +CONFIG_IP_SCTP=y +CONFIG_IP_DCCP=y +CONFIG_IP_DCCP_CCID3=y +CONFIG_MPTCP=y +CONFIG_MPTCP_IPV6=y +CONFIG_PACKET=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_INET_AH=y +CONFIG_INET_ESP=y +CONFIG_INET_IPCOMP=y +CONFIG_INET6_AH=y +CONFIG_INET6_ESP=y +CONFIG_INET6_IPCOMP=y +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_TUNNEL=y +CONFIG_NET_IPIP=y +CONFIG_NET_IPGRE_DEMUX=y +CONFIG_NET_IPGRE=y +CONFIG_IP_MROUTE=y +CONFIG_IPV6_MROUTE=y +CONFIG_TCP_CONG_RENO=y +CONFIG_TCP_CONG_HTCP=y +CONFIG_TCP_CONG_VEGAS=y +CONFIG_TCP_CONG_WESTWOOD=y +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_DES=y diff --git a/utils/lkl-patches/0001-sched-deadline-Don-t-count-nr_running-for-dl_server-.patch b/utils/lkl-patches/0001-sched-deadline-Don-t-count-nr_running-for-dl_server-.patch new file mode 100644 index 00000000..2ed82b0e --- /dev/null +++ b/utils/lkl-patches/0001-sched-deadline-Don-t-count-nr_running-for-dl_server-.patch @@ -0,0 +1,45 @@ +From dc1b4d1ff1445f22cf1003912caefaf40858d4a7 Mon Sep 17 00:00:00 2001 +From: Gabriel Ferreira +Date: Wed, 7 Oct 2026 20:22:33 +0200 +Subject: [PATCH 1/4] sched/deadline: Don't count nr_running for dl_server + proxy tasks + +Backport of upstream commit 52d15521eb75 ("sched/deadline: Don't count +nr_running for dl_server proxy tasks") to the 6.12 base of LKL. + +The fair server is a deadline entity that runs other tasks; counting it +in nr_running makes single_task_running() report two runnable tasks when +only one exists. LKL relies on single_task_running() to decide whether a +host task must yield the CPU, so the miscount made idle_host_task yield +to the idle task forever once the tick was stopped. +--- + kernel/sched/deadline.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c +index be1b917dc..7c81922f8 100644 +--- a/kernel/sched/deadline.c ++++ b/kernel/sched/deadline.c +@@ -1874,7 +1874,8 @@ void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) + u64 deadline = dl_se->deadline; + + dl_rq->dl_nr_running++; +- add_nr_running(rq_of_dl_rq(dl_rq), 1); ++ if (!dl_server(dl_se)) ++ add_nr_running(rq_of_dl_rq(dl_rq), 1); + + inc_dl_deadline(dl_rq, deadline); + } +@@ -1884,7 +1885,8 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) + { + WARN_ON(!dl_rq->dl_nr_running); + dl_rq->dl_nr_running--; +- sub_nr_running(rq_of_dl_rq(dl_rq), 1); ++ if (!dl_server(dl_se)) ++ sub_nr_running(rq_of_dl_rq(dl_rq), 1); + + dec_dl_deadline(dl_rq, dl_se->deadline); + } +-- +2.53.0 + From 534901dde8ec25daa9be1c5404ebc5da769c41be Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Thu, 8 Oct 2026 13:58:46 +0200 Subject: [PATCH 05/71] LKL: patches for kernels on simulated time (TOUPSTREAM) - 0002 lkl: stop the tick while the host is idle. The CPU idles in idle_host_task, a regular task, and after system calls while the host task stays current: the tick never stopped, and a simulator processes every tick (~100 per second per node). Enter NOHZ idle there and leave it when work resumes (system call, thread exit, woken task); re-stop the tick after interrupts that wake nothing, at the outermost level only; mark the clocksource valid for high resolution timers. - 0003 crypto: jitter - do not loop forever on a clock that does not advance. Outside FIPS mode, jent_gen_entropy() never gave up on stuck measurements: the kernel hung at boot on simulated time once the DRBG (pulled in by IPsec) was built in. - 0004 lkl: show_stack: stop at the first frame outside of the kernel. It followed the frame chain into host code without frame pointers and crashed the host on any kernel warning. TOUPSTREAM Co-Authored-By: Claude Opus 5.5 --- ...stop-the-tick-while-the-host-is-idle.patch | 222 ++++++++++++++++++ ...-not-loop-forever-on-a-clock-that-do.patch | 54 +++++ ...top-at-the-first-frame-outside-of-th.patch | 46 ++++ 3 files changed, 322 insertions(+) create mode 100644 utils/lkl-patches/0002-lkl-stop-the-tick-while-the-host-is-idle.patch create mode 100644 utils/lkl-patches/0003-crypto-jitter-do-not-loop-forever-on-a-clock-that-do.patch create mode 100644 utils/lkl-patches/0004-lkl-show_stack-stop-at-the-first-frame-outside-of-th.patch diff --git a/utils/lkl-patches/0002-lkl-stop-the-tick-while-the-host-is-idle.patch b/utils/lkl-patches/0002-lkl-stop-the-tick-while-the-host-is-idle.patch new file mode 100644 index 00000000..ff19a6d4 --- /dev/null +++ b/utils/lkl-patches/0002-lkl-stop-the-tick-while-the-host-is-idle.patch @@ -0,0 +1,222 @@ +From 4b5514bacfc3215b0907b7ad28fda7ddee54be0e Mon Sep 17 00:00:00 2001 +From: Gabriel Ferreira +Date: Wed, 7 Oct 2026 20:22:33 +0200 +Subject: [PATCH 2/4] lkl: stop the tick while the host is idle + +The CPU idles in idle_host_task, a regular task, rather than in the idle +task, so the idle loop restarts the tick when it switches to it. Also, +when a system call returns with nothing else to run, the calling host +task stays current while its thread runs host code: the kernel looks +busy although nothing runs. Either way an idle kernel took a timer +interrupt every jiffy, even with NO_HZ_IDLE. Hosts that run LKL in +virtual time (e.g. ns-3 DCE, where time only advances between events) +process every one of them. + +Enter NOHZ idle in both cases (idle_host_task, or the end of a system +call, when no other task is runnable), as the idle loop would, and +leave it when work resumes: a system call, the exit of a host thread, +or a task woken by an interrupt. After interrupts that wake nothing, +stop the tick again so that the next timer is programmed, as the next +pass of the idle loop would; only at the outermost level, as interrupts +can run nested in softirq context (a transmit completion while a +received packet is processed), where the softirqs they raise are still +pending. + +The clocksource also has to be marked valid for high resolution timers, +otherwise the tick never switches to one-shot mode and NO_HZ never +takes effect. + +With this, an idle kernel sleeping 10 s takes 7 timer events instead +of about 1000, and a program that makes a system call every second no +longer keeps the tick running between them. +--- + arch/lkl/include/asm/syscalls.h | 2 ++ + arch/lkl/kernel/cpu.c | 2 ++ + arch/lkl/kernel/irq.c | 2 ++ + arch/lkl/kernel/syscalls.c | 60 +++++++++++++++++++++++++++++++++ + arch/lkl/kernel/time.c | 2 +- + 5 files changed, 67 insertions(+), 1 deletion(-) + +diff --git a/arch/lkl/include/asm/syscalls.h b/arch/lkl/include/asm/syscalls.h +index 09b78212b..6e596127a 100644 +--- a/arch/lkl/include/asm/syscalls.h ++++ b/arch/lkl/include/asm/syscalls.h +@@ -5,6 +5,8 @@ int syscalls_init(void); + void syscalls_cleanup(void); + long lkl_syscall(long no, long *params); + void wakeup_idle_host_task(void); ++void lkl_idle_host_exit(void); ++void lkl_idle_host_irq_exit(void); + long lkl_sys_halt(void); + + #define sys_mmap sys_mmap_pgoff +diff --git a/arch/lkl/kernel/cpu.c b/arch/lkl/kernel/cpu.c +index 4afab4817..2a473a701 100644 +--- a/arch/lkl/kernel/cpu.c ++++ b/arch/lkl/kernel/cpu.c +@@ -122,6 +122,7 @@ void lkl_cpu_put(void) + cpu.irqs_pending = false; + lkl_ops->mutex_unlock(cpu.lock); + run_irqs(); ++ lkl_idle_host_irq_exit(); + lkl_ops->mutex_lock(cpu.lock); + } + +@@ -130,6 +131,7 @@ void lkl_cpu_put(void) + if (in_interrupt()) + lkl_bug("%s: in interrupt\n", __func__); + lkl_ops->mutex_unlock(cpu.lock); ++ lkl_idle_host_exit(); + thread_sched_jb(); + return; + } +diff --git a/arch/lkl/kernel/irq.c b/arch/lkl/kernel/irq.c +index fdbabc3e6..54c5686ab 100644 +--- a/arch/lkl/kernel/irq.c ++++ b/arch/lkl/kernel/irq.c +@@ -1,3 +1,4 @@ ++#include + #include + #include + #include +@@ -94,6 +95,7 @@ int lkl_trigger_irq(int irq) + } + + run_irq(irq); ++ lkl_idle_host_irq_exit(); + + lkl_cpu_put(); + +diff --git a/arch/lkl/kernel/syscalls.c b/arch/lkl/kernel/syscalls.c +index cd4431ff2..aaa68ef22 100644 +--- a/arch/lkl/kernel/syscalls.c ++++ b/arch/lkl/kernel/syscalls.c +@@ -1,3 +1,5 @@ ++#include ++#include + #include + #include + #include +@@ -95,6 +97,7 @@ static void del_host_task(void *arg) + if (lkl_cpu_get() < 0) + return; + ++ lkl_idle_host_exit(); + switch_to_host_task(task); + host_task_id--; + set_ti_thread_flag(ti, TIF_SCHED_JB); +@@ -103,6 +106,28 @@ static void del_host_task(void *arg) + + static struct lkl_tls_key *task_key; + ++/* ++ * The CPU is idle when no task runs: the host is busy elsewhere. That is ++ * the case in idle_host_task, a regular task, rather than in the idle task ++ * (the core idle loop restarts the tick when it switches to it), and when ++ * a system call returns with nothing else to run, as the host task stays ++ * current while its thread runs host code. Stop the tick then, as the idle ++ * loop would, so that an idle kernel gets no timer interrupts until its ++ * next timer actually expires. ++ */ ++static bool idle_host_nohz; ++ ++static void idle_host_enter(void) ++{ ++ if (idle_host_nohz) ++ return; ++ tick_nohz_idle_enter(); ++ local_irq_disable(); ++ tick_nohz_idle_stop_tick(); ++ local_irq_enable(); ++ idle_host_nohz = true; ++} ++ + long lkl_syscall(long no, long *params) + { + struct task_struct *task = host0; +@@ -112,6 +137,8 @@ long lkl_syscall(long no, long *params) + if (ret < 0) + return ret; + ++ lkl_idle_host_exit(); ++ + if (lkl_ops->tls_get) { + task = lkl_ops->tls_get(task_key); + if (!task) { +@@ -132,6 +159,8 @@ long lkl_syscall(long no, long *params) + } + + out: ++ if (single_task_running()) ++ idle_host_enter(); + lkl_cpu_put(); + + return ret; +@@ -139,6 +168,34 @@ long lkl_syscall(long no, long *params) + + static struct task_struct *idle_host_task; + ++ ++/* ++ * Called after interrupts ran while the host is idle and nothing became ++ * runnable: like the next pass of the idle loop, stop the tick again so ++ * that the next timer event is programmed. Not for interrupts that ran ++ * nested in interrupt or softirq context (e.g. a transmit completion ++ * while a received packet is processed): the softirqs they raised only ++ * run when that context ends. ++ */ ++void lkl_idle_host_irq_exit(void) ++{ ++ if (idle_host_nohz && test_thread_flag(TIF_HOST_THREAD) && ++ !in_interrupt() && single_task_running()) { ++ local_irq_disable(); ++ tick_nohz_idle_stop_tick(); ++ local_irq_enable(); ++ } ++} ++ ++/* Called when work resumes on the CPU: a system call or a woken task. */ ++void lkl_idle_host_exit(void) ++{ ++ if (idle_host_nohz) { ++ idle_host_nohz = false; ++ tick_nohz_idle_exit(); ++ } ++} ++ + /* called from idle, don't failed, don't block */ + void wakeup_idle_host_task(void) + { +@@ -155,6 +212,8 @@ static int idle_host_task_loop(void *unused) + idle_host_task = current; + + for (;;) { ++ if (single_task_running()) ++ idle_host_enter(); + lkl_cpu_put(); + lkl_ops->sem_down(ti->sched_sem); + if (idle_host_task == NULL) { +@@ -162,6 +221,7 @@ static int idle_host_task_loop(void *unused) + return 0; + } + schedule_tail(ti->prev_sched); ++ lkl_idle_host_exit(); + } + } + +diff --git a/arch/lkl/kernel/time.c b/arch/lkl/kernel/time.c +index 17d956fe6..c8f77086e 100644 +--- a/arch/lkl/kernel/time.c ++++ b/arch/lkl/kernel/time.c +@@ -60,7 +60,7 @@ static struct clocksource clocksource = { + .name = "lkl", + .rating = 499, + .read = clock_read, +- .flags = CLOCK_SOURCE_IS_CONTINUOUS, ++ .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_VALID_FOR_HRES, + .mask = CLOCKSOURCE_MASK(64), + }; + +-- +2.53.0 + diff --git a/utils/lkl-patches/0003-crypto-jitter-do-not-loop-forever-on-a-clock-that-do.patch b/utils/lkl-patches/0003-crypto-jitter-do-not-loop-forever-on-a-clock-that-do.patch new file mode 100644 index 00000000..4c19a16a --- /dev/null +++ b/utils/lkl-patches/0003-crypto-jitter-do-not-loop-forever-on-a-clock-that-do.patch @@ -0,0 +1,54 @@ +From d670f680e4370f6530fcc2252a340e82bf6071ba Mon Sep 17 00:00:00 2001 +From: Gabriel Ferreira +Date: Wed, 7 Oct 2026 23:46:57 +0200 +Subject: [PATCH 3/4] crypto: jitter - do not loop forever on a clock that does + not advance + +jent_entropy_collector_alloc() fills the entropy pool before +jent_entropy_init() checks that the clock is fine-grained enough. When +every measurement is stuck, as with a clock that does not advance, +jent_gen_entropy() repeats the measurement until the health tests fail, +but outside of FIPS mode jent_health_failure() never reports a failure, +so the kernel hangs while initializing the jitterentropy RNG. + +Stop when the repetition count test failed. Only its intermittent +failure can be seen here: the count restarts after it, so it never +reaches the permanent failure cutoff. The time tests of +jent_entropy_init() then reject the clock, and the jitterentropy RNG is +not registered, as for any other unsuitable clock. + +This happens with LKL in DCE, where the kernel runs on simulated time, +which does not advance while the kernel initializes. + +TOUPSTREAM +--- + crypto/jitterentropy.c | 12 +++++++++++- + 1 file changed, 11 insertions(+), 1 deletion(-) + +diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c +index 3b390bd6c..9025437f9 100644 +--- a/crypto/jitterentropy.c ++++ b/crypto/jitterentropy.c +@@ -566,8 +566,18 @@ static void jent_gen_entropy(struct rand_data *ec) + + while (!jent_health_failure(ec)) { + /* If a stuck measurement is received, repeat measurement */ +- if (jent_measure_jitter(ec, NULL)) ++ if (jent_measure_jitter(ec, NULL)) { ++ /* ++ * Outside of FIPS mode, the health tests do not end ++ * the loop: give up on a clock that does not advance. ++ * The count restarts after an intermittent failure, ++ * so a permanent failure is never reported. ++ */ ++ if (ec->health_failure & ++ (JENT_RCT_FAILURE | JENT_RCT_FAILURE_PERMANENT)) ++ break; + continue; ++ } + + /* + * We multiply the loop value with ->osr to obtain the +-- +2.53.0 + diff --git a/utils/lkl-patches/0004-lkl-show_stack-stop-at-the-first-frame-outside-of-th.patch b/utils/lkl-patches/0004-lkl-show_stack-stop-at-the-first-frame-outside-of-th.patch new file mode 100644 index 00000000..db2dd2a6 --- /dev/null +++ b/utils/lkl-patches/0004-lkl-show_stack-stop-at-the-first-frame-outside-of-th.patch @@ -0,0 +1,46 @@ +From a3df1d4a90b978e4744a18c9936913a30709e5e9 Mon Sep 17 00:00:00 2001 +From: Gabriel Ferreira +Date: Thu, 8 Oct 2026 08:31:24 +0200 +Subject: [PATCH 4/4] lkl: show_stack: stop at the first frame outside of the + kernel + +With frame pointers, show_stack() follows the frame chain for up to +THREAD_SIZE frames. Past the kernel, the chain continues into the host +code that called it, which may not keep frame pointers: the walk then +dereferences whatever the frame pointer slot holds, and printing a +warning crashes the host (e.g. ns-3 DCE, whose task stacks hold +simulator frames above the kernel). + +Stop at the first return address that is not kernel text, and at frames +that do not go up the stack. + +TOUPSTREAM +--- + arch/lkl/kernel/stacktrace.c | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/arch/lkl/kernel/stacktrace.c b/arch/lkl/kernel/stacktrace.c +index 0690fbc00..fb54c93d5 100644 +--- a/arch/lkl/kernel/stacktrace.c ++++ b/arch/lkl/kernel/stacktrace.c +@@ -31,9 +31,16 @@ void show_stack(struct task_struct *task, unsigned long *sp, + + while (stack && stack - frame <= THREAD_SIZE) { + addr = stack->return_address; ++ /* ++ * The host code that called into the kernel may not keep ++ * frame pointers: stop at the first frame outside of the ++ * kernel, and on frames that do not go up the stack. ++ */ ++ if (!__kernel_text_address(addr)) ++ break; + printk("%s #%02d [<0x%016lx>] %pS\n", + loglvl, cnt++, addr, (void *)addr); +- if (stack == stack->next_frame) ++ if (stack->next_frame <= stack) + break; + stack = stack->next_frame; + } +-- +2.53.0 + From deb2380b6b7d0fc44ee1309be5c62f57f33029f5 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Thu, 8 Oct 2026 13:59:10 +0200 Subject: [PATCH 06/71] LKL: Linux 6.12 network stack for DCE configure --with-lkl= builds DCE with LKL, the Linux kernel as a library, as a node's network stack. Without --enable-kernel-stack (libos, Linux 4.4), LKL provides ns3::LinuxSocketFdFactory, so the existing kernel examples and tests run on Linux 6.12 unchanged (the libos library name liblinux.so selects liblkl.so); with it, libos stays ns3::LinuxSocketFdFactory and LKL is ns3::LklSocketFdFactory. LklKernel (model/lkl/lkl-kernel.*): one kernel per node, its own copy of liblkl.so (a new one for each simulation in a process), with LKL's host operations on DCE: - Threads are DCE tasks; semaphores and mutexes wait lists of tasks; thread-local storage per task, dropped when the task ends (destructors run in a task of their own). - Time is the simulation time; timers are simulator events. - System calls from the main fiber (ns-3 sockets of LinuxSocketImpl) run in a task, driven without advancing time. - Kernel memory is released when the kernel object is destroyed. LklSocketFdFactory (model/lkl/lkl-socket-fd-factory.*): - Boots the kernel at simulation start, mounts procfs. - Every ns-3 NetDevice becomes a virtio-net device sim (found by MAC address), with the device's MTU; IFF_NOARP when the device cannot carry ARP (point-to-point). The netdev poll task does not re-report frames the kernel has no buffers for; at most 1024 wait. - Sockets are LklSocketFd, whose operations are system calls; poll and select through a per-node epoll watcher (persistent, edge-triggered: a kernel wait queue). - Set/Get: sysctls as libos names them (".net.ipv4.tcp_sack"), or any kernel file by absolute path (/proc/net/netstat); ScheduleTask; the Mptcp attribute makes applications' TCP sockets MPTCP sockets. Build (wscript): LKL_STACK and, without libos, LKL_LINUX; LINUX_STACK (any Linux stack provides ns3::LinuxSocketFdFactory) for LinuxStackHelper, the Linux socket factories and the iperf examples; KERNEL_STACK keeps meaning libos; dce-freebsd is built only with libos. Examples: dce-lkl-boot (kernels boot, exact 10 s sleep), dce-lkl-p2p (ip configuration, TCP and UDP between two kernels or over loopback; --stack=linux for libos). The cradle, process manager and quagga test suites look for liblkl.so in LKL-only builds (they skipped otherwise). Co-Authored-By: Claude Opus 5.5 --- example/dce-iperf-emulation.cc | 6 +- example/dce-iperf-heterogeneous-multihop.cc | 10 +- example/dce-iperf.cc | 6 +- example/dce-lkl-boot.cc | 95 +++ example/dce-lkl-p2p.cc | 137 +++ example/examples-to-run.py | 4 + helper/linux-stack-helper.cc | 18 +- model/linux-socket-fd-factory.h | 22 + model/linux/linux-dccp-socket-factory-impl.cc | 4 +- .../linux/linux-dccp6-socket-factory-impl.cc | 4 +- .../linux-ipv4-raw-socket-factory-impl.cc | 4 +- .../linux-ipv6-raw-socket-factory-impl.cc | 4 +- model/linux/linux-sctp-socket-factory-impl.cc | 4 +- .../linux/linux-sctp6-socket-factory-impl.cc | 4 +- model/linux/linux-tcp-socket-factory-impl.cc | 4 +- model/linux/linux-tcp6-socket-factory-impl.cc | 4 +- model/linux/linux-udp-socket-factory-impl.cc | 4 +- model/linux/linux-udp6-socket-factory-impl.cc | 4 +- model/lkl/lkl-kernel.cc | 784 ++++++++++++++++++ model/lkl/lkl-kernel.h | 83 ++ model/lkl/lkl-linux-socket-fd-factory.cc | 18 + model/lkl/lkl-socket-fd-factory.cc | 636 ++++++++++++++ model/lkl/lkl-socket-fd-factory.h | 94 +++ model/lkl/lkl-socket-fd.cc | 287 +++++++ model/lkl/lkl-socket-fd.h | 69 ++ test/dce-cradle-test.cc | 5 + test/dce-manager-test.cc | 5 + utils/ns-3-dce-quagga-ns3.patch | 15 +- wscript | 58 +- 29 files changed, 2346 insertions(+), 46 deletions(-) create mode 100644 example/dce-lkl-boot.cc create mode 100644 example/dce-lkl-p2p.cc create mode 100644 model/lkl/lkl-kernel.cc create mode 100644 model/lkl/lkl-kernel.h create mode 100644 model/lkl/lkl-linux-socket-fd-factory.cc create mode 100644 model/lkl/lkl-socket-fd-factory.cc create mode 100644 model/lkl/lkl-socket-fd-factory.h create mode 100644 model/lkl/lkl-socket-fd.cc create mode 100644 model/lkl/lkl-socket-fd.h diff --git a/example/dce-iperf-emulation.cc b/example/dce-iperf-emulation.cc index ddbcfade..9c53b3f6 100644 --- a/example/dce-iperf-emulation.cc +++ b/example/dce-iperf-emulation.cc @@ -133,7 +133,7 @@ int main (int argc, char *argv[]) } else if (stack == "linux") { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); dceManager.Install (nodes); LinuxStackHelper stack; @@ -146,7 +146,7 @@ int main (int argc, char *argv[]) } else if (stack == "freebsd") { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK dceManager.SetNetworkStack ("ns3::FreeBSDSocketFdFactory", "Library", StringValue ("libfreebsd.so")); dceManager.Install (nodes); FreeBSDStackHelper stack; @@ -162,7 +162,7 @@ int main (int argc, char *argv[]) address.SetBase ("10.0.0.0", "255.255.255.252"); Ipv4InterfaceContainer interfaces = address.Assign (devices); -#ifdef KERNEL_STACK +#ifdef LINUX_STACK if (stack == "linux") { LinuxStackHelper::PopulateRoutingTables (); diff --git a/example/dce-iperf-heterogeneous-multihop.cc b/example/dce-iperf-heterogeneous-multihop.cc index 0504f67b..08c0c6de 100644 --- a/example/dce-iperf-heterogeneous-multihop.cc +++ b/example/dce-iperf-heterogeneous-multihop.cc @@ -162,7 +162,7 @@ main (int argc, char *argv[]) ns3stack.Install (routersL); ns3stack.Install (routersR); -#ifdef KERNEL_STACK +#ifdef LINUX_STACK LinuxStackHelper linuxStack; #endif @@ -175,7 +175,7 @@ main (int argc, char *argv[]) } else if (stack == "linux") { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); dceManager.Install (client); dceManager.Install (server); @@ -259,7 +259,7 @@ main (int argc, char *argv[]) // Calculate and populate routing tables // ---------------------------------------------------------------------- Ipv4GlobalRoutingHelper::PopulateRoutingTables (); -#ifdef KERNEL_STACK +#ifdef LINUX_STACK if (stack == "linux") { LinuxStackHelper::PopulateRoutingTables (); @@ -294,7 +294,7 @@ main (int argc, char *argv[]) apps.Start (Seconds (0.75)); apps.Stop (Seconds (iperfDurationSeconds+3.0)); -#ifdef KERNEL_STACK +#ifdef LINUX_STACK if (stack == "linux") { if(clientTcp != "") @@ -345,7 +345,7 @@ main (int argc, char *argv[]) apps.Start (Seconds (0.5)); apps.Stop (Seconds (iperfDurationSeconds+3.0)); -#ifdef KERNEL_STACK +#ifdef LINUX_STACK if (stack == "linux") { if (serverTcp != "") diff --git a/example/dce-iperf.cc b/example/dce-iperf.cc index 966a7e5d..f527884f 100644 --- a/example/dce-iperf.cc +++ b/example/dce-iperf.cc @@ -63,7 +63,7 @@ int main (int argc, char *argv[]) } else if (stack == "linux") { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); dceManager.Install (nodes); LinuxStackHelper stack; @@ -76,7 +76,7 @@ int main (int argc, char *argv[]) } else if (stack == "freebsd") { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK dceManager.SetNetworkStack ("ns3::FreeBSDSocketFdFactory", "Library", StringValue ("libfreebsd.so")); dceManager.Install (nodes); FreeBSDStackHelper stack; @@ -98,7 +98,7 @@ int main (int argc, char *argv[]) // setup ip routes Ipv4GlobalRoutingHelper::PopulateRoutingTables (); -#ifdef KERNEL_STACK +#ifdef LINUX_STACK if (stack == "linux") { LinuxStackHelper::PopulateRoutingTables (); diff --git a/example/dce-lkl-boot.cc b/example/dce-lkl-boot.cc new file mode 100644 index 00000000..f6a991f3 --- /dev/null +++ b/example/dce-lkl-boot.cc @@ -0,0 +1,95 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Boot an LKL (Linux Kernel Library) kernel on each node and check that + * kernel time follows the simulation clock: a 10 s nanosleep inside the + * kernel must take 10 s of simulated time and almost no wall-clock time. + */ +#include "ns3/core-module.h" +#include "ns3/network-module.h" +#include "ns3/dce-module.h" +#include "ns3/lkl-kernel.h" +#include "ns3/task-manager.h" +#include +#include +#include +#include + +using namespace ns3; + +static int g_failures = 0; +static uint32_t g_completed = 0; + +static void +CheckNode (Ptr node) +{ + Ptr kernel = node->GetObject (); + kernel->Boot (); + + char uts[6][65]; + memset (uts, 0, sizeof (uts)); + long params[6] = { (long)uts, 0, 0, 0, 0, 0 }; + long ret = kernel->Syscall (__lkl__NR_uname, params); + std::cout << "node " << node->GetId () << ": uname=" << ret << " " + << uts[0] << " " << uts[2] << std::endl; + + struct + { + long long tv_sec; + long long tv_nsec; + } duration = { 10, 0 }; + Time before = Simulator::Now (); + auto wallBefore = std::chrono::steady_clock::now (); + long sleepParams[6] = { (long)&duration, 0, 0, 0, 0, 0 }; + ret = kernel->Syscall (__lkl__NR_nanosleep, sleepParams); + Time slept = Simulator::Now () - before; + double wall = std::chrono::duration (std::chrono::steady_clock::now () - wallBefore).count (); + std::cout << "node " << node->GetId () << ": nanosleep(10s)=" << ret + << " simulated " << slept.GetNanoSeconds () << " ns, wall " << wall << " s" << std::endl; + // Linux adds the task's timer slack (50 us by default) to the sleep. + if (ret != 0 || slept < Seconds (10) || slept > Seconds (10) + MilliSeconds (1)) + { + g_failures++; + } + g_completed++; +} + +static void +CheckTask (void *context) +{ + CheckNode ((Node *)context); + TaskManager::Current ()->Exit (); +} + +static void +StartCheck (Ptr node) +{ + node->GetObject ()->Start (&CheckTask, PeekPointer (node), 1 << 18); +} + +int +main (int argc, char *argv[]) +{ + uint32_t nNodes = 2; + CommandLine cmd; + cmd.AddValue ("nodes", "Number of nodes", nNodes); + cmd.Parse (argc, argv); + + NodeContainer nodes; + nodes.Create (nNodes); + DceManagerHelper dceManager; + dceManager.Install (nodes); + for (uint32_t i = 0; i < nNodes; i++) + { + nodes.Get (i)->AggregateObject (CreateObject ()); + // Start the nodes at different times so that their kernels interleave. + Simulator::ScheduleWithContext (i, Seconds (1.0 + 0.5 * i), &StartCheck, nodes.Get (i)); + } + + Simulator::Stop (Seconds (30)); + Simulator::Run (); + Simulator::Destroy (); + + bool ok = g_failures == 0 && g_completed == nNodes; + std::cout << (ok ? "OK" : "FAILED") << " (" << g_completed << "/" << nNodes << " nodes completed)" << std::endl; + return ok ? 0 : 1; +} diff --git a/example/dce-lkl-p2p.cc b/example/dce-lkl-p2p.cc new file mode 100644 index 00000000..3975e42b --- /dev/null +++ b/example/dce-lkl-p2p.cc @@ -0,0 +1,137 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Two nodes running Linux (LKL) kernels on a point-to-point link: the + * kernels' sim0 devices are configured with ip, then a TCP and a UDP + * client on node 0 send to servers on node 1. Every program must exit + * with status 0. + * + * With --loopback, a single node runs the TCP client and server over its + * loopback device instead (like dce-linux-simple). --stack=linux runs the + * same programs on the libos (Linux 4.4) stack, for comparison. + */ +#include "ns3/core-module.h" +#include "ns3/network-module.h" +#include "ns3/point-to-point-module.h" +#include "ns3/dce-module.h" +#include +#include +#include + +using namespace ns3; + +static uint32_t g_started = 0; +static uint32_t g_succeeded = 0; + +// A program succeeds if it exits with 0 and, for the servers, if it +// received everything the client sent. +static void +Finished (std::string name, uint32_t node, uint16_t pid, int status) +{ + bool ok = status == 0; + if (ok && name.find ("server") != std::string::npos) + { + std::ostringstream path; + path << "files-" << node << "/var/log/" << pid << "/stdout"; + std::ifstream file (path.str ()); + std::stringstream out; + out << file.rdbuf (); + ok = out.str ().find ("did read all buffers") != std::string::npos; + } + std::cout << Simulator::Now ().GetSeconds () << "s " << name << " exited with " << status + << (ok ? "" : " FAILED") << std::endl; + if (ok) + { + g_succeeded++; + } +} + +static void +Run (Ptr node, Time at, std::string binary, std::string args, bool check = true) +{ + DceApplicationHelper dce; + dce.SetBinary (binary); + dce.SetStackSize (1 << 16); + dce.ResetArguments (); + if (!args.empty ()) + { + dce.ParseArguments (args.c_str ()); + } + if (check) + { + dce.SetFinishedCallback (MakeBoundCallback (&Finished, binary + " " + args, node->GetId ())); + g_started++; + } + ApplicationContainer apps = dce.Install (node); + apps.Start (at); +} + +int +main (int argc, char *argv[]) +{ + bool loopback = false; + bool pcap = false; + std::string stack = "lkl"; + CommandLine cmd; + cmd.AddValue ("stack", "Kernel: lkl (Linux 6.12) or linux (libos, Linux 4.4)", stack); + cmd.AddValue ("loopback", "Use one node and its loopback device", loopback); + cmd.AddValue ("pcap", "Write pcap traces of the link", pcap); + cmd.Parse (argc, argv); + + NodeContainer nodes; + nodes.Create (loopback ? 1 : 2); + if (!loopback) + { + PointToPointHelper p2p; + p2p.SetDeviceAttribute ("DataRate", StringValue ("100Mbps")); + p2p.SetChannelAttribute ("Delay", StringValue ("1ms")); + p2p.Install (nodes); + if (pcap) + { + p2p.EnablePcapAll ("dce-lkl-p2p"); + } + } + DceManagerHelper dceManager; + if (stack == "linux") + { + dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); + } + else + { + dceManager.SetNetworkStack ("ns3::LklSocketFdFactory"); + } + dceManager.Install (nodes); + + for (uint32_t i = 0; i < nodes.GetN (); i++) + { + std::ostringstream addr; + addr << "-f inet addr add 10.0.0." << i + 1 << "/24 dev sim0"; + Run (nodes.Get (i), Seconds (0.1), "ip", "link set lo up"); + if (!loopback) + { + Run (nodes.Get (i), Seconds (0.2), "ip", addr.str ()); + Run (nodes.Get (i), Seconds (0.3), "ip", "link set sim0 up"); + } + Run (nodes.Get (i), Seconds (0.5), "ip", "addr show", false); + Run (nodes.Get (i), Seconds (0.5), "ip", "route show", false); + } + + Ptr server = nodes.Get (nodes.GetN () - 1); + std::string serverAddr = loopback ? "127.0.0.1" : "10.0.0.2"; + Run (server, Seconds (1.0), "tcp-server", ""); + Run (nodes.Get (0), Seconds (1.5), "tcp-client", serverAddr); + if (!loopback) + { + Run (server, Seconds (1.0), "udp-server", ""); + Run (nodes.Get (0), Seconds (1.5), "udp-client", serverAddr); + } + + // udp-client sends one datagram per second, 1000 times. + Simulator::Stop (Seconds (1100)); + Simulator::Run (); + Simulator::Destroy (); + + bool ok = g_succeeded == g_started; + std::cout << (ok ? "OK" : "FAILED") << " (" << g_succeeded << "/" << g_started + << " programs succeeded)" << std::endl; + return ok ? 0 : 1; +} diff --git a/example/examples-to-run.py b/example/examples-to-run.py index 78f27339..545accb1 100644 --- a/example/examples-to-run.py +++ b/example/examples-to-run.py @@ -89,6 +89,10 @@ # ("dce-wifi-ccnx", "True", "True"), ("dce-sctp-simple", "True", "True"), ("dce-freebsd", "True", "True"), + # Linux kernels from LKL (configure --with-lkl) + ("dce-lkl-boot", "True", "True"), + ("dce-lkl-p2p --loopback", "True", "True"), + ("dce-lkl-p2p", "True", "True"), ] # A list of Python examples to run in order to ensure that they remain diff --git a/helper/linux-stack-helper.cc b/helper/linux-stack-helper.cc index 18c09192..5e29916c 100644 --- a/helper/linux-stack-helper.cc +++ b/helper/linux-stack-helper.cc @@ -63,7 +63,7 @@ LinuxStackHelper::SetRoutingHelper (const Ipv4RoutingHelper &routing) void LinuxStackHelper::Install (Ptr node) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ipv4Linux::InstallNode (node); #if 0 // Set routing @@ -77,7 +77,7 @@ LinuxStackHelper::Install (Ptr node) void LinuxStackHelper::Install (std::string nodeName) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr node = Names::Find (nodeName); Install (node); #endif @@ -85,7 +85,7 @@ LinuxStackHelper::Install (std::string nodeName) void LinuxStackHelper::Install (NodeContainer c) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK for (NodeContainer::Iterator i = c.Begin (); i != c.End (); ++i) { Install (*i); @@ -95,7 +95,7 @@ LinuxStackHelper::Install (NodeContainer c) void LinuxStackHelper::InstallAll (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Install (NodeContainer::GetGlobal ()); #endif } @@ -103,7 +103,7 @@ LinuxStackHelper::InstallAll (void) void LinuxStackHelper::PopulateRoutingTables () { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK NodeContainer c = NodeContainer::GetGlobal (); for (NodeContainer::Iterator i = c.Begin (); i != c.End (); ++i) { @@ -120,7 +120,7 @@ LinuxStackHelper::PopulateRoutingTables () void LinuxStackHelper::RunIp (Ptr node, Time at, std::string str) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK DceApplicationHelper process; ApplicationContainer apps; process.SetBinary ("ip"); @@ -136,7 +136,7 @@ void LinuxStackHelper::SysctlGetCallback (Ptr node, std::string path, void (*callback)(std::string, std::string)) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr sock = node->GetObject (); if (!sock) { @@ -156,7 +156,7 @@ void LinuxStackHelper::SysctlGet (Ptr node, Time at, std::string path, void (*callback)(std::string, std::string)) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr sock = node->GetObject (); if (!sock) { @@ -175,7 +175,7 @@ LinuxStackHelper::SysctlGet (Ptr node, Time at, std::string path, void LinuxStackHelper::SysctlSet (NodeContainer c, std::string path, std::string value) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK for (NodeContainer::Iterator i = c.Begin (); i != c.End (); ++i) { Ptr node = *i; diff --git a/model/linux-socket-fd-factory.h b/model/linux-socket-fd-factory.h index 2ff94cc1..a383c585 100644 --- a/model/linux-socket-fd-factory.h +++ b/model/linux-socket-fd-factory.h @@ -1,6 +1,26 @@ #ifndef LINUX_SOCKET_FD_FACTORY_H #define LINUX_SOCKET_FD_FACTORY_H +#ifdef LKL_LINUX + +#include "lkl-socket-fd-factory.h" + +namespace ns3 { + +/** + * Without libos (configure --with-lkl only), the Linux network stack is + * LKL; see LklSocketFdFactory. + */ +class LinuxSocketFdFactory : public LklSocketFdFactory +{ +public: + static TypeId GetTypeId (void); +}; + +} // namespace ns3 + +#else + #include "kernel-socket-fd-factory.h" #include @@ -36,4 +56,6 @@ class LinuxSocketFdFactory : public KernelSocketFdFactory } // namespace ns3 +#endif /* LKL_LINUX */ + #endif /* LINUX_SOCKET_FD_FACTORY_H */ diff --git a/model/linux/linux-dccp-socket-factory-impl.cc b/model/linux/linux-dccp-socket-factory-impl.cc index f01d3c6a..aa71a240 100644 --- a/model/linux/linux-dccp-socket-factory-impl.cc +++ b/model/linux/linux-dccp-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxDccpSocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxDccpSocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxDccpSocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxDccpSocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-dccp6-socket-factory-impl.cc b/model/linux/linux-dccp6-socket-factory-impl.cc index c60057f3..ffcd66c8 100644 --- a/model/linux/linux-dccp6-socket-factory-impl.cc +++ b/model/linux/linux-dccp6-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxDccp6SocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxDccp6SocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxDccp6SocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxDccp6SocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-ipv4-raw-socket-factory-impl.cc b/model/linux/linux-ipv4-raw-socket-factory-impl.cc index 7725e02e..7b0c6865 100644 --- a/model/linux/linux-ipv4-raw-socket-factory-impl.cc +++ b/model/linux/linux-ipv4-raw-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxIpv4RawSocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxIpv4RawSocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxIpv4RawSocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxIpv4RawSocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-ipv6-raw-socket-factory-impl.cc b/model/linux/linux-ipv6-raw-socket-factory-impl.cc index 4322b9ca..13031bb2 100644 --- a/model/linux/linux-ipv6-raw-socket-factory-impl.cc +++ b/model/linux/linux-ipv6-raw-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxIpv6RawSocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxIpv6RawSocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxIpv6RawSocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxIpv6RawSocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-sctp-socket-factory-impl.cc b/model/linux/linux-sctp-socket-factory-impl.cc index 8025034a..a30d8237 100644 --- a/model/linux/linux-sctp-socket-factory-impl.cc +++ b/model/linux/linux-sctp-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxSctpSocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxSctpSocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxSctpSocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxSctpSocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-sctp6-socket-factory-impl.cc b/model/linux/linux-sctp6-socket-factory-impl.cc index 39b37445..ed6ba377 100644 --- a/model/linux/linux-sctp6-socket-factory-impl.cc +++ b/model/linux/linux-sctp6-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxSctp6SocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxSctp6SocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxSctp6SocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxSctp6SocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-tcp-socket-factory-impl.cc b/model/linux/linux-tcp-socket-factory-impl.cc index 5b0db5ab..1e9c83c6 100644 --- a/model/linux/linux-tcp-socket-factory-impl.cc +++ b/model/linux/linux-tcp-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxTcpSocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxTcpSocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxTcpSocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxTcpSocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-tcp6-socket-factory-impl.cc b/model/linux/linux-tcp6-socket-factory-impl.cc index 40875c7b..9606a7f4 100644 --- a/model/linux/linux-tcp6-socket-factory-impl.cc +++ b/model/linux/linux-tcp6-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxTcp6SocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxTcp6SocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxTcp6SocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxTcp6SocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-udp-socket-factory-impl.cc b/model/linux/linux-udp-socket-factory-impl.cc index e9d576be..7d9dcfb8 100644 --- a/model/linux/linux-udp-socket-factory-impl.cc +++ b/model/linux/linux-udp-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxUdpSocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxUdpSocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxUdpSocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxUdpSocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/linux/linux-udp6-socket-factory-impl.cc b/model/linux/linux-udp6-socket-factory-impl.cc index e4e7648b..03819060 100644 --- a/model/linux/linux-udp6-socket-factory-impl.cc +++ b/model/linux/linux-udp6-socket-factory-impl.cc @@ -32,7 +32,7 @@ namespace ns3 { Ptr LinuxUdp6SocketFactoryImpl::CreateSocket (void) { -#ifdef KERNEL_STACK +#ifdef LINUX_STACK Ptr socket = CreateObject (); Ptr node = this->GetObject (); socket->SetNode (node); @@ -42,7 +42,7 @@ LinuxUdp6SocketFactoryImpl::CreateSocket (void) socket->CreateSocket (); return socket; #else - NS_LOG_WARN ("LinuxUdp6SocketFactoryImpl::CreateSocket(): require KERNEL_STACK"); + NS_LOG_WARN ("LinuxUdp6SocketFactoryImpl::CreateSocket(): require LINUX_STACK"); return nullptr; #endif } diff --git a/model/lkl/lkl-kernel.cc b/model/lkl/lkl-kernel.cc new file mode 100644 index 00000000..52417509 --- /dev/null +++ b/model/lkl/lkl-kernel.cc @@ -0,0 +1,784 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +#include "lkl-kernel.h" +#include "task-manager.h" +#include "utils.h" +#include "exec-utils.h" +#include "ns3/log.h" +#include "ns3/node.h" +#include "ns3/simulator.h" +#include "ns3/string.h" +#include "ns3/event-impl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +NS_LOG_COMPONENT_DEFINE ("DceLklKernel"); + +namespace ns3 { + +NS_OBJECT_ENSURE_REGISTERED (LklKernel); + +namespace { + +// Stack size of the tasks running kernel threads. +const uint32_t LKL_TASK_STACK = 1 << 18; + +LklKernel * +Kernel (void) +{ + LklKernel *kernel = LklKernel::Current (); + NS_ASSERT_MSG (kernel, "LKL host operation called outside of a node with an LKL kernel"); + return kernel; +} + +TaskManager * +Manager (void) +{ + return TaskManager::Current (); +} + +/* Threads ---------------------------------------------------------------- */ + +struct LklThread +{ + Task *task; + void (*fn)(void *); + void *arg; + bool done; + bool detached; + std::list joiners; +}; + +// Tasks of all nodes are distinct, so a single map is enough. +std::map g_threads; + +LklThread * +ThreadOf (Task *task) +{ + std::map::iterator i = g_threads.find (task); + if (i != g_threads.end ()) + { + return i->second; + } + // A task that was not created by the kernel (an application or a timer + // task) calls into the kernel: give it an identity. + LklThread *thread = new LklThread (); + thread->task = task; + thread->fn = 0; + thread->arg = 0; + thread->done = false; + thread->detached = true; + g_threads[task] = thread; + return thread; +} + +void +ThreadFinish (void) +{ + Task *task = Manager ()->RunningTask (); + LklThread *thread = ThreadOf (task); + thread->done = true; + for (std::list::iterator i = thread->joiners.begin (); i != thread->joiners.end (); ++i) + { + Manager ()->Wakeup (*i); + } + thread->joiners.clear (); + g_threads.erase (task); + if (thread->detached) + { + delete thread; + } + else + { + thread->task = 0; + } + Manager ()->Exit (); +} + +void +ThreadTrampoline (void *context) +{ + LklThread *thread = (LklThread *)context; + thread->fn (thread->arg); + ThreadFinish (); +} + +lkl_thread_t +ThreadCreate (void (*fn)(void *), void *arg) +{ + LklKernel *kernel = Kernel (); + LklThread *thread = new LklThread (); + thread->fn = fn; + thread->arg = arg; + thread->done = false; + thread->detached = false; + thread->task = kernel->GetTaskManager ()->Start (&ThreadTrampoline, thread, LKL_TASK_STACK); + g_threads[thread->task] = thread; + return (lkl_thread_t)thread; +} + +void +ThreadDetach (void) +{ + ThreadOf (Manager ()->RunningTask ())->detached = true; +} + +void +ThreadExit (void) +{ + ThreadFinish (); +} + +int +ThreadJoin (lkl_thread_t tid) +{ + LklThread *thread = (LklThread *)tid; + while (!thread->done) + { + thread->joiners.push_back (Manager ()->RunningTask ()); + Manager ()->Sleep (); + } + delete thread; + return 0; +} + +lkl_thread_t +ThreadSelf (void) +{ + return (lkl_thread_t)ThreadOf (Manager ()->RunningTask ()); +} + +int +ThreadEqual (lkl_thread_t a, lkl_thread_t b) +{ + return a == b; +} + +/* Semaphores and mutexes -------------------------------------------------- */ + +struct LklWaitList +{ + std::list waiters; + void Wait (void) + { + waiters.push_back (Manager ()->RunningTask ()); + Manager ()->Sleep (); + } + void WakeOne (void) + { + if (!waiters.empty ()) + { + Task *task = waiters.front (); + waiters.pop_front (); + Manager ()->Wakeup (task); + } + } +}; + +struct LklSem +{ + int count; + LklWaitList wait; +}; + +struct lkl_sem * +SemAlloc (int count) +{ + LklSem *sem = new LklSem (); + sem->count = count; + return (struct lkl_sem *)sem; +} + +void +SemFree (struct lkl_sem *s) +{ + delete (LklSem *)s; +} + +void +SemUp (struct lkl_sem *s) +{ + LklSem *sem = (LklSem *)s; + sem->count++; + sem->wait.WakeOne (); +} + +void +SemDown (struct lkl_sem *s) +{ + LklSem *sem = (LklSem *)s; + while (sem->count <= 0) + { + sem->wait.Wait (); + } + sem->count--; +} + +struct LklMutex +{ + bool recursive; + Task *owner; + int depth; + LklWaitList wait; +}; + +struct lkl_mutex * +MutexAlloc (int recursive) +{ + LklMutex *mutex = new LklMutex (); + mutex->recursive = recursive; + mutex->owner = 0; + mutex->depth = 0; + return (struct lkl_mutex *)mutex; +} + +void +MutexFree (struct lkl_mutex *m) +{ + delete (LklMutex *)m; +} + +void +MutexLock (struct lkl_mutex *m) +{ + LklMutex *mutex = (LklMutex *)m; + Task *self = Manager ()->RunningTask (); + if (mutex->recursive && mutex->owner == self) + { + mutex->depth++; + return; + } + while (mutex->owner != 0) + { + mutex->wait.Wait (); + } + mutex->owner = self; + mutex->depth = 1; +} + +void +MutexUnlock (struct lkl_mutex *m) +{ + LklMutex *mutex = (LklMutex *)m; + if (--mutex->depth > 0) + { + return; + } + mutex->owner = 0; + mutex->wait.WakeOne (); +} + +/* Thread-local storage ---------------------------------------------------- */ + +struct LklTlsKey +{ + void (*destructor)(void *); + std::map values; +}; + +struct lkl_tls_key * +TlsAlloc (void (*destructor)(void *)) +{ + LklTlsKey *key = new LklTlsKey (); + key->destructor = destructor; + Kernel ()->GetTlsKeys ().insert (key); + return (struct lkl_tls_key *)key; +} + +void +TlsFree (struct lkl_tls_key *k) +{ + Kernel ()->GetTlsKeys ().erase (k); + delete (LklTlsKey *)k; +} + +int +TlsSet (struct lkl_tls_key *k, void *data) +{ + ((LklTlsKey *)k)->values[Manager ()->RunningTask ()] = data; + return 0; +} + +void * +TlsGet (struct lkl_tls_key *k) +{ + LklTlsKey *key = (LklTlsKey *)k; + std::map::iterator i = key->values.find (Manager ()->RunningTask ()); + return i == key->values.end () ? 0 : i->second; +} + +/* Time and timers --------------------------------------------------------- */ + +unsigned long long +Time_ (void) +{ + return Simulator::Now ().GetNanoSeconds (); +} + +struct LklTimer +{ + void (*fn)(void); + LklKernel *kernel; + EventId event; +}; + +void +TimerTaskTrampoline (void *context) +{ + LklTimer *timer = (LklTimer *)context; + timer->fn (); + Manager ()->Exit (); +} + +// Runs the timer callback in its own task: it raises the timer interrupt, +// and kernel code reached from there may block on the CPU lock. +void +TimerExpire (LklTimer *timer) +{ + timer->kernel->GetTaskManager ()->Start (&TimerTaskTrampoline, timer, LKL_TASK_STACK); +} + +void * +TimerAlloc (void (*fn)(void)) +{ + LklTimer *timer = new LklTimer (); + timer->fn = fn; + timer->kernel = Kernel (); + return timer; +} + +int +TimerSetOneshot (void *t, unsigned long delta) +{ + LklTimer *timer = (LklTimer *)t; + timer->event.Cancel (); + // Called from the node's own tasks, so the event keeps the node context. + timer->event = Simulator::Schedule (NanoSeconds (delta), &TimerExpire, timer); + return 0; +} + +void +TimerFree (void *t) +{ + LklTimer *timer = (LklTimer *)t; + timer->event.Cancel (); + delete timer; +} + +/* Memory and misc --------------------------------------------------------- */ + +void +Print (const char *str, int len) +{ + fwrite (str, 1, len, stdout); +} + +void +Panic (void) +{ + NS_FATAL_ERROR ("LKL kernel panic on node " << Kernel ()->GetNodeId ()); +} + +void * +MemAlloc (unsigned long size) +{ + return malloc (size); +} + +void +MemFree (void *p) +{ + free (p); +} + +void * +PageAlloc (unsigned long size) +{ + // Anonymous mappings are committed lazily: idle kernel memory costs nothing. + void *p = mmap (0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + { + return 0; + } + Kernel ()->GetPages ()[p] = size; + return p; +} + +void +PageFree (void *addr, unsigned long size) +{ + Kernel ()->GetPages ().erase (addr); + munmap (addr, size); +} + +void +JmpBufSet (struct lkl_jmp_buf *jmpb, void (*f)(void)) +{ + static_assert (sizeof (jmpb->buf) >= sizeof (jmp_buf), "lkl_jmp_buf too small"); + if (!setjmp (*((jmp_buf *)jmpb->buf))) + { + f (); + } +} + +void +JmpBufLongjmp (struct lkl_jmp_buf *jmpb, int val) +{ + longjmp (*((jmp_buf *)jmpb->buf), val); +} + +void * +Memcpy (void *dest, const void *src, unsigned long count) +{ + return memcpy (dest, src, count); +} + +void * +Memset (void *s, int c, unsigned long count) +{ + return memset (s, c, count); +} + +void * +Memmove (void *dest, const void *src, unsigned long count) +{ + return memmove (dest, src, count); +} + +/* Files ------------------------------------------------------------------ */ + +void +CreateDirectory (const std::string &path) +{ + for (std::string::size_type i = path.find ('/'); ; i = path.find ('/', i + 1)) + { + mkdir (path.substr (0, i).c_str (), S_IRWXU); + if (i == std::string::npos) + { + break; + } + } +} + +// Copy unless an identical-looking copy (same size, not older) is there. +void +CopyFile (const std::string &from, const std::string &to) +{ + struct stat src, dst; + NS_ASSERT_MSG (stat (from.c_str (), &src) == 0, "Cannot stat " << from); + if (stat (to.c_str (), &dst) == 0 && dst.st_size == src.st_size && dst.st_mtime >= src.st_mtime) + { + return; + } + std::ifstream in (from, std::ios::binary); + std::ofstream out (to, std::ios::binary | std::ios::trunc); + out << in.rdbuf (); + NS_ASSERT_MSG (in && out, "Cannot copy " << from << " to " << to); +} + +} // namespace + +TypeId +LklKernel::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::LklKernel") + .SetParent () + .AddConstructor () + .AddAttribute ("Library", "The LKL shared library to load.", + StringValue ("liblkl.so"), + MakeStringAccessor (&LklKernel::m_library), + MakeStringChecker ()) + .AddAttribute ("CommandLine", "The kernel command line.", + StringValue ("mem=64M loglevel=4"), + MakeStringAccessor (&LklKernel::m_cmdline), + MakeStringChecker ()) + ; + return tid; +} + +LklKernel::LklKernel () + : m_manager (0), + m_handle (0), + m_init (0), + m_start (0), + m_syscall (0), + m_booted (false) +{ +} + +LklKernel::~LklKernel () +{ + // The simulation is over: nothing runs the kernel any more. + for (std::map::iterator i = m_pages.begin (); i != m_pages.end (); ++i) + { + munmap (i->first, i->second); + } + if (m_handle) + { + dlclose (m_handle); + } +} + +std::map & +LklKernel::GetPages (void) +{ + return m_pages; +} + +void +LklKernel::DoDispose (void) +{ + m_manager = 0; + Object::DoDispose (); +} + +void +LklKernel::NotifyNewAggregate (void) +{ + Ptr taskManager = GetObject (); + if (taskManager) + { + m_manager = PeekPointer (taskManager); + } + Object::NotifyNewAggregate (); +} + +LklKernel * +LklKernel::Current (void) +{ + TaskManager *manager = TaskManager::Current (); + if (!manager) + { + return 0; + } + return PeekPointer (manager->GetObject ()); +} + +TaskManager * +LklKernel::GetTaskManager (void) const +{ + return m_manager; +} + +uint32_t +LklKernel::GetNodeId (void) const +{ + return GetObject ()->GetId (); +} + +void +LklKernel::InstallHostOps (struct lkl_host_operations *ops) +{ + // The library's own host code (virtio, network devices) calls this table + // directly, so replace every POSIX operation with a DCE one. + ops->print = &Print; + ops->panic = &Panic; + ops->sem_alloc = &SemAlloc; + ops->sem_free = &SemFree; + ops->sem_up = &SemUp; + ops->sem_down = &SemDown; + ops->mutex_alloc = &MutexAlloc; + ops->mutex_free = &MutexFree; + ops->mutex_lock = &MutexLock; + ops->mutex_unlock = &MutexUnlock; + ops->thread_create = &ThreadCreate; + ops->thread_detach = &ThreadDetach; + ops->thread_exit = &ThreadExit; + ops->thread_join = &ThreadJoin; + ops->thread_self = &ThreadSelf; + ops->thread_equal = &ThreadEqual; + ops->thread_stack = 0; + ops->tls_alloc = &TlsAlloc; + ops->tls_free = &TlsFree; + ops->tls_set = &TlsSet; + ops->tls_get = &TlsGet; + ops->mem_alloc = &MemAlloc; + ops->mem_free = &MemFree; + ops->page_alloc = &PageAlloc; + ops->page_free = &PageFree; + ops->time = &Time_; + ops->timer_alloc = &TimerAlloc; + ops->timer_set_oneshot = &TimerSetOneshot; + ops->timer_free = &TimerFree; + ops->ioremap = (void * (*)(long, int)) dlsym (m_handle, "lkl_ioremap"); + ops->iomem_access = (int (*)(const volatile void *, void *, int, int)) + dlsym (m_handle, "lkl_iomem_access"); + ops->jmp_buf_set = &JmpBufSet; + ops->jmp_buf_longjmp = &JmpBufLongjmp; + ops->memcpy = &Memcpy; + ops->memset = &Memset; + ops->memmove = &Memmove; + ops->mmap = 0; + ops->munmap = 0; + ops->shmem_init = 0; + ops->shmem_mmap = 0; + ops->pci_ops = 0; +} + +void +LklKernel::Load (void) +{ + std::string path = SearchExecFile ("DCE_PATH", m_library, 0); + NS_ASSERT_MSG (path.length () > 0, "LKL library '" << m_library + << "' not found; check the DCE_PATH environment variable."); + // Each node needs its own instance of the kernel, with its own global + // data. The dynamic linker only loads a file once, so give each kernel + // its own copy of the library; a node of a later simulation in the same + // process (as in test suites) gets a new one. + static std::map loads; + uint32_t generation = loads[GetNodeId ()]++; + std::ostringstream dir; + dir << "elf-cache/lkl/" << GetNodeId (); + if (generation > 0) + { + dir << "-" << generation; + } + std::string copy = dir.str () + "/" + m_library; + CreateDirectory (dir.str ()); + CopyFile (path, copy); + m_handle = dlopen (copy.c_str (), RTLD_NOW | RTLD_LOCAL); + NS_ASSERT_MSG (m_handle, "Cannot load " << copy << ": " << dlerror ()); + m_init = (int (*)(struct lkl_host_operations *)) dlsym (m_handle, "lkl_init"); + m_start = (int (*)(const char *, ...)) dlsym (m_handle, "lkl_start_kernel"); + m_syscall = (long (*)(long, long *)) dlsym (m_handle, "lkl_syscall"); + NS_ASSERT_MSG (m_init && m_start && m_syscall, "Missing LKL entry points in " << path); +} + +std::set & +LklKernel::GetTlsKeys (void) +{ + return m_tlsKeys; +} + +namespace { +struct TlsDestructors +{ + std::list > calls; +}; +} // namespace + +void +LklKernel::TlsDestructorsTrampoline (void *context) +{ + TlsDestructors *destructors = (TlsDestructors *)context; + for (std::list >::iterator i = destructors->calls.begin (); + i != destructors->calls.end (); ++i) + { + i->first (i->second); + } + delete destructors; + TaskManager::Current ()->Exit (); +} + +// Like pthread thread-local storage: drop the task's values and, when the +// task exited, call their destructors. They run in a task of their own: +// the exiting task may belong to a process DCE already deleted. +void +LklKernel::TaskEnd (Task *task, bool running, void *context) +{ + LklKernel *kernel = (LklKernel *)context; + TlsDestructors *destructors = new TlsDestructors (); + for (std::set::iterator k = kernel->m_tlsKeys.begin (); k != kernel->m_tlsKeys.end (); ++k) + { + LklTlsKey *key = (LklTlsKey *)*k; + std::map::iterator i = key->values.find (task); + if (i == key->values.end ()) + { + continue; + } + if (running && i->second && key->destructor) + { + destructors->calls.push_back (std::make_pair (key->destructor, i->second)); + } + key->values.erase (i); + } + if (destructors->calls.empty ()) + { + delete destructors; + return; + } + kernel->m_manager->Start (&LklKernel::TlsDestructorsTrampoline, destructors, LKL_TASK_STACK); +} + +void +LklKernel::Boot (void) +{ + NS_LOG_FUNCTION (this); + NS_ASSERT (!m_booted); + NS_ASSERT_MSG (TaskManager::Current () == m_manager && m_manager->CurrentTask (), + "LklKernel::Boot must run in a task of its node"); + Load (); + struct lkl_host_operations *ops = + (struct lkl_host_operations *) dlsym (m_handle, "lkl_host_ops"); + NS_ASSERT_MSG (ops, "Missing lkl_host_ops in " << m_library); + InstallHostOps (ops); + m_manager->SetTaskEndNotifier (&LklKernel::TaskEnd, this); + int ret = m_init (ops); + NS_ASSERT_MSG (ret == 0, "lkl_init failed: " << ret); + ret = m_start (m_cmdline.c_str ()); + NS_ASSERT_MSG (ret == 0, "lkl_start_kernel failed: " << ret); + m_booted = true; +} + +bool +LklKernel::IsBooted (void) const +{ + return m_booted; +} + +void * +LklKernel::Lookup (const char *symbol) const +{ + NS_ASSERT (m_handle); + return dlsym (m_handle, symbol); +} + +namespace { +struct MainSyscall +{ + LklKernel *kernel; + long no; + long *params; + long ret; + bool done; +}; +} // namespace + +void +LklKernel::MainSyscallTrampoline (void *context) +{ + MainSyscall *call = (MainSyscall *)context; + call->ret = call->kernel->m_syscall (call->no, call->params); + call->done = true; + TaskManager::Current ()->Exit (); +} + +long +LklKernel::Syscall (long no, long *params) +{ + NS_ASSERT (m_booted); + if (m_manager->RunningTask ()) + { + return m_syscall (no, params); + } + // From the main fiber, e.g. ns-3 sockets (LinuxSocketImpl): the call may + // wait for kernel threads, so it runs in a task, now. + NS_ASSERT (TaskManager::Current () == m_manager); + MainSyscall call = { this, no, params, 0, false }; + m_manager->Start (&LklKernel::MainSyscallTrampoline, &call, LKL_TASK_STACK); + bool done = m_manager->RunNow (&call.done); + NS_LOG_DEBUG ("system call " << no << " from the main fiber returns " << call.ret); + NS_ASSERT_MSG (done, "system call " << no << " from the main fiber needs the simulation time to advance"); + return call.ret; +} + +} // namespace ns3 diff --git a/model/lkl/lkl-kernel.h b/model/lkl/lkl-kernel.h new file mode 100644 index 00000000..d6820c9d --- /dev/null +++ b/model/lkl/lkl-kernel.h @@ -0,0 +1,83 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +#ifndef LKL_KERNEL_H +#define LKL_KERNEL_H + +#include "ns3/object.h" +#include "ns3/event-id.h" +#include +#include +#include + +struct lkl_host_operations; + +namespace ns3 { + +class TaskManager; +class Task; + +/** + * A Linux kernel (LKL, the Linux Kernel Library) for one node. + * + * Each node loads its own copy of liblkl.so (a separate file, so that the + * dynamic linker gives it separate global data) and runs it on DCE tasks: + * the LKL host operations map threads to tasks, semaphores and mutexes to + * task wait lists, the clock to the simulation clock and the one-shot timer + * to simulator events. The kernel therefore only runs when a + * task or an event drives it, and an idle kernel lets the simulator jump to + * the next event. + */ +class LklKernel : public Object +{ +public: + static TypeId GetTypeId (void); + LklKernel (); + virtual ~LklKernel (); + + /** + * Boot the kernel. Must be called from a DCE task, as booting blocks + * until the kernel init thread has started. + */ + void Boot (void); + bool IsBooted (void) const; + + /** Issue a kernel system call (LKL syscall number, parameters). */ + long Syscall (long no, long *params); + + /** Look up a symbol of this node's copy of the kernel library. */ + void * Lookup (const char *symbol) const; + + /** The kernel of the node running the current task. */ + static LklKernel *Current (void); + + // Used by the host operations. + TaskManager *GetTaskManager (void) const; + uint32_t GetNodeId (void) const; + /** The thread-local storage keys the kernel allocated. */ + std::set &GetTlsKeys (void); + /** The memory the kernel allocated with page_alloc. */ + std::map &GetPages (void); + +private: + virtual void DoDispose (void); + virtual void NotifyNewAggregate (void); + void Load (void); + void InstallHostOps (struct lkl_host_operations *ops); + + std::string m_library; + std::string m_cmdline; + TaskManager *m_manager; + void *m_handle; + int (*m_init)(struct lkl_host_operations *ops); + int (*m_start)(const char *cmdline, ...); + static void MainSyscallTrampoline (void *context); + static void TaskEnd (Task *task, bool running, void *context); + static void TlsDestructorsTrampoline (void *context); + long (*m_syscall)(long no, long *params); + bool m_booted; + std::set m_tlsKeys; + std::map m_pages; +}; + +} // namespace ns3 + +#endif /* LKL_KERNEL_H */ diff --git a/model/lkl/lkl-linux-socket-fd-factory.cc b/model/lkl/lkl-linux-socket-fd-factory.cc new file mode 100644 index 00000000..f31e9a3c --- /dev/null +++ b/model/lkl/lkl-linux-socket-fd-factory.cc @@ -0,0 +1,18 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +#include "linux-socket-fd-factory.h" + +namespace ns3 { + +NS_OBJECT_ENSURE_REGISTERED (LinuxSocketFdFactory); + +TypeId +LinuxSocketFdFactory::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::LinuxSocketFdFactory") + .SetParent () + .AddConstructor () + ; + return tid; +} + +} // namespace ns3 diff --git a/model/lkl/lkl-socket-fd-factory.cc b/model/lkl/lkl-socket-fd-factory.cc new file mode 100644 index 00000000..62c8d58d --- /dev/null +++ b/model/lkl/lkl-socket-fd-factory.cc @@ -0,0 +1,636 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +#include "lkl-socket-fd-factory.h" +#include "lkl-socket-fd.h" +#include "lkl-kernel.h" +#include "task-manager.h" +#include "ns3/log.h" +#include "ns3/node.h" +#include "ns3/packet.h" +#include "ns3/simulator.h" +#include "ns3/string.h" +#include "ns3/boolean.h" +#include "ns3/mac48-address.h" +#include "ns3/loopback-net-device.h" +#include "ns3/make-event.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +NS_LOG_COMPONENT_DEFINE ("DceLklSocketFdFactory"); + +/* + * LKL's network device interface (tools/lkl/include/lkl_host.h and lkl.h). + * Those headers are not valid C++, so the few definitions needed here are + * repeated; they must match the LKL version built by utils/build_lkl.sh. + */ +extern "C" { +struct lkl_netdev; +struct lkl_dev_net_ops +{ + int (*tx)(struct lkl_netdev *nd, struct iovec *iov, int cnt); + int (*rx)(struct lkl_netdev *nd, struct iovec *iov, int cnt); + int (*poll)(struct lkl_netdev *nd); + void (*poll_hup)(struct lkl_netdev *nd); + void (*free)(struct lkl_netdev *nd); +}; +struct lkl_netdev +{ + struct lkl_dev_net_ops *ops; + int id; + uint8_t has_vnet_hdr : 1; + uint8_t mac[6]; +}; +struct lkl_netdev_args +{ + void *mac; + unsigned int offload; +}; +} +#define LKL_DEV_NET_POLL_RX 1 +#define LKL_DEV_NET_POLL_HUP 4 + +namespace ns3 { + +NS_OBJECT_ENSURE_REGISTERED (LklSocketFdFactory); + +/** + * A kernel network device backed by an ns-3 NetDevice. LKL drives it + * through the network device host operations below. + */ +struct LklNetDevice +{ + struct lkl_netdev nd; // must stay first: LKL passes &nd to the operations + LklSocketFdFactory *factory; + Ptr device; + std::deque > rx; // received Ethernet frames + Task *poller; + bool hup; + // The kernel had no receive buffers for the queued frames: it takes them + // when it adds buffers, so do not report them again. + bool stalled; +}; + +namespace { + +const uint32_t LKL_TASK_STACK = 1 << 18; +const uint32_t ETH_HEADER = 14; +// Received frames waiting for the kernel, like the ring of a network card. +const uint32_t RX_BACKLOG = 1024; + +LklNetDevice * +DeviceOf (struct lkl_netdev *nd) +{ + return (LklNetDevice *)nd; +} + +void +SendMain (Ptr device, Ptr p, Mac48Address dest, uint16_t protocol) +{ + device->Send (p, dest, protocol); +} + +// The kernel sends a frame: hand it to the ns-3 device. +int +NetTx (struct lkl_netdev *nd, struct iovec *iov, int cnt) +{ + LklNetDevice *dev = DeviceOf (nd); + std::vector frame; + for (int i = 0; i < cnt; i++) + { + const uint8_t *base = (const uint8_t *)iov[i].iov_base; + frame.insert (frame.end (), base, base + iov[i].iov_len); + } + if (frame.size () < ETH_HEADER) + { + return frame.size (); + } + Mac48Address dest; + dest.CopyFrom (&frame[0]); + uint16_t protocol = (frame[12] << 8) | frame[13]; + NS_LOG_DEBUG ("tx " << dev->device << " " << dest << " protocol " << protocol + << " " << frame.size () << " bytes"); + Ptr p = Create (&frame[ETH_HEADER], frame.size () - ETH_HEADER); + TaskManager::Current ()->ExecOnMain (MakeEvent (&SendMain, dev->device, p, dest, protocol)); + return frame.size (); +} + +// The kernel reads the next received frame, if any. +int +NetRx (struct lkl_netdev *nd, struct iovec *iov, int cnt) +{ + LklNetDevice *dev = DeviceOf (nd); + if (dev->rx.empty ()) + { + return -1; + } + std::vector &frame = dev->rx.front (); + size_t done = 0; + for (int i = 0; i < cnt && done < frame.size (); i++) + { + size_t n = std::min (iov[i].iov_len, frame.size () - done); + memcpy (iov[i].iov_base, &frame[done], n); + done += n; + } + dev->rx.pop_front (); + dev->stalled = false; + return done; +} + +// Blocks the device's poll task until the kernel can take a received frame. +// Sending never blocks, so transmission readiness is never reported. +int +NetPoll (struct lkl_netdev *nd) +{ + LklNetDevice *dev = DeviceOf (nd); + TaskManager *manager = TaskManager::Current (); + while ((dev->rx.empty () || dev->stalled) && !dev->hup) + { + dev->poller = manager->RunningTask (); + manager->Sleep (); + } + dev->poller = 0; + if (dev->hup) + { + return LKL_DEV_NET_POLL_HUP; + } + // Until the kernel takes a frame or another one arrives. + dev->stalled = true; + return LKL_DEV_NET_POLL_RX; +} + +void +NetPollHup (struct lkl_netdev *nd) +{ + LklNetDevice *dev = DeviceOf (nd); + dev->hup = true; + if (dev->poller) + { + TaskManager::Current ()->Wakeup (dev->poller); + } +} + +void +NetFree (struct lkl_netdev *nd) +{ + // The factory owns the device. +} + +struct lkl_dev_net_ops g_netOps = { + &NetTx, &NetRx, &NetPoll, &NetPollHup, &NetFree +}; + +} // namespace + +TypeId +LklSocketFdFactory::GetTypeId (void) +{ + static TypeId tid = TypeId ("ns3::LklSocketFdFactory") + .SetParent () + .AddConstructor () + .AddAttribute ("Library", "The LKL shared library to load.", + StringValue ("liblkl.so"), + MakeStringAccessor (&LklSocketFdFactory::m_library), + MakeStringChecker ()) + .AddAttribute ("CommandLine", "The kernel command line.", + StringValue ("mem=64M loglevel=4"), + MakeStringAccessor (&LklSocketFdFactory::m_cmdline), + MakeStringChecker ()) + .AddAttribute ("Mptcp", + "Create MPTCP sockets for the TCP sockets of applications, " + "like mptcpize: Linux uses MPTCP only when asked for.", + BooleanValue (false), + MakeBooleanAccessor (&LklSocketFdFactory::m_mptcp), + MakeBooleanChecker ()) + ; + return tid; +} + +LklSocketFdFactory::LklSocketFdFactory () + : m_mptcp (false), + m_booted (false), + m_nextDevice (0), + m_epollFd (-1) +{ +} + +LklSocketFdFactory::~LklSocketFdFactory () +{ + for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) + { + delete *i; + } +} + +void +LklSocketFdFactory::DoDispose (void) +{ + // The devices refer to their node, which refers to this factory. + for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) + { + (*i)->device = 0; + } + m_kernel = 0; + SocketFdFactory::DoDispose (); +} + +Ptr +LklSocketFdFactory::GetKernel (void) const +{ + return m_kernel; +} + +void +LklSocketFdFactory::NotifyNewAggregate (void) +{ + Ptr node = GetObject (); + Ptr manager = GetObject (); + if (node && manager && !m_kernel) + { + m_kernel = CreateObject (); + // Scripts written for libos name its library; LKL replaces it. + std::string library = m_library == "liblinux.so" ? "liblkl.so" : m_library; + m_kernel->SetAttribute ("Library", StringValue (library)); + m_kernel->SetAttribute ("CommandLine", StringValue (m_cmdline)); + node->AggregateObject (m_kernel); + Simulator::ScheduleWithContext (node->GetId (), Seconds (0), + &TaskManager::Start, PeekPointer (manager), + &LklSocketFdFactory::BootTrampoline, (void *)this, + LKL_TASK_STACK); + } + SocketFdFactory::NotifyNewAggregate (); +} + +void +LklSocketFdFactory::BootTrampoline (void *context) +{ + ((LklSocketFdFactory *)context)->Boot (); + TaskManager::Current ()->Exit (); +} + +void +LklSocketFdFactory::Boot (void) +{ + NS_LOG_FUNCTION (this); + m_kernel->Boot (); + // procfs, for sysctls. + long ret = Call (__lkl__NR_mkdirat, AT_FDCWD, (long)"/proc", 0555); + NS_ASSERT_MSG (ret == 0, "mkdir /proc failed: " << ret); + ret = Call (__lkl__NR_mount, (long)"proc", (long)"/proc", (long)"proc", 0, 0); + NS_ASSERT_MSG (ret == 0, "mount /proc failed: " << ret); + m_epollFd = Call (__lkl__NR_epoll_create1, 0); + NS_ASSERT_MSG (m_epollFd >= 0, "epoll_create1 failed: " << m_epollFd); + m_kernel->GetTaskManager ()->Start (&LklSocketFdFactory::WatcherTrampoline, this, LKL_TASK_STACK); + // Called right away for the devices the node already has. + GetObject ()->RegisterDeviceAdditionListener (MakeCallback (&LklSocketFdFactory::NotifyAddDevice, this)); + m_booted = true; + for (std::list::iterator i = m_bootWaiters.begin (); i != m_bootWaiters.end (); ++i) + { + m_kernel->GetTaskManager ()->Wakeup (*i); + } + m_bootWaiters.clear (); + for (std::list >::iterator i = m_earlySysctls.begin (); + i != m_earlySysctls.end (); ++i) + { + SetTask (i->first, i->second); + } + m_earlySysctls.clear (); +} + +namespace { +std::string +SysctlFile (std::string path) +{ + if (!path.empty () && path[0] == '/') + { + return path; // any kernel file, e.g. /proc/net/netstat + } + std::string file = "/proc/sys"; + if (path.empty () || path[0] != '.') + { + file += '/'; + } + for (std::string::iterator i = path.begin (); i != path.end (); ++i) + { + file += *i == '.' ? '/' : *i; + } + return file; +} +} // namespace + +void +LklSocketFdFactory::Set (std::string path, std::string value) +{ + NS_LOG_FUNCTION (this << path << value); + if (!m_booted) + { + m_earlySysctls.push_back (std::make_pair (path, value)); + return; + } + ScheduleTask (MakeEvent (&LklSocketFdFactory::SetTask, this, path, value)); +} + +void +LklSocketFdFactory::SetTask (std::string path, std::string value) +{ + std::string file = SysctlFile (path); + long fd = Call (__lkl__NR_openat, AT_FDCWD, (long)file.c_str (), O_WRONLY); + if (fd < 0) + { + NS_LOG_WARN ("sysctl " << path << ": cannot open " << file << ": " << fd); + return; + } + long ret = Call (__lkl__NR_write, fd, (long)value.c_str (), value.size ()); + if (ret < 0) + { + NS_LOG_WARN ("sysctl " << path << " = " << value << ": " << ret); + } + Call (__lkl__NR_close, fd); +} + +std::string +LklSocketFdFactory::Get (std::string path) +{ + NS_LOG_FUNCTION (this << path); + WaitBooted (); + std::string file = SysctlFile (path); + long fd = Call (__lkl__NR_openat, AT_FDCWD, (long)file.c_str (), O_RDONLY); + if (fd < 0) + { + NS_LOG_WARN ("sysctl " << path << ": cannot open " << file << ": " << fd); + return ""; + } + std::string value; + char buffer[4096]; + long n; + while ((n = Call (__lkl__NR_read, fd, (long)buffer, sizeof (buffer))) > 0) + { + value.append (buffer, n); + } + Call (__lkl__NR_close, fd); + return value; +} + +void +LklSocketFdFactory::ScheduleTask (EventImpl *event) +{ + m_kernel->GetTaskManager ()->Start (&LklSocketFdFactory::ScheduleTaskTrampoline, + event, LKL_TASK_STACK); +} + +void +LklSocketFdFactory::ScheduleTaskTrampoline (void *context) +{ + EventImpl *event = (EventImpl *)context; + event->Invoke (); + event->Unref (); + TaskManager::Current ()->Exit (); +} + +void +LklSocketFdFactory::WaitBooted (void) +{ + TaskManager *manager = TaskManager::Current (); + while (!m_booted) + { + m_bootWaiters.push_back (manager->RunningTask ()); + manager->Sleep (); + } +} + +long +LklSocketFdFactory::Call (long no, long a0, long a1, long a2, long a3, long a4, long a5) +{ + long params[6] = { a0, a1, a2, a3, a4, a5 }; + return m_kernel->Syscall (no, params); +} + +namespace { +struct AddDeviceContext +{ + LklSocketFdFactory *factory; + Ptr device; +}; +} // namespace + +void +LklSocketFdFactory::NotifyAddDevice (Ptr device) +{ + TaskManager *manager = m_kernel->GetTaskManager (); + if (TaskManager::Current () == manager && manager->RunningTask ()) + { + AddDevice (device); + return; + } + // Adding a kernel device needs a task. + AddDeviceContext *context = new AddDeviceContext (); + context->factory = this; + context->device = device; + manager->Start (&LklSocketFdFactory::AddDeviceTrampoline, context, LKL_TASK_STACK); +} + +void +LklSocketFdFactory::AddDeviceTrampoline (void *c) +{ + AddDeviceContext *context = (AddDeviceContext *)c; + context->factory->AddDevice (context->device); + delete context; + TaskManager::Current ()->Exit (); +} + +void +LklSocketFdFactory::AddDevice (Ptr device) +{ + NS_LOG_FUNCTION (this << device); + if (DynamicCast (device)) + { + return; // the kernel has its own loopback device + } + LklNetDevice *dev = new LklNetDevice (); + memset (&dev->nd, 0, sizeof (dev->nd)); + dev->nd.ops = &g_netOps; + dev->factory = this; + dev->device = device; + dev->poller = 0; + dev->hup = false; + dev->stalled = false; + m_devices.push_back (dev); + + uint8_t mac[6]; + Address address = device->GetAddress (); + Mac48Address mac48 = Mac48Address::IsMatchingType (address) + ? Mac48Address::ConvertFrom (address) : Mac48Address::Allocate (); + mac48.CopyTo (mac); + struct lkl_netdev_args args; + memset (&args, 0, sizeof (args)); + args.mac = mac; + + typedef int (*NetdevAdd)(struct lkl_netdev *, struct lkl_netdev_args *); + NetdevAdd netdevAdd = (NetdevAdd) m_kernel->Lookup ("lkl_netdev_add"); + int id = netdevAdd (&dev->nd, &args); + NS_ASSERT_MSG (id >= 0, "lkl_netdev_add failed: " << id); + + // Find the new device, named eth, by its address (lkl_netdev_get_ifindex + // looks for eth, which the devices renamed below break), and name it + // sim like the libos stack, with the MTU of the ns-3 device. + long s = Call (__lkl__NR_socket, AF_INET, SOCK_DGRAM, 0); + NS_ASSERT (s >= 0); + struct ifreq ifr; + bool found = false; + for (int ifindex = 1; ifindex < 4096 && !found; ifindex++) + { + memset (&ifr, 0, sizeof (ifr)); + ifr.ifr_ifindex = ifindex; + if (Call (__lkl__NR_ioctl, s, SIOCGIFNAME, (long)&ifr) != 0 + || strncmp (ifr.ifr_name, "eth", 3) != 0 + || Call (__lkl__NR_ioctl, s, SIOCGIFHWADDR, (long)&ifr) != 0) + { + continue; + } + found = memcmp (ifr.ifr_hwaddr.sa_data, mac, 6) == 0; + } + NS_ASSERT_MSG (found, "no kernel device for " << mac48); + long ret; + snprintf (ifr.ifr_newname, IFNAMSIZ, "sim%u", m_nextDevice++); + ret = Call (__lkl__NR_ioctl, s, SIOCSIFNAME, (long)&ifr); + NS_ASSERT_MSG (ret == 0, "SIOCSIFNAME failed: " << ret); + memcpy (ifr.ifr_name, ifr.ifr_newname, IFNAMSIZ); + ifr.ifr_mtu = device->GetMtu (); + ret = Call (__lkl__NR_ioctl, s, SIOCSIFMTU, (long)&ifr); + NS_ASSERT_MSG (ret == 0, "SIOCSIFMTU failed: " << ret); + if (!device->NeedsArp ()) + { + // e.g. point-to-point links, which cannot carry ARP: the kernel then + // sends to neighbors without resolving them. + ret = Call (__lkl__NR_ioctl, s, SIOCGIFFLAGS, (long)&ifr); + NS_ASSERT_MSG (ret == 0, "SIOCGIFFLAGS failed: " << ret); + ifr.ifr_flags |= IFF_NOARP; + ret = Call (__lkl__NR_ioctl, s, SIOCSIFFLAGS, (long)&ifr); + NS_ASSERT_MSG (ret == 0, "SIOCSIFFLAGS failed: " << ret); + } + Call (__lkl__NR_close, s); + + GetObject ()->RegisterProtocolHandler (MakeCallback (&LklSocketFdFactory::RxFromDevice, this), + 0, device, true); +} + +void +LklSocketFdFactory::RxFromDevice (Ptr device, Ptr p, uint16_t protocol, + const Address &from, const Address &to, NetDevice::PacketType type) +{ + LklNetDevice *dev = 0; + for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) + { + if ((*i)->device == device) + { + dev = *i; + break; + } + } + if (!dev || dev->rx.size () >= RX_BACKLOG) + { + return; + } + // Rebuild the Ethernet frame the kernel expects. + std::vector frame (ETH_HEADER + p->GetSize ()); + Mac48Address dest = Mac48Address::IsMatchingType (to) + ? Mac48Address::ConvertFrom (to) : Mac48Address::ConvertFrom (device->GetAddress ()); + Mac48Address source = Mac48Address::IsMatchingType (from) + ? Mac48Address::ConvertFrom (from) : Mac48Address ("00:00:00:00:00:00"); + dest.CopyTo (&frame[0]); + source.CopyTo (&frame[6]); + frame[12] = protocol >> 8; + frame[13] = protocol & 0xff; + p->CopyData (&frame[ETH_HEADER], p->GetSize ()); + NS_LOG_DEBUG ("rx " << device << " protocol " << protocol << " " << frame.size () << " bytes"); + dev->rx.push_back (frame); + dev->stalled = false; + if (dev->poller) + { + m_kernel->GetTaskManager ()->Wakeup (dev->poller); + } +} + +UnixFd * +LklSocketFdFactory::CreateSocket (int domain, int type, int protocol) +{ + NS_LOG_FUNCTION (this << domain << type << protocol); + WaitBooted (); + if (m_mptcp && (domain == AF_INET || domain == AF_INET6) + && (type & 0xf) == SOCK_STREAM && (protocol == 0 || protocol == IPPROTO_TCP)) + { + protocol = 262; // IPPROTO_MPTCP + } + long fd = Call (__lkl__NR_socket, domain, type, protocol); + if (fd < 0) + { + return 0; + } + return new LklSocketFd (this, fd); +} + +void +LklSocketFdFactory::Register (LklSocketFd *socket) +{ + m_sockets[socket->GetKernelFd ()] = socket; +} + +void +LklSocketFdFactory::Unregister (LklSocketFd *socket) +{ + m_sockets.erase (socket->GetKernelFd ()); + m_watched.erase (socket->GetKernelFd ()); +} + +void +LklSocketFdFactory::Watch (LklSocketFd *socket) +{ + // Like a kernel wait queue: once watched, every change of the socket's + // state wakes its poll waiters, which check what they wait for. + int fd = socket->GetKernelFd (); + if (m_watched.count (fd)) + { + return; + } + struct epoll_event event; + memset (&event, 0, sizeof (event)); + event.events = EPOLLIN | EPOLLOUT | EPOLLPRI | EPOLLRDHUP | EPOLLET; + event.data.fd = fd; + long ret = Call (__lkl__NR_epoll_ctl, m_epollFd, EPOLL_CTL_ADD, fd, (long)&event); + NS_ASSERT_MSG (ret == 0, "epoll_ctl failed: " << ret); + m_watched.insert (fd); +} + +void +LklSocketFdFactory::WatcherTrampoline (void *context) +{ + ((LklSocketFdFactory *)context)->Watcher (); +} + +void +LklSocketFdFactory::Watcher (void) +{ + struct epoll_event events[32]; + while (true) + { + long n = Call (__lkl__NR_epoll_pwait, m_epollFd, (long)events, 32, -1, 0, 8); + for (long i = 0; i < n; i++) + { + std::map::iterator s = m_sockets.find (events[i].data.fd); + if (s != m_sockets.end ()) + { + s->second->NotifyEvents (events[i].events & 0xffff); + } + } + } +} + +} // namespace ns3 diff --git a/model/lkl/lkl-socket-fd-factory.h b/model/lkl/lkl-socket-fd-factory.h new file mode 100644 index 00000000..be977a3c --- /dev/null +++ b/model/lkl/lkl-socket-fd-factory.h @@ -0,0 +1,94 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +#ifndef LKL_SOCKET_FD_FACTORY_H +#define LKL_SOCKET_FD_FACTORY_H + +#include "socket-fd-factory.h" +#include "ns3/net-device.h" +#include +#include +#include +#include +#include + +namespace ns3 { + +class EventImpl; +class LklKernel; +class LklSocketFd; +class Task; +struct LklNetDevice; + +/** + * Network stack of a node backed by a Linux kernel (LKL): select it with + * DceManagerHelper::SetNetworkStack ("ns3::LklSocketFdFactory"). + * + * The kernel boots when the simulation starts. Every ns-3 NetDevice of the + * node becomes a kernel network device named sim, in device order; + * sockets of DCE applications are kernel sockets. + */ +class LklSocketFdFactory : public SocketFdFactory +{ +public: + static TypeId GetTypeId (void); + LklSocketFdFactory (); + virtual ~LklSocketFdFactory (); + + virtual UnixFd * CreateSocket (int domain, int type, int protocol); + + Ptr GetKernel (void) const; + + /** + * Write a sysctl, like libos: path ".net.ipv4.tcp_sack" is + * /proc/sys/net/ipv4/tcp_sack. Sysctls set before the kernel booted are + * written right after boot. + */ + void Set (std::string path, std::string value); + /** + * Read a sysctl, or any kernel file given by its absolute path (e.g. + * /proc/net/netstat); must be called from a task of this node. + */ + std::string Get (std::string path); + /** Run the event in a new task of this node. */ + void ScheduleTask (EventImpl *event); + + // Used by LklSocketFd. + void Register (LklSocketFd *socket); + void Unregister (LklSocketFd *socket); + /** From now on, wake the socket's poll waiters when its state changes. */ + void Watch (LklSocketFd *socket); + +private: + virtual void DoDispose (void); + virtual void NotifyNewAggregate (void); + static void BootTrampoline (void *context); + static void ScheduleTaskTrampoline (void *context); + void SetTask (std::string path, std::string value); + void Boot (void); + void WaitBooted (void); + void NotifyAddDevice (Ptr device); + void AddDevice (Ptr device); + static void AddDeviceTrampoline (void *context); + void RxFromDevice (Ptr device, Ptr p, uint16_t protocol, + const Address &from, const Address &to, NetDevice::PacketType type); + static void WatcherTrampoline (void *context); + void Watcher (void); + long Call (long no, long a0 = 0, long a1 = 0, long a2 = 0, + long a3 = 0, long a4 = 0, long a5 = 0); + + std::string m_library; + std::string m_cmdline; + bool m_mptcp; + Ptr m_kernel; + bool m_booted; + std::list m_bootWaiters; + std::list > m_earlySysctls; + std::vector m_devices; + uint32_t m_nextDevice; + int m_epollFd; + std::map m_sockets; + std::set m_watched; +}; + +} // namespace ns3 + +#endif /* LKL_SOCKET_FD_FACTORY_H */ diff --git a/model/lkl/lkl-socket-fd.cc b/model/lkl/lkl-socket-fd.cc new file mode 100644 index 00000000..40161aba --- /dev/null +++ b/model/lkl/lkl-socket-fd.cc @@ -0,0 +1,287 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +#include "lkl-socket-fd.h" +#include "lkl-socket-fd-factory.h" +#include "lkl-kernel.h" +#include "process.h" +#include "utils.h" +#include "file-usage.h" +#include "wait-queue.h" +#include "ns3/log.h" +#include +#include +#include +#include +#include +#include +#include + +NS_LOG_COMPONENT_DEFINE ("DceLklSocketFd"); + +namespace ns3 { + +LklSocketFd::LklSocketFd (Ptr factory, int fd) + : m_factory (factory), + m_fd (fd) +{ + NS_LOG_FUNCTION (this << fd); + m_factory->Register (this); +} + +LklSocketFd::~LklSocketFd () +{ +} + +int +LklSocketFd::GetKernelFd (void) const +{ + return m_fd; +} + +void +LklSocketFd::NotifyEvents (short events) +{ + WakeWaiters (&events); +} + +long +LklSocketFd::Call (long no, long a0, long a1, long a2, long a3, long a4, long a5) const +{ + long params[6] = { a0, a1, a2, a3, a4, a5 }; + long ret = m_factory->GetKernel ()->Syscall (no, params); + if (ret < 0) + { + Current ()->err = -ret; + return -1; + } + return ret; +} + +int +LklSocketFd::Close (void) +{ + NS_LOG_FUNCTION (this); + m_factory->Unregister (this); + return Call (__lkl__NR_close, m_fd); +} + +ssize_t +LklSocketFd::Write (const void *buf, size_t count) +{ + return Call (__lkl__NR_write, m_fd, (long)buf, count); +} + +ssize_t +LklSocketFd::Read (void *buf, size_t count) +{ + return Call (__lkl__NR_read, m_fd, (long)buf, count); +} + +ssize_t +LklSocketFd::Recvmsg (struct msghdr *msg, int flags) +{ + return Call (__lkl__NR_recvmsg, m_fd, (long)msg, flags); +} + +ssize_t +LklSocketFd::Sendmsg (const struct msghdr *msg, int flags) +{ + return Call (__lkl__NR_sendmsg, m_fd, (long)msg, flags); +} + +bool +LklSocketFd::Isatty (void) const +{ + return false; +} + +int +LklSocketFd::Setsockopt (int level, int optname, const void *optval, socklen_t optlen) +{ + return Call (__lkl__NR_setsockopt, m_fd, level, optname, (long)optval, optlen); +} + +int +LklSocketFd::Getsockopt (int level, int optname, void *optval, socklen_t *optlen) +{ + return Call (__lkl__NR_getsockopt, m_fd, level, optname, (long)optval, (long)optlen); +} + +int +LklSocketFd::Getsockname (struct sockaddr *name, socklen_t *namelen) +{ + return Call (__lkl__NR_getsockname, m_fd, (long)name, (long)namelen); +} + +int +LklSocketFd::Getpeername (struct sockaddr *name, socklen_t *namelen) +{ + return Call (__lkl__NR_getpeername, m_fd, (long)name, (long)namelen); +} + +int +LklSocketFd::Ioctl (unsigned long request, char *argp) +{ + return Call (__lkl__NR_ioctl, m_fd, request, (long)argp); +} + +int +LklSocketFd::Bind (const struct sockaddr *my_addr, socklen_t addrlen) +{ + return Call (__lkl__NR_bind, m_fd, (long)my_addr, addrlen); +} + +int +LklSocketFd::Connect (const struct sockaddr *my_addr, socklen_t addrlen) +{ + return Call (__lkl__NR_connect, m_fd, (long)my_addr, addrlen); +} + +int +LklSocketFd::Listen (int backlog) +{ + return Call (__lkl__NR_listen, m_fd, backlog); +} + +int +LklSocketFd::Shutdown (int how) +{ + return Call (__lkl__NR_shutdown, m_fd, how); +} + +int +LklSocketFd::Accept (struct sockaddr *my_addr, socklen_t *addrlen) +{ + NS_LOG_FUNCTION (this); + long kernelFd = Call (__lkl__NR_accept4, m_fd, (long)my_addr, (long)addrlen, 0); + if (kernelFd < 0) + { + return -1; + } + Thread *current = Current (); + int fd = UtilsAllocateFd (); + if (fd == -1) + { + Call (__lkl__NR_close, kernelFd); + current->err = EMFILE; + return -1; + } + UnixFd *unixFd = new LklSocketFd (m_factory, kernelFd); + unixFd->IncFdCount (); + current->process->openFiles[fd] = new FileUsage (fd, unixFd); + return fd; +} + +void * +LklSocketFd::Mmap (void *start, size_t length, int prot, int flags, off64_t offset) +{ + Current ()->err = ENODEV; + return MAP_FAILED; +} + +off64_t +LklSocketFd::Lseek (off64_t offset, int whence) +{ + Current ()->err = ESPIPE; + return -1; +} + +int +LklSocketFd::Fxstat (int ver, struct ::stat *buf) +{ + buf->st_mode = S_IFSOCK; + buf->st_dev = -1; + buf->st_blksize = 0; + return 0; +} + +int +LklSocketFd::Fxstat64 (int ver, struct ::stat64 *buf) +{ + buf->st_mode = S_IFSOCK; + buf->st_dev = -1; + buf->st_blksize = 0; + return 0; +} + +int +LklSocketFd::Fcntl (int cmd, unsigned long arg) +{ + switch (cmd) + { + case F_GETFL: + case F_SETFL: + // The kernel keeps O_NONBLOCK and applies it to every operation. + return Call (__lkl__NR_fcntl, m_fd, cmd, arg); + case F_GETFD: + return m_fdFlags; + case F_SETFD: + m_fdFlags = arg; + return 0; + default: + return UnixFd::Fcntl (cmd, arg); + } +} + +int +LklSocketFd::Settime (int flags, const struct itimerspec *new_value, struct itimerspec *old_value) +{ + Current ()->err = EINVAL; + return -1; +} + +int +LklSocketFd::Gettime (struct itimerspec *cur_value) const +{ + Current ()->err = EINVAL; + return -1; +} + +int +LklSocketFd::Ftruncate (off_t length) +{ + Current ()->err = EINVAL; + return -1; +} + +bool +LklSocketFd::HangupReceived (void) const +{ + return false; +} + +int +LklSocketFd::Poll (PollTable* ptable) +{ + short wanted = ptable ? ptable->GetEventMask () : (POLLIN | POLLOUT | POLLPRI | POLLRDHUP); + struct pollfd pfd; + pfd.fd = m_fd; + pfd.events = wanted; + pfd.revents = 0; + struct + { + long long tv_sec; + long long tv_nsec; + } zero = { 0, 0 }; + long params[6] = { (long)&pfd, 1, (long)&zero, 0, 8, 0 }; + long ret = m_factory->GetKernel ()->Syscall (__lkl__NR_ppoll, params); + if (ret < 0) + { + return POLLERR; + } + if (ptable) + { + // Get woken when the kernel reports one of these events. + ptable->PollWait (this); + m_factory->Watch (this); + } + return pfd.revents; +} + +int +LklSocketFd::Fsync (void) +{ + Current ()->err = EINVAL; + return -1; +} + +} // namespace ns3 diff --git a/model/lkl/lkl-socket-fd.h b/model/lkl/lkl-socket-fd.h new file mode 100644 index 00000000..d66b7dec --- /dev/null +++ b/model/lkl/lkl-socket-fd.h @@ -0,0 +1,69 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +#ifndef LKL_SOCKET_FD_H +#define LKL_SOCKET_FD_H + +#include "unix-fd.h" +#include "ns3/ptr.h" + +namespace ns3 { + +class LklSocketFdFactory; + +/** + * A socket of a node's LKL kernel: every operation is the corresponding + * kernel system call on the kernel's file descriptor. + */ +class LklSocketFd : public UnixFd +{ +public: + LklSocketFd (Ptr factory, int fd); + virtual ~LklSocketFd (); + + int GetKernelFd (void) const; + /** Wake the tasks waiting in poll or select for these events. */ + void NotifyEvents (short events); + + virtual int Close (void); + virtual ssize_t Write (const void *buf, size_t count); + virtual ssize_t Read (void *buf, size_t count); + virtual ssize_t Recvmsg (struct msghdr *msg, int flags); + virtual ssize_t Sendmsg (const struct msghdr *msg, int flags); + virtual bool Isatty (void) const; + virtual int Setsockopt (int level, int optname, + const void *optval, socklen_t optlen); + virtual int Getsockopt (int level, int optname, + void *optval, socklen_t *optlen); + virtual int Getsockname (struct sockaddr *name, socklen_t *namelen); + virtual int Getpeername (struct sockaddr *name, socklen_t *namelen); + virtual int Ioctl (unsigned long request, char *argp); + virtual int Bind (const struct sockaddr *my_addr, socklen_t addrlen); + virtual int Connect (const struct sockaddr *my_addr, socklen_t addrlen); + virtual int Listen (int backlog); + virtual int Shutdown (int how); + virtual int Accept (struct sockaddr *my_addr, socklen_t *addrlen); + virtual void * Mmap (void *start, size_t length, int prot, int flags, off64_t offset); + virtual off64_t Lseek (off64_t offset, int whence); + virtual int Fxstat (int ver, struct ::stat *buf); + virtual int Fxstat64 (int ver, struct ::stat64 *buf); + virtual int Fcntl (int cmd, unsigned long arg); + virtual int Settime (int flags, + const struct itimerspec *new_value, + struct itimerspec *old_value); + virtual int Gettime (struct itimerspec *cur_value) const; + virtual int Ftruncate (off_t length); + virtual bool HangupReceived (void) const; + virtual int Poll (PollTable* ptable); + virtual int Fsync (void); + +private: + // Kernel system call; on error, sets the current thread's errno. + long Call (long no, long a0 = 0, long a1 = 0, long a2 = 0, + long a3 = 0, long a4 = 0, long a5 = 0) const; + + Ptr m_factory; + int m_fd; +}; + +} // namespace ns3 + +#endif /* LKL_SOCKET_FD_H */ diff --git a/test/dce-cradle-test.cc b/test/dce-cradle-test.cc index 63a6ad77..4da99803 100644 --- a/test/dce-cradle-test.cc +++ b/test/dce-cradle-test.cc @@ -160,7 +160,12 @@ DceCradleTestSuite::DceCradleTestSuite () }; // for the moment: not supported mptcp for freebsd +#ifdef LKL_LINUX + // ns3::LinuxSocketFdFactory is LKL + std::string filePath = SearchExecFile ("DCE_PATH", "liblkl.so", 0); +#else std::string filePath = SearchExecFile ("DCE_PATH", "liblinux.so", 0); +#endif for (unsigned int i = 0; i < sizeof(tests)/sizeof(testPair); i++) { if (filePath.length () <= 0) diff --git a/test/dce-manager-test.cc b/test/dce-manager-test.cc index 9cc4d001..605447e5 100644 --- a/test/dce-manager-test.cc +++ b/test/dce-manager-test.cc @@ -264,7 +264,12 @@ DceManagerTestSuite::DceManagerTestSuite () // linux stack TypeId tid; bool kern_linux = TypeId::LookupByNameFailSafe ("ns3::LinuxSocketFdFactory", &tid); +#ifdef LKL_LINUX + // ns3::LinuxSocketFdFactory is LKL + std::string filePath = SearchExecFile ("DCE_PATH", "liblkl.so", 0); +#else std::string filePath = SearchExecFile ("DCE_PATH", "liblinux.so", 0); +#endif if (kern_linux && (filePath.length () > 0)) { for (unsigned int i = 0; i < sizeof(tests) / sizeof(testPair); i++) diff --git a/utils/ns-3-dce-quagga-ns3.patch b/utils/ns-3-dce-quagga-ns3.patch index c050bced..429b863e 100644 --- a/utils/ns-3-dce-quagga-ns3.patch +++ b/utils/ns-3-dce-quagga-ns3.patch @@ -468,7 +468,20 @@ diff -ruN a/test/dce-quagga-test.cc b/test/dce-quagga-test.cc }; ::system ("/bin/rm -rf files-*/usr/local/etc/*.pid"); -@@ -452,7 +461,7 @@ +@@ -442,7 +451,12 @@ + TypeId tid; + bool kern = TypeId::LookupByNameFailSafe ("ns3::LinuxSocketFdFactory", &tid); + // for the moment: not supported quagga for freebsd ++#ifdef LKL_LINUX ++ // ns3::LinuxSocketFdFactory is LKL ++ std::string filePath = SearchExecFile ("DCE_PATH", "liblkl.so", 0); ++#else + std::string filePath = SearchExecFile ("DCE_PATH", "liblinux.so", 0); ++#endif + for (unsigned int i = 0; i < sizeof(tests) / sizeof(testPair); i++) + { + if (filePath.length () <= 0 && (kern && tests[i].useKernel)) +@@ -452,7 +466,7 @@ AddTestCase (new DceQuaggaTestCase (std::string (tests[i].name), Seconds (tests[i].duration), tests[i].useKernel, (!kern && tests[i].useKernel)), diff --git a/wscript b/wscript index 7fc419db..47dc0213 100755 --- a/wscript +++ b/wscript @@ -19,6 +19,10 @@ def options(opt): help=('Path to the prefix where the kernel wrapper headers are installed'), default=None, dest='kernel_stack', type="string") + opt.add_option('--with-lkl', + help=('Path to the output of utils/build_lkl.sh, or to an LKL source tree built with it'), + default=None, + dest='with_lkl', type="string") opt.add_option('--enable-mpi', help=('Enable MPI and distributed simulation support'), dest='enable_mpi', action='store_true', @@ -168,6 +172,22 @@ def configure(conf): includes=os.path.join(kernel_stack_dir, 'include')) conf.env['KERNEL_STACK'] = kernel_stack_dir conf.env.append_value ('DEFINES', 'KERNEL_STACK=Y') + conf.env.append_value ('DEFINES', 'LINUX_STACK=Y') + + if Options.options.with_lkl: + lkl_dir = os.path.abspath(Options.options.with_lkl) + lkl_include = os.path.join(lkl_dir, 'include') + if not os.path.isfile(os.path.join(lkl_include, 'lkl', 'asm', 'host_ops.h')): + lkl_include = os.path.join(lkl_dir, 'tools', 'lkl', 'include') + conf.check(header_name='lkl/asm/host_ops.h', includes=lkl_include) + conf.env['LKL_INCLUDE'] = lkl_include + conf.env.append_value('DEFINES', 'LKL_STACK=Y') + if not conf.env['KERNEL_STACK']: + # Without libos, LKL is the Linux stack: it provides + # ns3::LinuxSocketFdFactory. + conf.env['LKL_LINUX'] = True + conf.env.append_value('DEFINES', 'LKL_LINUX=Y') + conf.env.append_value('DEFINES', 'LINUX_STACK=Y') conf.env['ENABLE_PYTHON_BINDINGS'] = False conf.env['EXAMPLE_DIRECTORIES'] = '.' @@ -270,7 +290,7 @@ def build_dce_tests(module, bld): tests_source = [ 'test/dce-manager-test.cc', ] - if bld.env['KERNEL_STACK']: + if bld.env['KERNEL_STACK'] or bld.env['LKL_LINUX']: tests_source += [ 'test/dce-cradle-test.cc', 'test/dce-mptcp-test.cc', @@ -517,9 +537,10 @@ def build_dce_kernel_examples(module, bld): target='bin/dce-sctp-simple', source=['example/dce-sctp-simple.cc']) - module.add_example(needed = ['core', 'network', 'dce', 'wifi', 'point-to-point', 'csma', 'mobility' ], - target='bin/dce-freebsd', - source=['example/dce-freebsd.cc']) + if bld.env['KERNEL_STACK']: + module.add_example(needed = ['core', 'network', 'dce', 'wifi', 'point-to-point', 'csma', 'mobility' ], + target='bin/dce-freebsd', + source=['example/dce-freebsd.cc']) # Add a script to build system @@ -619,11 +640,29 @@ def build(bld): 'model/linux/linux-socket-impl.h', ] kernel_includes = [bld.env['KERNEL_STACK']] + elif bld.env['LKL_LINUX']: + kernel_source = [ + 'model/lkl/lkl-linux-socket-fd-factory.cc', + 'model/linux/linux-socket-impl.cc', + ] + kernel_headers = [ + 'model/linux-socket-fd-factory.h', + 'model/linux/linux-socket-impl.h', + ] + kernel_includes = ['model/lkl'] else: kernel_source = [] kernel_headers = [] kernel_includes = [] + if bld.env['LKL_INCLUDE']: + kernel_source += ['model/lkl/lkl-kernel.cc', + 'model/lkl/lkl-socket-fd.cc', + 'model/lkl/lkl-socket-fd-factory.cc'] + kernel_headers += ['model/lkl/lkl-kernel.h', + 'model/lkl/lkl-socket-fd-factory.h'] + kernel_includes += [bld.env['LKL_INCLUDE']] + module_source = [ 'model/dce-manager.cc', 'model/dce-application.cc', @@ -787,8 +826,17 @@ def build(bld): source=['test/netlink-socket-test.cc'], name='netlink') - if bld.env['KERNEL_STACK']: + if bld.env['KERNEL_STACK'] or bld.env['LKL_LINUX']: build_dce_kernel_examples(module, bld) + + if bld.env['LKL_INCLUDE']: + module.add_example(needed = ['core', 'network', 'dce'], + target='bin/dce-lkl-boot', + includes=[bld.env['LKL_INCLUDE']], + source=['example/dce-lkl-boot.cc']) + module.add_example(needed = ['core', 'network', 'dce', 'point-to-point'], + target='bin/dce-lkl-p2p', + source=['example/dce-lkl-p2p.cc']) # build test-runner module.add_example(target='bin/test-runner', From bc7f2f03f6da0cc488fa6b89ecf0b7186a19feb2 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Thu, 8 Oct 2026 13:59:31 +0200 Subject: [PATCH 07/71] Examples and tests: forwarding, Linux MPTCP (v1), iproute2 6.12 - Routers enable IPv4 forwarding where the examples assumed it, with conf.all.forwarding (conf.default.forwarding only applies to interfaces created afterwards): dce-tcp-ns3-nsc-comparison received nothing on LKL. - The MPTCP test and examples were written for the out-of-tree MPTCP (v0) that libos never had: the test skipped itself, the examples ran single-path TCP if anything. Now on Linux MPTCP (v1), with LKL: - Applications get MPTCP sockets (LklSocketFdFactory::Mptcp); subflows come from "ip mptcp limits/endpoint" (v0's fullmesh path manager, "ip link set multipath" and the net.mptcp.mptcp_* sysctls are gone). - iperf servers no longer use -P 1: they stop listening after that many connections, and Linux MPTCP accepts subflows on the listener. - dce-mptcp test: runs when .net.mptcp.enabled is 1 and checks that packets from both client addresses reach the server. - dce-cradle-mptcp, dce-iperf-mptcp fail unless both paths carry data; the handoff examples report what each address carried. - build_kernel_deps.sh builds iproute2 6.12 (has "ip mptcp"; also works with libos); LIBOS=0 skips net-next-nuse and freebsd-sim. Co-Authored-By: Claude Opus 5.5 --- example/dce-cradle-mptcp.cc | 62 ++++++++++++++-- example/dce-iperf-mptcp.cc | 65 +++++++++++++--- example/dce-mptcp-handoff-v4v6.cc | 62 ++++++++++++++-- example/dce-mptcp-handoff-v6.cc | 58 +++++++++++++-- example/dce-mptcp-lte-wifi-v6.cc | 15 +++- example/dce-mptcp-lte-wifi.cc | 18 ++++- example/dce-tcp-ns3-nsc-comparison.cc | 2 +- test/dce-mptcp-test.cc | 52 ++++++++++--- utils/build_kernel_deps.sh | 102 ++++++++++++++------------ 9 files changed, 341 insertions(+), 95 deletions(-) diff --git a/example/dce-cradle-mptcp.cc b/example/dce-cradle-mptcp.cc index d639bf06..c3154d3d 100644 --- a/example/dce-cradle-mptcp.cc +++ b/example/dce-cradle-mptcp.cc @@ -8,6 +8,24 @@ #include "ns3/constant-position-mobility-model.h" using namespace ns3; + +// Bytes the server received from each address of the client. +static std::map g_received; + +static void +ServerRx (Ptr p) +{ + Ptr packet = p->Copy (); + PppHeader ppp; + Ipv4Header ip; + packet->RemoveHeader (ppp); + if (ppp.GetProtocol () == 0x0021) // IPv4 + { + packet->RemoveHeader (ip); + g_received[ip.GetSource ()] += packet->GetSize (); + } +} + void setPos (Ptr n, int x, int y, int z) { Ptr loc = CreateObject (); @@ -27,6 +45,10 @@ int main (int argc, char *argv[]) nodes.Create (2); routers.Create (nRtrs); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only + // with LKL; libos has no MPTCP). + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + DceManagerHelper dceManager; dceManager.SetTaskManagerAttribute ("FiberManagerType", StringValue ("UcontextFiberManager")); @@ -99,16 +121,26 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.1), "route add default via 10.2.0.2 dev sim0"); LinuxStackHelper::RunIp (nodes.Get (0), Seconds (0.1), "rule show"); - // Schedule Up/Down (XXX: didn't work...) - LinuxStackHelper::RunIp (nodes.Get (1), Seconds (1.0), "link set dev sim0 multipath off"); - LinuxStackHelper::RunIp (nodes.Get (1), Seconds (15.0), "link set dev sim0 multipath on"); - LinuxStackHelper::RunIp (nodes.Get (1), Seconds (30.0), "link set dev sim0 multipath off"); - + stack.SysctlSet (routers, ".net.ipv4.conf.all.forwarding", "1"); - // debug - stack.SysctlSet (nodes, ".net.mptcp.mptcp_debug", "1"); + // MPTCP: up to 4 subflows per connection; the client opens subflows from + // its other addresses, the server announces its other addresses. + for (uint32_t n = 0; n < 2; n++) + { + LinuxStackHelper::RunIp (nodes.Get (n), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + } + for (uint32_t i = 1; i < nRtrs; i++) + { + cmd_oss.str (""); + cmd_oss << "mptcp endpoint add 10.1." << i << ".1 dev sim" << i << " subflow fullmesh"; + LinuxStackHelper::RunIp (nodes.Get (0), Seconds (0.5), cmd_oss.str ().c_str ()); + cmd_oss.str (""); + cmd_oss << "mptcp endpoint add 10.2." << i << ".1 dev sim" << i << " signal"; + LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.5), cmd_oss.str ().c_str ()); + } + Config::ConnectWithoutContext ("/NodeList/1/DeviceList/*/$ns3::PointToPointNetDevice/PhyRxEnd", + MakeCallback (&ServerRx)); - // Launch iperf client on node 0 ApplicationContainer apps; OnOffHelper onoff = OnOffHelper ("ns3::LinuxTcpSocketFactory", InetSocketAddress ("10.2.0.1", 9)); @@ -136,5 +168,19 @@ int main (int argc, char *argv[]) Simulator::Run (); Simulator::Destroy (); + bool ok = true; + for (uint32_t i = 0; i < nRtrs; i++) + { + std::ostringstream addr; + addr << "10.1." << i << ".1"; + uint64_t bytes = g_received[Ipv4Address (addr.str ().c_str ())]; + std::cout << "received from " << addr.str () << ": " << bytes << " bytes" << std::endl; + ok = ok && bytes > 0; + } +#ifdef LKL_LINUX + // With MPTCP, every path carries data. + return ok ? 0 : 1; +#else return 0; +#endif } diff --git a/example/dce-iperf-mptcp.cc b/example/dce-iperf-mptcp.cc index 0c5c620b..75e205f5 100644 --- a/example/dce-iperf-mptcp.cc +++ b/example/dce-iperf-mptcp.cc @@ -8,6 +8,24 @@ #include "ns3/constant-position-mobility-model.h" using namespace ns3; + +// Bytes the server received from each address of the client. +static std::map g_received; + +static void +ServerRx (Ptr p) +{ + Ptr packet = p->Copy (); + PppHeader ppp; + Ipv4Header ip; + packet->RemoveHeader (ppp); + if (ppp.GetProtocol () == 0x0021) // IPv4 + { + packet->RemoveHeader (ip); + g_received[ip.GetSource ()] += packet->GetSize (); + } +} + void setPos (Ptr n, int x, int y, int z) { Ptr loc = CreateObject (); @@ -27,6 +45,10 @@ int main (int argc, char *argv[]) nodes.Create (2); routers.Create (nRtrs); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only + // with LKL; libos has no MPTCP). + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + DceManagerHelper dceManager; dceManager.SetTaskManagerAttribute ("FiberManagerType", StringValue ("UcontextFiberManager")); @@ -99,14 +121,25 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.1), "route add default via 10.2.0.2 dev sim0"); LinuxStackHelper::RunIp (nodes.Get (0), Seconds (0.1), "rule show"); - // Schedule Up/Down (XXX: didn't work...) - LinuxStackHelper::RunIp (nodes.Get (1), Seconds (1.0), "link set dev sim0 multipath off"); - LinuxStackHelper::RunIp (nodes.Get (1), Seconds (15.0), "link set dev sim0 multipath on"); - LinuxStackHelper::RunIp (nodes.Get (1), Seconds (30.0), "link set dev sim0 multipath off"); + stack.SysctlSet (routers, ".net.ipv4.conf.all.forwarding", "1"); - - // debug - stack.SysctlSet (nodes, ".net.mptcp.mptcp_debug", "1"); + // MPTCP: up to 4 subflows per connection; the client opens subflows from + // its other addresses, the server announces its other addresses. + for (uint32_t n = 0; n < 2; n++) + { + LinuxStackHelper::RunIp (nodes.Get (n), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + } + for (uint32_t i = 1; i < nRtrs; i++) + { + cmd_oss.str (""); + cmd_oss << "mptcp endpoint add 10.1." << i << ".1 dev sim" << i << " subflow fullmesh"; + LinuxStackHelper::RunIp (nodes.Get (0), Seconds (0.5), cmd_oss.str ().c_str ()); + cmd_oss.str (""); + cmd_oss << "mptcp endpoint add 10.2." << i << ".1 dev sim" << i << " signal"; + LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.5), cmd_oss.str ().c_str ()); + } + Config::ConnectWithoutContext ("/NodeList/1/DeviceList/*/$ns3::PointToPointNetDevice/PhyRxEnd", + MakeCallback (&ServerRx)); DceApplicationHelper dce; ApplicationContainer apps; @@ -132,9 +165,9 @@ int main (int argc, char *argv[]) dce.SetBinary ("iperf"); dce.ResetArguments (); dce.ResetEnvironment (); + // Not -P 1: the server stops listening after that many connections, + // and Linux MPTCP accepts the subflows of a connection on its listener. dce.AddArgument ("-s"); - dce.AddArgument ("-P"); - dce.AddArgument ("1"); apps = dce.Install (nodes.Get (1)); pointToPoint.EnablePcapAll ("iperf-mptcp", false); @@ -148,5 +181,19 @@ int main (int argc, char *argv[]) Simulator::Run (); Simulator::Destroy (); + bool ok = true; + for (uint32_t i = 0; i < nRtrs; i++) + { + std::ostringstream addr; + addr << "10.1." << i << ".1"; + uint64_t bytes = g_received[Ipv4Address (addr.str ().c_str ())]; + std::cout << "received from " << addr.str () << ": " << bytes << " bytes" << std::endl; + ok = ok && bytes > 0; + } +#ifdef LKL_LINUX + // With MPTCP, every path carries data. + return ok ? 0 : 1; +#else return 0; +#endif } diff --git a/example/dce-mptcp-handoff-v4v6.cc b/example/dce-mptcp-handoff-v4v6.cc index a555d0fe..c3a8950a 100644 --- a/example/dce-mptcp-handoff-v4v6.cc +++ b/example/dce-mptcp-handoff-v4v6.cc @@ -32,7 +32,7 @@ #include "ns3/core-module.h" #include "ns3/internet-module.h" #include "ns3/dce-module.h" -#include "ns3/point-to-point-helper.h" +#include "ns3/point-to-point-module.h" #include "ns3/wifi-module.h" #include "ns3/yans-wifi-helper.h" #include "ns3/mobility-module.h" @@ -41,6 +41,35 @@ using namespace ns3; +// Bytes the server received from each address of the mobile node. +static std::map g_received; + +static void +ServerRx (Ptr p) +{ + Ptr packet = p->Copy (); + PppHeader ppp; + packet->RemoveHeader (ppp); + std::ostringstream source; + if (ppp.GetProtocol () == 0x0057) // IPv6 + { + Ipv6Header ip; + packet->RemoveHeader (ip); + source << ip.GetSource (); + } + else if (ppp.GetProtocol () == 0x0021) // IPv4 + { + Ipv4Header ip; + packet->RemoveHeader (ip); + source << ip.GetSource (); + } + else + { + return; + } + g_received[source.str ()] += packet->GetSize (); +} + static void AddAddress (Ptr node, Time at, const char *name, const char *address) { @@ -139,6 +168,10 @@ int main (int argc, char *argv[]) wifi.Install (phy, mac, ar.Get (1)); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only + // with LKL; libos has no MPTCP). + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + DceManagerHelper dceMng; DceApplicationHelper dce; LinuxStackHelper stack; @@ -179,6 +212,7 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (router.Get (0), Seconds (3.15), "-6 route add 2001:1:2:4::/64 via 2001:1:2:1::2 dev sim1"); LinuxStackHelper::RunIp (router.Get (0), Seconds (3.15), "-6 route add 2001:1:2:7::/64 via 2001:1:2:2::2 dev sim2"); stack.SysctlSet (router, ".net.ipv6.conf.all.forwarding", "1"); + stack.SysctlSet (router, ".net.ipv4.conf.all.forwarding", "1"); // For AR1 (the intermediate node) AddAddress (ar.Get (0), Seconds (0.1), "sim0", "2001:1:2:1::2/64"); @@ -199,6 +233,7 @@ int main (int argc, char *argv[]) AddAddress (ar.Get (1), Seconds (0.12), "sim1", "2001:1:2:7::2/64"); LinuxStackHelper::RunIp (ar.Get (1), Seconds (0.13), "link set sim1 up"); stack.SysctlSet (ar, ".net.ipv6.conf.all.forwarding", "1"); + stack.SysctlSet (ar, ".net.ipv4.conf.all.forwarding", "1"); // For MN LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.11), "link set lo up"); @@ -229,8 +264,19 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.0), "route show table all"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.1), "route get 2001:1:2:3::1 from 2001:1:2:4:200:ff:fe00:7"); - stack.SysctlSet (mn, ".net.mptcp.mptcp_debug", "1"); - stack.SysctlSet (sv, ".net.mptcp.mptcp_debug", "1"); + // MPTCP: up to 4 subflows per connection; the mobile node opens a + // subflow from each of its addresses (Linux MPTCP does not by itself), + // and the server announces its address of the other family. + LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (sv.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (sv.Get (0), Seconds (0.5), + std::string ("mptcp endpoint add ") + (v6Primary ? "192.168.0.1" : "2001:1:2:3::1") + " dev sim0 signal"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (9.0), "mptcp endpoint add 192.168.2.2 dev sim0 subflow"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (9.0), "mptcp endpoint add 2001:1:2:4:200:ff:fe00:7 dev sim0 subflow"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (9.0), "mptcp endpoint add 2001:1:2:7:200:ff:fe00:9 dev sim1 subflow"); + Config::ConnectWithoutContext ("/NodeList/" + std::to_string (sv.Get (0)->GetId ()) + + "/DeviceList/0/$ns3::PointToPointNetDevice/PhyRxEnd", + MakeCallback (&ServerRx)); { ApplicationContainer apps; @@ -241,10 +287,10 @@ int main (int argc, char *argv[]) dce.SetBinary ("iperf"); dce.ResetArguments (); dce.ResetEnvironment (); + // Not -P 1: the server stops listening after that many connections, + // and Linux MPTCP accepts the subflows of a connection on its listener. dce.AddArgument ("-V"); dce.AddArgument ("-s"); - dce.AddArgument ("-P"); - dce.AddArgument ("1"); apps = dce.Install (sv); apps.Start (Seconds (4)); // apps.Stop (Seconds (45.0)); @@ -285,5 +331,11 @@ int main (int argc, char *argv[]) Simulator::Run (); Simulator::Destroy (); + // Which addresses of the mobile node carried data depends on which access + // points it reaches from where it was placed. + for (std::map::iterator i = g_received.begin (); i != g_received.end (); ++i) + { + std::cout << "received from " << i->first << ": " << i->second << " bytes" << std::endl; + } return 0; } diff --git a/example/dce-mptcp-handoff-v6.cc b/example/dce-mptcp-handoff-v6.cc index 0a590493..850688db 100644 --- a/example/dce-mptcp-handoff-v6.cc +++ b/example/dce-mptcp-handoff-v6.cc @@ -32,7 +32,7 @@ #include "ns3/core-module.h" #include "ns3/internet-module.h" #include "ns3/dce-module.h" -#include "ns3/point-to-point-helper.h" +#include "ns3/point-to-point-module.h" #include "ns3/wifi-module.h" #include "ns3/yans-wifi-helper.h" #include "ns3/mobility-module.h" @@ -41,6 +41,35 @@ using namespace ns3; +// Bytes the server received from each address of the mobile node. +static std::map g_received; + +static void +ServerRx (Ptr p) +{ + Ptr packet = p->Copy (); + PppHeader ppp; + packet->RemoveHeader (ppp); + std::ostringstream source; + if (ppp.GetProtocol () == 0x0057) // IPv6 + { + Ipv6Header ip; + packet->RemoveHeader (ip); + source << ip.GetSource (); + } + else if (ppp.GetProtocol () == 0x0021) // IPv4 + { + Ipv4Header ip; + packet->RemoveHeader (ip); + source << ip.GetSource (); + } + else + { + return; + } + g_received[source.str ()] += packet->GetSize (); +} + static void AddAddress (Ptr node, Time at, const char *name, const char *address) { @@ -138,6 +167,10 @@ int main (int argc, char *argv[]) wifi.Install (phy, mac, ar.Get (1)); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only + // with LKL; libos has no MPTCP). + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + DceManagerHelper dceMng; DceApplicationHelper dce; LinuxStackHelper stack; @@ -218,8 +251,17 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.0), "route show table all"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.1), "route get 2001:1:2:3::1 from 2001:1:2:4:200:ff:fe00:7"); - stack.SysctlSet (mn, ".net.mptcp.mptcp_debug", "1"); - stack.SysctlSet (sv, ".net.mptcp.mptcp_debug", "1"); + // MPTCP: up to 4 subflows per connection, and a subflow from each + // address the mobile node gets on sim1 (from AR2's router advertisements, + // then the one added at 15.2 s). Linux MPTCP does not open subflows from + // new addresses by itself. + LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (sv.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (9.0), "mptcp endpoint add 2001:1:2:7:200:ff:fe00:9 dev sim1 subflow"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (15.3), "mptcp endpoint add 2001:1:2:7::3939 dev sim1 subflow"); + Config::ConnectWithoutContext ("/NodeList/" + std::to_string (sv.Get (0)->GetId ()) + + "/DeviceList/0/$ns3::PointToPointNetDevice/PhyRxEnd", + MakeCallback (&ServerRx)); { ApplicationContainer apps; @@ -230,10 +272,10 @@ int main (int argc, char *argv[]) dce.SetBinary ("iperf"); dce.ResetArguments (); dce.ResetEnvironment (); + // Not -P 1: the server stops listening after that many connections, + // and Linux MPTCP accepts the subflows of a connection on its listener. dce.AddArgument ("-V"); dce.AddArgument ("-s"); - dce.AddArgument ("-P"); - dce.AddArgument ("1"); apps = dce.Install (sv); apps.Start (Seconds (4)); @@ -267,5 +309,11 @@ int main (int argc, char *argv[]) Simulator::Run (); Simulator::Destroy (); + // Which addresses of the mobile node carried data depends on which access + // points it reaches from where it was placed. + for (std::map::iterator i = g_received.begin (); i != g_received.end (); ++i) + { + std::cout << "received from " << i->first << ": " << i->second << " bytes" << std::endl; + } return 0; } diff --git a/example/dce-mptcp-lte-wifi-v6.cc b/example/dce-mptcp-lte-wifi-v6.cc index e3775b24..eb21af2a 100644 --- a/example/dce-mptcp-lte-wifi-v6.cc +++ b/example/dce-mptcp-lte-wifi-v6.cc @@ -160,6 +160,9 @@ int main (int argc, char *argv[]) DceManagerHelper dceMng; DceApplicationHelper dce; LinuxStackHelper stack; + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only + // with LKL; libos has no MPTCP). + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); dceMng.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); // dceMng.SetLoader ("ns3::DlmLoaderFactory"); @@ -280,8 +283,12 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.0), "route show table all"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.1), "route get 2001:1:2:3::1 from 2001:1:2:4:200:ff:fe00:a"); - stack.SysctlSet (mn, ".net.mptcp.mptcp_debug", "1"); - stack.SysctlSet (sv, ".net.mptcp.mptcp_debug", "1"); + // MPTCP: a subflow from the address the mobile node gets on its second + // interface (Linux MPTCP does not open subflows from new addresses by + // itself). + LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (sv.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (9.0), "mptcp endpoint add 2001:1:2:4:200:ff:fe00:a dev sim1 subflow"); { ApplicationContainer apps; @@ -293,9 +300,9 @@ int main (int argc, char *argv[]) dce.ResetArguments (); dce.ResetEnvironment (); dce.AddArgument ("-V"); + // Not -P 1: Linux MPTCP accepts the subflows of a connection on its + // listener. dce.AddArgument ("-s"); - dce.AddArgument ("-P"); - dce.AddArgument ("1"); dce.AddArgument ("-i"); dce.AddArgument ("1"); apps = dce.Install (sv); diff --git a/example/dce-mptcp-lte-wifi.cc b/example/dce-mptcp-lte-wifi.cc index cdcfaa0e..e6d664da 100644 --- a/example/dce-mptcp-lte-wifi.cc +++ b/example/dce-mptcp-lte-wifi.cc @@ -84,6 +84,9 @@ int main (int argc, char *argv[]) routers.Create (1); DceManagerHelper dceManager; + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only + // with LKL; libos has no MPTCP). + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); LinuxStackHelper stack; @@ -185,6 +188,11 @@ int main (int argc, char *argv[]) cmd_oss.str (""); cmd_oss << "route add 10.1.0.0/16 via " << if1.GetAddress (0, 0) << " dev sim0"; LinuxStackHelper::RunIp (routers.Get (0), Seconds (0.2), cmd_oss.str ().c_str ()); + // MPTCP: a subflow over WiFi + cmd_oss.str (""); + cmd_oss << "mptcp endpoint add " << if1.GetAddress (0, 0) << " dev sim" + << devices1.Get (0)->GetIfIndex () << " subflow"; + LinuxStackHelper::RunIp (nodes.Get (0), Seconds (0.5), cmd_oss.str ().c_str ()); // Global default route if (disLte) @@ -237,7 +245,11 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (nodes.Get (1), Seconds (5.1), "route show table all"); // debug - stack.SysctlSet (nodes, ".net.mptcp.mptcp_debug", "1"); + for (uint32_t n = 0; n < 2; n++) + { + LinuxStackHelper::RunIp (nodes.Get (n), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + } + stack.SysctlSet (routers, ".net.ipv4.conf.all.forwarding", "1"); #if 1 LinuxStackHelper::SysctlGet (nodes.Get (0), NanoSeconds (0), @@ -309,9 +321,9 @@ int main (int argc, char *argv[]) dce.SetBinary ("iperf"); dce.ResetArguments (); dce.ResetEnvironment (); + // Not -P 1: Linux MPTCP accepts the subflows of a connection on its + // listener. dce.AddArgument ("-s"); - dce.AddArgument ("-P"); - dce.AddArgument ("1"); #if 0 if (bufSize.length () != 0) { diff --git a/example/dce-tcp-ns3-nsc-comparison.cc b/example/dce-tcp-ns3-nsc-comparison.cc index 70b8a085..60b56442 100644 --- a/example/dce-tcp-ns3-nsc-comparison.cc +++ b/example/dce-tcp-ns3-nsc-comparison.cc @@ -178,7 +178,7 @@ main (int argc, char *argv[]) { LinuxStackHelper::PopulateRoutingTables (); dceManager.Install (nodes); - stack.SysctlSet (nodes, ".net.ipv4.conf.default.forwarding", "1"); + stack.SysctlSet (nodes, ".net.ipv4.conf.all.forwarding", "1"); } // dceManager.RunIp (lefts.Get (0), Seconds (0.2), "route add default via 10.0.0.2"); diff --git a/test/dce-mptcp-test.cc b/test/dce-mptcp-test.cc index 5f04d186..0146e032 100644 --- a/test/dce-mptcp-test.cc +++ b/test/dce-mptcp-test.cc @@ -74,23 +74,38 @@ ReceivedBytes (std::string context, Ptr originalPacket) } } +// Linux MPTCP (upstream, v1): the test runs if the kernel has it. bool isMptcpEnabled = false; static void SetMptcpEnabled (std::string key, std::string value) { std::cout << key << "=" << value << std::endl; - if (key == ".net.mptcp.mptcp_debug" && + if (key == ".net.mptcp.enabled" && value.find ("1") != std::string::npos) { isMptcpEnabled = true; } } +// Let each connection have more subflows and accept the addresses its +// peer announces. The applications' TCP sockets are made MPTCP sockets +// (LklSocketFdFactory::Mptcp) at the start of each test. +static void +EnableMptcp (NodeContainer nodes) +{ + for (uint32_t i = 0; i < nodes.GetN (); i++) + { + LinuxStackHelper::RunIp (nodes.Get (i), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + } +} + void DceMptcpTestCase::DoBasicRun (void) { g_rcv0 = false; g_rcv1 = false; + isMptcpEnabled = false; + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); uint32_t nRtrs = 2; std::string m_rate = "200Bps"; @@ -110,7 +125,8 @@ DceMptcpTestCase::DoBasicRun (void) dceManager.Install (routers); LinuxStackHelper::SysctlGet (nodes.Get (0), Seconds (1.0), - ".net.mptcp.mptcp_debug", &SetMptcpEnabled); + ".net.mptcp.enabled", &SetMptcpEnabled); + EnableMptcp (nodes); PointToPointHelper pointToPoint; NetDeviceContainer devices1, devices2; @@ -141,6 +157,13 @@ DceMptcpTestCase::DoBasicRun (void) cmd_oss.str (""); cmd_oss << "route add 10.1.0.0/16 via " << if1.GetAddress (1, 0) << " dev sim0"; LinuxStackHelper::RunIp (routers.Get (i), Seconds (0.2), cmd_oss.str ().c_str ()); + if (i > 0) + { + // a subflow from each other address of the client + cmd_oss.str (""); + cmd_oss << "mptcp endpoint add " << if1.GetAddress (0, 0) << " dev sim" << i << " subflow"; + LinuxStackHelper::RunIp (nodes.Get (0), Seconds (0.6), cmd_oss.str ().c_str ()); + } // delete address at 20sec LinuxStackHelper::RunIp (nodes.Get (0), Seconds (20), "-f inet addr delete 10.1.1.1/24 dev sim1"); @@ -171,9 +194,7 @@ DceMptcpTestCase::DoBasicRun (void) LinuxStackHelper::RunIp (nodes.Get (0), Seconds (0.1), "route add default via 10.1.0.2 dev sim0"); LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.1), "route add default via 10.2.0.2 dev sim0"); LinuxStackHelper::RunIp (nodes.Get (0), Seconds (0.1), "rule show"); - - // debug - stack.SysctlSet (nodes, ".net.mptcp.mptcp_debug", "1"); + stack.SysctlSet (routers, ".net.ipv4.conf.all.forwarding", "1"); ApplicationContainer apps; OnOffHelper onoff = OnOffHelper (m_sockf, @@ -204,6 +225,7 @@ DceMptcpTestCase::DoBasicRun (void) " bytes"; std::cout << std::endl; Simulator::Destroy (); + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (false)); if (!isMptcpEnabled) { @@ -220,6 +242,8 @@ DceMptcpTestCase::DoAddrTestRun (void) { g_rcv0 = false; g_rcv1 = false; + isMptcpEnabled = false; + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); double stopTime = 15.0; std::string p2pdelay = "10ms"; @@ -238,7 +262,8 @@ DceMptcpTestCase::DoAddrTestRun (void) dceManager.Install (nodes); LinuxStackHelper::SysctlGet (nodes.Get (0), Seconds (1.0), - ".net.mptcp.mptcp_debug", &SetMptcpEnabled); + ".net.mptcp.enabled", &SetMptcpEnabled); + EnableMptcp (nodes); pointToPoint.SetDeviceAttribute ("DataRate", StringValue ("100Kb/s")); pointToPoint.SetChannelAttribute ("Delay", StringValue (p2pdelay)); @@ -268,18 +293,15 @@ DceMptcpTestCase::DoAddrTestRun (void) LinuxStackHelper::RunIp (nodes.Get (0), Seconds (3.4), "rule add from 10.1.1.0/24 lookup 1"); LinuxStackHelper::RunIp (nodes.Get (0), Seconds (3.3), "route add default via 10.1.1.10 dev sim0 table 1"); LinuxStackHelper::RunIp (nodes.Get (0), Seconds (3.1), "route add 10.1.1.0/24 dev sim0 table 1"); + // the new address gets a subflow + LinuxStackHelper::RunIp (nodes.Get (0), Seconds (3.5), "mptcp endpoint add 10.1.1.1 dev sim0 subflow"); /*Setup Gateway Addresses*/ LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.1), "link set up dev sim1"); LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.2), "addr add 10.1.0.10/24 dev sim1"); LinuxStackHelper::RunIp (nodes.Get (1), Seconds (3), "addr add 10.1.1.10/24 dev sim1"); - /*Enable Multipath and debugging etc*/ - stack.SysctlSet (nodes, ".net.ipv4.conf.default.forwarding", "1"); - stack.SysctlSet (nodes, ".net.mptcp.mptcp_debug", "1"); - stack.SysctlSet (nodes, ".net.mptcp.mptcp_enabled", "1"); - stack.SysctlSet (nodes, ".net.mptcp.mptcp_path_manager", "fullmesh"); - stack.SysctlSet (nodes, ".net.ipv4.tcp_congestion_control", "olia"); + stack.SysctlSet (nodes, ".net.ipv4.conf.all.forwarding", "1"); DceApplicationHelper dce; ApplicationContainer apps; @@ -310,6 +332,7 @@ DceMptcpTestCase::DoAddrTestRun (void) Simulator::Stop (Seconds (stopTime)); Simulator::Run (); Simulator::Destroy (); + Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (false)); if (!isMptcpEnabled) { @@ -369,7 +392,12 @@ DceMptcpTestSuite::DceMptcpTestSuite () Packet::EnablePrinting (); // for the moment: not supported dce cradle for freebsd +#ifdef LKL_LINUX + // ns3::LinuxSocketFdFactory is LKL + std::string filePath = SearchExecFile ("DCE_PATH", "liblkl.so", 0); +#else std::string filePath = SearchExecFile ("DCE_PATH", "liblinux.so", 0); +#endif for (unsigned int i = 0; i < sizeof(tests)/sizeof(testPair); i++) { if (filePath.length () <= 0) diff --git a/utils/build_kernel_deps.sh b/utils/build_kernel_deps.sh index d01df69d..0f91e417 100755 --- a/utils/build_kernel_deps.sh +++ b/utils/build_kernel_deps.sh @@ -11,11 +11,13 @@ # Usage: ./utils/build_kernel_deps.sh [deps_dir] # deps_dir: output directory (default: ../dce-kernel-deps) # JOBS: parallel make jobs (default: nproc) +# LIBOS: 0 skips net-next-nuse and freebsd-sim, for builds whose Linux +# stack is LKL (utils/build_lkl.sh) (default: 1) # # Outputs (sources and build trees stay in /src, which can be # deleted afterwards): # /bin_dce/ binaries and libraries for DCE_PATH -# /kernel/ argument for --enable-kernel-stack +# /kernel/ argument for --enable-kernel-stack (LIBOS=1) # /ns-3-dce-quagga/ module to copy into myscripts/ # @@ -26,10 +28,11 @@ DEPS_DIR="$(mkdir -p "${1:-${DCE_DIR}/../dce-kernel-deps}" && cd "${1:-${DCE_DIR BIN_DCE="${DEPS_DIR}/bin_dce" SRC="${DEPS_DIR}/src" JOBS="${JOBS:-$(nproc)}" +LIBOS="${LIBOS:-1}" NUSE_REV="libos-v4.4-fix1" FREEBSD_REV="sim-ns3-dev-branch" -IPROUTE2_REV="v4.4.0" +IPROUTE2_REV="v6.12.0" # matches LKL's kernel; has "ip mptcp" IPUTILS_REV="s20101006" QUAGGA_MODULE_REV="b57e0f3184e34107c7e452b3c41807e0aff5b5ce" IPERF_VERSION="2.0.5" @@ -66,60 +69,63 @@ fetch_tar () { fi } -echo "== net-next-nuse (Linux 4.4 libos)" -fetch_git https://github.com/libos-nuse/net-next-nuse.git "${NUSE_REV}" net-next-nuse -( - cd "${SRC}/net-next-nuse" - make defconfig ARCH=lib > /dev/null - # Generate objs.mk serially: under -j, GNU Make >= 4.4 runs this rule - # twice concurrently and the output file gets interleaved. - make ARCH=lib arch/lib/objs.mk > /dev/null - # The NUSE/rump tools fail to build and are not used by DCE; only the - # libsim-linux library from arch/lib/tools is required. - make library ARCH=lib -j"${JOBS}" -k > build.log 2>&1 || true - test -f arch/lib/tools/libsim-linux-4.4.0.so || { tail -50 build.log; exit 1; } -) -cp "${SRC}/net-next-nuse/arch/lib/tools/libsim-linux-4.4.0.so" "${BIN_DCE}/" -ln -sf libsim-linux-4.4.0.so "${BIN_DCE}/liblinux.so" -# DCE only needs the libos API headers from the kernel tree. -rm -rf "${DEPS_DIR}/kernel" -mkdir -p "${DEPS_DIR}/kernel/lib" -cp -a "${SRC}/net-next-nuse/arch/lib/include" "${DEPS_DIR}/kernel/lib/" - -echo "== freebsd-sim" -fetch_git https://github.com/direct-code-execution/freebsd-sim.git "${FREEBSD_REV}" freebsd-sim -( - cd "${SRC}/freebsd-sim" - git apply --check "${DCE_DIR}/utils/freebsd-sim-dce.patch" 2> /dev/null \ - && git apply "${DCE_DIR}/utils/freebsd-sim-dce.patch" - cd sys/sim - # The kernel config tool and generated headers must exist before the - # parallel kernel build; the Makefile does not order this correctly. - make -f Makefile.config clean > /dev/null - make -f Makefile.config CC="gcc ${LEGACY_CFLAGS}" > /dev/null - rm -rf compile && mkdir -p compile - (cd conf && ../config -C DCE > /dev/null) +if [ "${LIBOS}" = 1 ]; then + echo "== net-next-nuse (Linux 4.4 libos)" + fetch_git https://github.com/libos-nuse/net-next-nuse.git "${NUSE_REV}" net-next-nuse ( - cd compile/DCE - for m in device_if cpufreq_if bus_if linker_if clock_if; do - awk -f ../../../tools/makeobjops.awk ../../../kern/$m.m -h - done - awk -f ../../../tools/vnode_if.awk ../../../kern/vnode_if.src -q -p -h -c + cd "${SRC}/net-next-nuse" + make defconfig ARCH=lib > /dev/null + # Generate objs.mk serially: under -j, GNU Make >= 4.4 runs this rule + # twice concurrently and the output file gets interleaved. + make ARCH=lib arch/lib/objs.mk > /dev/null + # The NUSE/rump tools fail to build and are not used by DCE; only the + # libsim-linux library from arch/lib/tools is required. + make library ARCH=lib -j"${JOBS}" -k > build.log 2>&1 || true + test -f arch/lib/tools/libsim-linux-4.4.0.so || { tail -50 build.log; exit 1; } ) - touch config - make buildkernel -j"${JOBS}" CC="gcc ${LEGACY_CFLAGS}" > build.log 2>&1 || { tail -50 build.log; exit 1; } -) -cp "${SRC}/freebsd-sim/libsim-freebsd.git.so" "${BIN_DCE}/" -ln -sf libsim-freebsd.git.so "${BIN_DCE}/libfreebsd.so" + cp "${SRC}/net-next-nuse/arch/lib/tools/libsim-linux-4.4.0.so" "${BIN_DCE}/" + ln -sf libsim-linux-4.4.0.so "${BIN_DCE}/liblinux.so" + # DCE only needs the libos API headers from the kernel tree. + rm -rf "${DEPS_DIR}/kernel" + mkdir -p "${DEPS_DIR}/kernel/lib" + cp -a "${SRC}/net-next-nuse/arch/lib/include" "${DEPS_DIR}/kernel/lib/" + + echo "== freebsd-sim" + fetch_git https://github.com/direct-code-execution/freebsd-sim.git "${FREEBSD_REV}" freebsd-sim + ( + cd "${SRC}/freebsd-sim" + git apply --check "${DCE_DIR}/utils/freebsd-sim-dce.patch" 2> /dev/null \ + && git apply "${DCE_DIR}/utils/freebsd-sim-dce.patch" + cd sys/sim + # The kernel config tool and generated headers must exist before the + # parallel kernel build; the Makefile does not order this correctly. + make -f Makefile.config clean > /dev/null + make -f Makefile.config CC="gcc ${LEGACY_CFLAGS}" > /dev/null + rm -rf compile && mkdir -p compile + (cd conf && ../config -C DCE > /dev/null) + ( + cd compile/DCE + for m in device_if cpufreq_if bus_if linker_if clock_if; do + awk -f ../../../tools/makeobjops.awk ../../../kern/$m.m -h + done + awk -f ../../../tools/vnode_if.awk ../../../kern/vnode_if.src -q -p -h -c + ) + touch config + make buildkernel -j"${JOBS}" CC="gcc ${LEGACY_CFLAGS}" > build.log 2>&1 || { tail -50 build.log; exit 1; } + ) + cp "${SRC}/freebsd-sim/libsim-freebsd.git.so" "${BIN_DCE}/" + ln -sf libsim-freebsd.git.so "${BIN_DCE}/libfreebsd.so" +fi echo "== iproute2 ${IPROUTE2_REV}" fetch_git https://git.kernel.org/pub/scm/network/iproute2/iproute2.git "${IPROUTE2_REV}" iproute2 ( cd "${SRC}/iproute2" ./configure > /dev/null - # glibc has setns(); the configure probe for it fails under GCC 14. - make -j"${JOBS}" "CCOPTS=-fpic -D_GNU_SOURCE -O0 -g -DUINT16_MAX=65535 -DHAVE_SETNS ${LEGACY_CFLAGS}" \ - "LDFLAGS=-pie -rdynamic" > /dev/null 2>&1 + # Only ip, linked with libc only (configure finds none of the optional + # libraries in a plain build environment). + make -j"${JOBS}" SUBDIRS="lib ip" "CCOPTS=-fpic -D_GNU_SOURCE -O0 -g -U_FORTIFY_SOURCE" \ + "LDFLAGS=-pie -rdynamic" > build.log 2>&1 || { tail -50 build.log; exit 1; } ) cp "${SRC}/iproute2/ip/ip" "${BIN_DCE}/" From a10ab76b347182ad99728d27a774e5a5305a0d0c Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Thu, 8 Oct 2026 13:59:32 +0200 Subject: [PATCH 08/71] CI: LKL is the Linux stack; one libos job keeps libos and FreeBSD covered - LKL is built (utils/build_lkl.sh, cached on the script, the patches and the kernel configuration) and installed with the DCE binaries. - Matrix: Ubuntu 24.04 and Debian trixie build DCE with LKL as ns3::LinuxSocketFdFactory (no --enable-kernel-stack, dependencies with LIBOS=0); Ubuntu 24.04 also builds the libos configuration (Linux 4.4 and FreeBSD, LKL as ns3::LklSocketFdFactory). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-and-test.yml | 45 ++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 57480e4b..64d0a437 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -17,16 +17,25 @@ env: jobs: build: - name: Build on ${{ matrix.distro }} + name: Build on ${{ matrix.distro }} (${{ matrix.stack }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: + # stack: the Linux kernel of ns3::LinuxSocketFdFactory + # lkl: Linux 6.12 (LKL) + # libos: Linux 4.4 (net-next-nuse), with FreeBSD; LKL is + # ns3::LklSocketFdFactory include: - distro: ubuntu-24.04 image: ubuntu:24.04 + stack: lkl - distro: debian-trixie image: debian:trixie + stack: lkl + - distro: ubuntu-24.04 + image: ubuntu:24.04 + stack: libos container: image: ${{ matrix.image }} steps: @@ -101,8 +110,9 @@ jobs: working-directory: ns-${{ env.NS3_VER }} run: ./ns3 install - # Linux/FreeBSD kernel libraries, DCE builds of ip, iperf, thttpd, - # wget, ping and quagga, and the ns-3-dce-quagga module. + # DCE builds of ip, iperf, thttpd, wget, ping and quagga, the + # ns-3-dce-quagga module and, for libos, the Linux 4.4 and FreeBSD + # kernel libraries. - name: Cache kernel stack dependencies id: cache-kernel-deps uses: actions/cache@v4 @@ -111,25 +121,47 @@ jobs: deps/bin_dce deps/kernel deps/ns-3-dce-quagga - key: kernel-deps-${{ matrix.distro }}-${{ hashFiles('ns-3-dce/utils/build_kernel_deps.sh', 'ns-3-dce/utils/*.patch') }} + key: kernel-deps-${{ matrix.distro }}-${{ matrix.stack }}-${{ hashFiles('ns-3-dce/utils/build_kernel_deps.sh', 'ns-3-dce/utils/*.patch') }} - name: Build kernel stack dependencies if: steps.cache-kernel-deps.outputs.cache-hit != 'true' working-directory: ns-3-dce + env: + LIBOS: ${{ matrix.stack == 'libos' && '1' || '0' }} run: | ./utils/build_kernel_deps.sh "$GITHUB_WORKSPACE/deps" rm -rf "$GITHUB_WORKSPACE/deps/src" + # Linux 6.12 kernel library (LKL) and its headers. + - name: Cache LKL + id: cache-lkl + uses: actions/cache@v4 + with: + path: deps/lkl + key: lkl-${{ matrix.distro }}-${{ hashFiles('ns-3-dce/utils/build_lkl.sh', 'ns-3-dce/utils/lkl-patches/*.patch', 'ns-3-dce/utils/lkl-dce.config') }} + + - name: Build LKL + if: steps.cache-lkl.outputs.cache-hit != 'true' + working-directory: ns-3-dce + run: | + ./utils/build_lkl.sh "$GITHUB_WORKSPACE/lkl-src" "$GITHUB_WORKSPACE/deps/lkl" + rm -rf "$GITHUB_WORKSPACE/lkl-src" "$GITHUB_WORKSPACE/deps/lkl/liblkl.so.debug" + - name: Add Quagga module run: cp -a deps/ns-3-dce-quagga ns-3-dce/myscripts/ - name: Configure DCE working-directory: ns-3-dce run: | + # Without --enable-kernel-stack, LKL is the Linux stack. + if [ "${{ matrix.stack }}" = libos ]; then + KERNEL_STACK="--enable-kernel-stack=$GITHUB_WORKSPACE/deps/kernel" + fi ./waf configure \ --with-ns3="$GITHUB_WORKSPACE/build" \ --prefix="$GITHUB_WORKSPACE/build" \ - --enable-kernel-stack="$GITHUB_WORKSPACE/deps/kernel" \ + ${KERNEL_STACK} \ + --with-lkl="$GITHUB_WORKSPACE/deps/lkl" \ --enable-opt - name: Build DCE @@ -140,6 +172,7 @@ jobs: working-directory: ns-3-dce run: | cp -a "$GITHUB_WORKSPACE/deps/bin_dce/." build/bin_dce/ + cp "$GITHUB_WORKSPACE/deps/lkl/liblkl.so" build/bin_dce/ ./utils/build_bash.sh build/bin_dce - name: Run DCE tests @@ -157,6 +190,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: testpy-output-${{ matrix.distro }} + name: testpy-output-${{ matrix.distro }}-${{ matrix.stack }} path: ns-3-dce/testpy-output if-no-files-found: ignore From 9b1744c00d050ecc6f9396795d53b2827ba84232 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Fri, 9 Oct 2026 10:07:15 +0200 Subject: [PATCH 09/71] Remove libos (Linux 4.4) and FreeBSD LKL (Linux 6.12) runs every kernel example and test the libos kernel did, so the libos kernel stack goes, and with it the FreeBSD stack, which was built on the same libos API (KernelSocketFdFactory and libos' sim.h headers). - configure: --enable-kernel-stack is gone; --with-lkl provides ns3::LinuxSocketFdFactory and defines LINUX_STACK (KERNEL_STACK, LKL_LINUX and LKL_STACK are gone). - LKL takes over the libos classes and files: LklSocketFdFactory and LklSocketFd become KernelSocketFdFactory and KernelSocketFd (model/kernel-socket-fd*), and the LKL LinuxSocketFdFactory moves to model/linux-socket-fd-factory.cc. - Removed: the libos KernelSocketFdFactory/KernelSocketFd and LinuxSocketFdFactory code, FreeBSDSocketFdFactory, Ipv4FreeBSD, FreeBSDStackHelper, the dce-freebsd and freebsd-iproute examples and the FreeBSD test cases, utils/freebsd-sim-dce.patch, utils/mptcp-build.sh (libos MPTCP kernel); the Python bindings drop their KERNEL_STACK switch. - utils/build_kernel_deps.sh builds only the programs (ip, iperf, ping, quagga, ...) and the quagga module; CI has one job per distribution. - Tests and examples drop their libos branches; the process manager test's stack masks are renamed (LINUX_STACK is the build's macro). - Documentation: the kernel guide, getting started and the subprojects page describe LKL; the FreeBSD page is removed; the kernel developer page is marked as describing the former kernel support. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-and-test.yml | 28 +- .../ns3modulegen_core_customizations.py | 14 +- bindings/python/wscript | 3 - doc/source/dce-kernel.rst | 6 + doc/source/dce-subprojects.rst | 20 +- doc/source/dce-user-basic.rst | 1 - doc/source/dce-user-freebsd.rst | 95 -- doc/source/dce-user-kernel.rst | 282 +--- doc/source/getting-started.rst | 30 +- doc/source/images/dce-freebsd.png | Bin 23309 -> 0 bytes example/dce-cradle-mptcp.cc | 9 +- example/dce-freebsd.cc | 158 -- example/dce-iperf-emulation.cc | 15 +- example/dce-iperf-mptcp.cc | 9 +- example/dce-iperf.cc | 15 +- example/dce-lkl-p2p.cc | 14 +- example/dce-mptcp-handoff-v4v6.cc | 5 +- example/dce-mptcp-handoff-v6.cc | 5 +- example/dce-mptcp-lte-wifi-v6.cc | 5 +- example/dce-mptcp-lte-wifi.cc | 5 +- example/examples-to-run.py | 1 - example/freebsd-iproute.cc | 81 -- helper/freebsd-stack-helper.cc | 153 -- helper/freebsd-stack-helper.h | 130 -- model/dce.cc | 2 +- model/freebsd-socket-fd-factory.cc | 174 --- model/freebsd-socket-fd-factory.h | 39 - model/freebsd/ipv4-freebsd.cc | 637 --------- model/freebsd/ipv4-freebsd.h | 373 ----- model/kernel-socket-fd-factory.cc | 1268 ++++++----------- model/kernel-socket-fd-factory.h | 191 +-- model/kernel-socket-fd.cc | 233 +-- model/kernel-socket-fd.h | 31 +- model/linux-socket-fd-factory.cc | 155 +- model/linux-socket-fd-factory.h | 47 +- model/lkl/lkl-linux-socket-fd-factory.cc | 18 - model/lkl/lkl-socket-fd-factory.cc | 636 --------- model/lkl/lkl-socket-fd-factory.h | 94 -- model/lkl/lkl-socket-fd.cc | 287 ---- model/lkl/lkl-socket-fd.h | 69 - myscripts/btorrent1/wscript | 2 +- myscripts/iperf-multi/wscript | 2 +- test/dce-cradle-test.cc | 6 - test/dce-manager-test.cc | 121 +- test/dce-mptcp-test.cc | 16 +- utils/build_kernel_deps.sh | 60 +- utils/freebsd-sim-dce.patch | 52 - utils/mptcp-build.sh | 26 - utils/ns-3-dce-quagga-ns3.patch | 10 +- wscript | 84 +- 50 files changed, 895 insertions(+), 4822 deletions(-) delete mode 100644 doc/source/dce-user-freebsd.rst delete mode 100644 doc/source/images/dce-freebsd.png delete mode 100644 example/dce-freebsd.cc delete mode 100644 example/freebsd-iproute.cc delete mode 100644 helper/freebsd-stack-helper.cc delete mode 100644 helper/freebsd-stack-helper.h delete mode 100644 model/freebsd-socket-fd-factory.cc delete mode 100644 model/freebsd-socket-fd-factory.h delete mode 100644 model/freebsd/ipv4-freebsd.cc delete mode 100644 model/freebsd/ipv4-freebsd.h delete mode 100644 model/lkl/lkl-linux-socket-fd-factory.cc delete mode 100644 model/lkl/lkl-socket-fd-factory.cc delete mode 100644 model/lkl/lkl-socket-fd-factory.h delete mode 100644 model/lkl/lkl-socket-fd.cc delete mode 100644 model/lkl/lkl-socket-fd.h delete mode 100644 utils/freebsd-sim-dce.patch delete mode 100644 utils/mptcp-build.sh diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 64d0a437..a2ef5660 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -17,25 +17,16 @@ env: jobs: build: - name: Build on ${{ matrix.distro }} (${{ matrix.stack }}) + name: Build on ${{ matrix.distro }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - # stack: the Linux kernel of ns3::LinuxSocketFdFactory - # lkl: Linux 6.12 (LKL) - # libos: Linux 4.4 (net-next-nuse), with FreeBSD; LKL is - # ns3::LklSocketFdFactory include: - distro: ubuntu-24.04 image: ubuntu:24.04 - stack: lkl - distro: debian-trixie image: debian:trixie - stack: lkl - - distro: ubuntu-24.04 - image: ubuntu:24.04 - stack: libos container: image: ${{ matrix.image }} steps: @@ -110,24 +101,20 @@ jobs: working-directory: ns-${{ env.NS3_VER }} run: ./ns3 install - # DCE builds of ip, iperf, thttpd, wget, ping and quagga, the - # ns-3-dce-quagga module and, for libos, the Linux 4.4 and FreeBSD - # kernel libraries. + # DCE builds of ip, iperf, thttpd, wget, ping and quagga, and the + # ns-3-dce-quagga module. - name: Cache kernel stack dependencies id: cache-kernel-deps uses: actions/cache@v4 with: path: | deps/bin_dce - deps/kernel deps/ns-3-dce-quagga - key: kernel-deps-${{ matrix.distro }}-${{ matrix.stack }}-${{ hashFiles('ns-3-dce/utils/build_kernel_deps.sh', 'ns-3-dce/utils/*.patch') }} + key: kernel-deps-${{ matrix.distro }}-${{ hashFiles('ns-3-dce/utils/build_kernel_deps.sh', 'ns-3-dce/utils/*.patch') }} - name: Build kernel stack dependencies if: steps.cache-kernel-deps.outputs.cache-hit != 'true' working-directory: ns-3-dce - env: - LIBOS: ${{ matrix.stack == 'libos' && '1' || '0' }} run: | ./utils/build_kernel_deps.sh "$GITHUB_WORKSPACE/deps" rm -rf "$GITHUB_WORKSPACE/deps/src" @@ -153,14 +140,9 @@ jobs: - name: Configure DCE working-directory: ns-3-dce run: | - # Without --enable-kernel-stack, LKL is the Linux stack. - if [ "${{ matrix.stack }}" = libos ]; then - KERNEL_STACK="--enable-kernel-stack=$GITHUB_WORKSPACE/deps/kernel" - fi ./waf configure \ --with-ns3="$GITHUB_WORKSPACE/build" \ --prefix="$GITHUB_WORKSPACE/build" \ - ${KERNEL_STACK} \ --with-lkl="$GITHUB_WORKSPACE/deps/lkl" \ --enable-opt @@ -190,6 +172,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: testpy-output-${{ matrix.distro }}-${{ matrix.stack }} + name: testpy-output-${{ matrix.distro }} path: ns-3-dce/testpy-output if-no-files-found: ignore diff --git a/bindings/python/ns3modulegen_core_customizations.py b/bindings/python/ns3modulegen_core_customizations.py index 04c59ad9..9a122df3 100644 --- a/bindings/python/ns3modulegen_core_customizations.py +++ b/bindings/python/ns3modulegen_core_customizations.py @@ -449,18 +449,16 @@ def add_ipv4_address_tp_hash(module): def post_register_types(root_module): remove_classes = [ - # These are classes which get introduced when KERNEL_STACK is enabled , so should be removed for non-linux DCE builds + # The Linux stack (LKL) is not part of the Python bindings 'KernelSocketFdFactory', 'LinuxSocketImpl', 'KernelSocketFd', 'LinuxSocketFdFactory', - 'FreeBSDSocketFdFactory' ] - if "KERNEL_STACK" not in os.environ: - for classname in remove_classes: - try: - root_module.classes.remove(root_module['ns3::%s' % classname]) - except: - pass + for classname in remove_classes: + try: + root_module.classes.remove(root_module['ns3::%s' % classname]) + except: + pass diff --git a/bindings/python/wscript b/bindings/python/wscript index cda0936b..e74c09f7 100644 --- a/bindings/python/wscript +++ b/bindings/python/wscript @@ -584,9 +584,6 @@ def build(bld): if not env['ENABLE_PYTHON_BINDINGS']: return - if bld.env['KERNEL_STACK']: - os.environ['KERNEL_STACK'] = bld.env['KERNEL_STACK'] - dce_generate_python_bindings(bld) diff --git a/doc/source/dce-kernel.rst b/doc/source/dce-kernel.rst index 14fe220f..169e4956 100644 --- a/doc/source/dce-kernel.rst +++ b/doc/source/dce-kernel.rst @@ -2,6 +2,12 @@ Kernel Developer Information **************************** +.. note:: + + This page describes the former kernel support, net-next-sim and libos + (Linux up to 4.4), which DCE no longer uses: DCE now runs the Linux + Kernel Library (LKL); see :doc:`dce-user-kernel`. + This technical documentation is intended for developers who want to build a Linux kernel in order to use with DCE. A first part will describe the architecture and the second will show how we went from a net-next kernel 2.6 has a Linux kernel-stable 3.4.5. diff --git a/doc/source/dce-subprojects.rst b/doc/source/dce-subprojects.rst index 276d0ec5..5ef18317 100644 --- a/doc/source/dce-subprojects.rst +++ b/doc/source/dce-subprojects.rst @@ -44,19 +44,15 @@ The `umip `_ (Usagi-Patched Mobile IPv6 stack) support on DCE e For more information, see the latest support `document `_. -Linux kernel (from 2.6.36 to 3.14 version) -============================================ -Linux kernel support is built with a separate 'dce-linux' module, -available on `github -`_. Many +Linux kernel (6.12) +=================== +Linux kernel support uses the `Linux Kernel Library (LKL) +`_, built by ``utils/build_lkl.sh``. Many protocols implemented in kernel space such as TCP, IPv4/IPv6, Mobile -IPv6, Multipath-TCP, SCTP, DCCP, etc, are available with ns-3. - -FreeBSD kernel (10.0.0 version) -========================================= -`FreeBSD kernel support `_ is based on Linux kernel module of DCE. A few -protocols implemented in kernel space such as TCP, IPv4, etc, are -available with ns-3. +IPv6, Multipath TCP, SCTP, DCCP, etc, are available with ns-3. It +replaces the former net-next-sim/libos (Linux up to 4.4) kernel modules; +the FreeBSD kernel support, which was built on them, was removed with +them. thttpd diff --git a/doc/source/dce-user-basic.rst b/doc/source/dce-user-basic.rst index d8b282e0..0b91c660 100644 --- a/doc/source/dce-user-basic.rst +++ b/doc/source/dce-user-basic.rst @@ -13,5 +13,4 @@ Basic Use Cases dce-user-config dce-cradle dce-user-aspect-trace - dce-user-freebsd diff --git a/doc/source/dce-user-freebsd.rst b/doc/source/dce-user-freebsd.rst deleted file mode 100644 index 5290c4a7..00000000 --- a/doc/source/dce-user-freebsd.rst +++ /dev/null @@ -1,95 +0,0 @@ -.. include:: replace.txt - -FreeBSD kernel support with DCE -=============================== - -Overview --------- -This module provides an additional network stack support for DCE with FreeBSD kernel. - -.. u POSIX POSIX POSIX ns-3 -.. k ns-3 Linux FreeBSD Linux - -.. image:: images/dce-freebsd.png - :scale: 70% - :align: center - - -* This is a POSIX userspace application support over FreeBSD kernel over DCE, over ns-3 over Linux. -* This is **NOT** the DCE running on FreeBSD operating system (right now DCE only run on Linux). - -Usage ------ - -:: - - bake.py configure -e dce-freebsd-dev - bake.py download - bake.py build - -Once you finished to build dce-freebsd module, you can write a -simulation script that uses FreeBSD kernel as a network stack. All -you need is to specify the library name with an attribute **Library** -to **ns3::FreeBSDSocketFdFactory**, then install -**FreeBSDStackHelper** to the nodes. - -:: - - DceManagerHelper processManager; - processManager.SetNetworkStack ("ns3::FreeBSDSocketFdFactory", - "Library", StringValue ("libfreebsd.so")); - processManager.Install (nodes); - FreeBSDStackHelper stack; - stack.Install (nodes); - - -How to use your kernel extension with DCE ? -------------------------------------------- - -No configuration support (like *make menuconfig* in Linux) right -now. You need to add files into sys/sim/Makefile. - -The following represents an example to add Multipath-TCP feature of -FreeBSD by adding **mptcp_subr.o** to the -freebsd-sim/sys/sim/Makefile. If you want to add your code into the -kernel build, you may add object file names that is from your own -codes. - -:: - - diff --git a/sys/sim/Makefile b/sys/sim/Makefile - index 8115e3d..1b2feab 100644 - --- a/sys/sim/Makefile - +++ b/sys/sim/Makefile - @@ -100,7 +100,7 @@ ip_divert.o tcp_hostcache.o ip_ecn.o tcp_input.o ip_encap.o - \ - tcp_lro.o ip_fastfwd.o tcp_offload.o ip_gre.o tcp_output.o \ - ip_icmp.o tcp_reass.o ip_id.o tcp_sack.o ip_input.o tcp_subr.o \ - tcp_syncache.o ip_mroute.o tcp_timer.o ip_options.o tcp_timewait.o \ - -ip_output.o tcp_usrreq.o raw_ip.o udp_usrreq.o if_llatbl.o - +ip_output.o tcp_usrreq.o raw_ip.o udp_usrreq.o if_llatbl.o mptcp_subr.o - - -Limitations ------------ -While this release gives a proof of concept to use FreeBSD kernel with -DCE, there are tons of limitations (listed below) that have to be -improved in near future. - -* No sysctl -* No IPv6 address configuration -* No ifconfig - * No delete IP addresses support -* No route command -* No quagga, new FreeBSD-version glue code needed -* No extensive test -* No DCE Cradle -* No Poll implementation -* No getifaddr/routing socket implementations -* Socket options missing (incosistent defined value: SOL_SOCKET(Linux/FreeBSD)=20/0xffff) - -TODO ----- -* refactoring with LinuxStackHelper/FreeBSDStackHelper -* refactoring with wscript (--enable-kernel-stack uses both FreeBSD/Linux for now) - diff --git a/doc/source/dce-user-kernel.rst b/doc/source/dce-user-kernel.rst index 5ea9a0fc..ebd1daa1 100644 --- a/doc/source/dce-user-kernel.rst +++ b/doc/source/dce-user-kernel.rst @@ -1,237 +1,87 @@ .. include:: replace.txt -.. _net-next-sim: https://github.com/direct-code-execution/net-next-sim -.. _mptcp: https://github.com/multipath-tcp/mptcp -.. _mptcp_git: git://github.com/multipath-tcp/mptcp -.. _libos: https://github.com/libos-nuse/net-next-nuse .. _lkl: https://github.com/lkl/linux -.. _linux_vanilla_git: git@github.com:torvalds/linux.git Using your in-kernel protocol implementation ============================================ -There are a number of protocols implemented in kernel space, like many transport protocols (e.g., TCP, UDP, SCTP), Layer-3 forwarding plane (IPv4, v6 with related protocols ARP, Neighbor Discovery, etc). -DCE can simulate these protocols with |ns3| and a kernel compiled as a library. This is not possible with the linux vanilla kernel hence you need to use a slightly modified kernel (net-next-sim_ - deprecated - or libos_ which is the successor of net-next-sim_). +There are a number of protocols implemented in kernel space, like many +transport protocols (e.g., TCP, UDP, SCTP, DCCP, MPTCP) and the Layer-3 +forwarding plane (IPv4, IPv6 with related protocols ARP, Neighbor +Discovery, etc). DCE simulates these protocols with |ns3| and the Linux +kernel built as a library: the Linux Kernel Library, lkl_ (Linux 6.12). -This document describes how to retrieve, configure, compile and use this custom kernel in DCE. -As an example, it shows how to enable an in-tree but optional protocol - Stream Control Transmission Protocol (SCTP) - and an out of tree protocol: Multipath TCP (MPTCP). -Although other protocols may not adapt these patterns as-is, you will see what's needed to implement for your purpose. +Each node gets its own kernel; every ns-3 NetDevice of the node becomes a +kernel network device named ``sim``, and the sockets of DCE +applications are kernel sockets. + +This document describes how to build the kernel library, configure it, +and use it in DCE, with SCTP and Multipath TCP (MPTCP) as examples. .. contents:: :local: .. _configure: -1. Configure a kernel (make menuconfig in Linux) ------------------------------------------------------------ -In Linux kernel, there is a configuration system in Linux kernel to enable/disable features. This is typically done by *make menuconfig* command, and it writes a file (.config) at the kernel source directory. Build system (e.g., make bzImage) refers the file which source files are compiled. - -.. figure:: images/dce-make-menuconfig.png - :scale: 70 % - -In our DCE Linux kernel module (i.e. **net-next-sim** available at github.com), we have *arch/sim/defconfig* file to store the default configuration of kernel features. You may need to add proper configuration parameters (e.g., CONFIG_IP_SCTP) to build the protocol by default. - -For the Linux SCTP implementation, we need at least the following configuration parameters. - - - CONFIG_IP_SCTP=y - - CONFIG_SCTP_DEFAULT_COOKIE_HMAC_NONE=y - - CONFIG_CRYPTO_CRC32C=y - - CONFIG_CRC32=y - -These can be added to the file (*arch/sim/defconfig*) or added manually to *.config* when needed. - -**Note** - -These configuration options **SHOULD** be minimized at the beginning since adding new option may require new functions which DCE doesn't support at the time and need to implement glue code. - +1. Configure the kernel +----------------------- +``utils/build_lkl.sh`` configures the kernel with LKL's defconfig plus +``utils/lkl-dce.config``, which enables what the DCE examples and tests +use: SCTP, DCCP, MPTCP, AF_PACKET, IPsec, Mobile IPv6, tunnels, multicast +routing and more TCP congestion controls. To enable another feature, add +its options to ``utils/lkl-dce.config``; ``build_lkl.sh`` reconfigures the +kernel when the file changes. -2. Build a DCE compatible linux kernel ---------------------------------------------- +For instance, the Linux SCTP implementation needs ``CONFIG_IP_SCTP=y``. -DCE can simulate these protocols with |ns3| and a kernel compiled as a library named *liblinux.so*. This is not possible with the linux vanilla kernel hence you need to use a slightly modified kernel (net-next-sim_ - deprecated - or libos_ which is the successor of net-next-sim_). -The following section presents the two methods, net-next-sim_ being slightly easier to install. - -2.1 With *net-next-sim* (deprecated) ----------------------------------------------------------------------- -To build the liblinux.so, DCE version of Linux kernel, - -:: - - make defconfig ARCH=sim - make library ARCH=sim - -You can use *make menuconfig* command (below) instead of editing the defconfig file. -If everything is fine, you will see **liblinux.so** linked to **libsim-linuxv.y.z.so** file at the root directory of Linux kernel. +2. Build the kernel library +--------------------------- :: - make menuconfig ARCH=sim - - -2.2 Build an MPTCP kernel with *net-next-sim* --------------------------------------------------------------------- - -DISCLAIMER: This is a transcript of `Hajime's guide https://plus.google.com/+HajimeTazaki/posts/1QUmR3n3vNA` updated on a best effort basis. Hence it may be possible to compile newer versions, in which case patches are welcome. -Build steps for DCE integration is also available as a script in *utils/mptcp-build.sh*. - -#) Get linux kernel mptcp_ variant: - - .. code:: shell - - $ git clone mptcp_git_ - $ git checkout -b sim3.14 mptcp_v0.89 - -.. 3.14 can work with mptcp 0.89.8. - -#) Merge in mptcp code the changes required to make it work in DCE. Those changes are listed in the net-next-sim_ project - - .. code:: shell - - % cd mptcp - % git remote add dce git://github.com/direct-code-execution/net-next-sim.git - % git fetch dce - % git merge dce/sim-ns3-3.14.0-branch - -#) Enable mptcp in the kernel configuration. There are 2 ways possible: - - - patch the kernel config manually (you can - - .. code:: shell - - % cat >> arch/sim/defconfig <`_ and `sctp-server.cc `_. +---------------------------------------------------- -**Optional** -You may optinally need external libraries to build/run the applications. In this case, the applications need *lksctp-tools*, so that applications fully benefit the features of SCTP, rather than only using standard POSIX socket API. +Then, we need userspace applications using the new feature of the kernel +protocol. In case of SCTP, see ``example/sctp-client.cc`` and +``example/sctp-server.cc``. These need *lksctp-tools* (``libsctp-dev``) +to build. -Moreover, adding system dependency to bake configuration file (i.e., bakeconf.xml) would be nice to assist build procedure. -The following is an example of *lksctp-tools*, which above applications use. - -:: +4. Write ns-3 scenario to use above applications +------------------------------------------------ - - - - - - - - - - - - -4. Write ns-3 scenario to use above applications. ---------------------------------------------------- -The next step would be writing |ns3| simulation scenario to use the applications you prepared. -You need first to add to your *DCE_PATH* the path to the previously compiled *liblinux.so*: +``example/dce-sctp-simple.cc`` is the script that we prepared. The kernel +is selected with ``ns3::LinuxSocketFdFactory`` (its *Library* attribute +defaults to ``liblkl.so``; ``liblinux.so``, the name of the former libos +kernel, selects it too): :: - $ export DCE_PATH="$HOME/net-next-sim:$DCE_PATH" -`dce-sctp-simple.cc `_ is the script that we prepared. In the script, you may need to load the applications by using *DceApplicationHelper* as follows. - -:: + DceManagerHelper dceManager; + dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory"); + dceManager.Install (nodes); DceApplicationHelper process; ApplicationContainer apps; - /* by default DCE, loads liblinux.so. If for some reason your library has a different name, - either create a symlink or use these commands to change the name */ - DceManagerHelper dceManager; - dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); - process.SetBinary ("sctp-server"); process.ResetArguments (); process.SetStackSize (1<<16); @@ -244,19 +94,39 @@ You need first to add to your *DCE_PATH* the path to the previously compiled *li apps = process.Install (nodes.Get (1)); apps.Start (Seconds (1.5)); -5. run it ! -------------- +Network configuration is done as on Linux, with ``ip`` (iproute2 6.12, +built by ``utils/build_kernel_deps.sh``) through +``LinuxStackHelper::RunIp``, and with sysctls through +``LinuxStackHelper::SysctlSet`` and ``SysctlGet``: ``".net.ipv4.tcp_sack"`` +is ``/proc/sys/net/ipv4/tcp_sack``; ``SysctlGet`` also reads any kernel +file given by its absolute path, e.g. ``/proc/net/netstat``. + +Multipath TCP +~~~~~~~~~~~~~ + +Linux uses MPTCP only for sockets created with ``IPPROTO_MPTCP``. The +*Mptcp* attribute makes the TCP sockets of all applications MPTCP sockets, +like ``mptcpize`` does on Linux; the subflows are configured with +``ip mptcp``: :: - ./waf --run dce-simple-sctp + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); + ... + LinuxStackHelper::RunIp (client, Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (server, Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + // a subflow from the client's second address + LinuxStackHelper::RunIp (client, Seconds (0.5), "mptcp endpoint add 10.1.1.1 dev sim1 subflow"); -If you're lucky, it's done. +See ``example/dce-iperf-mptcp.cc`` and ``test/dce-mptcp-test.cc``. -If you aren't lucky, you may face errors of DCE, such as unresolved symbols in system calls (called by userspace applications) or missing kernel functions (used by newly added CONFIG_IP_SCTP option), or invalid memory access causing segmentation fault. In that case, adding missing functions, so called *glue-code* would be the next step. +5. Run it +--------- - +:: + ./waf --run dce-sctp-simple -.. 6. Write Wrapper classes for DCE Cradle -.. 7. Test +If you face errors, such as unresolved symbols in system calls (called by +userspace applications) or invalid memory access, adding the missing +functions to DCE, so called *glue-code*, is the next step. diff --git a/doc/source/getting-started.rst b/doc/source/getting-started.rst index 73b61017..312f10cb 100644 --- a/doc/source/getting-started.rst +++ b/doc/source/getting-started.rst @@ -200,7 +200,8 @@ In this case you need to install the single packages one by one. You may want to More detailed information on installation requirements can be found on the `ns-3 wiki `_. -Then you can download and install *net-next-nuse and DCE (*net-next-nuse* includes the linux stack module): +Then you can download and install DCE and the Linux kernel library (LKL), +which provides the Linux network stack: * GIT_DCE= |git_dce| @@ -208,23 +209,24 @@ Then you can download and install *net-next-nuse and DCE (*net-next-nuse* includ :caption: Kernel and DCE installation :linenos: - # Clone net-next-nuse - git clone https://github.com/libos-nuse/net-next-nuse.git - cd net-next-nuse - # Select a kernel version - git checkout libos-v4.4 - # Configure and build - make defconfig OPT=yes ARCH=sim - make library OPT=yes ARCH=sim - cd .. - - # Download, configure, build and install DCE - git clone GIT_DCE + # Download DCE + git clone GIT_DCE ns-3-dce + cd ns-3-dce + + # Build the Linux kernel library (Linux 6.12) into ../lkl/dce + ./utils/build_lkl.sh ../lkl ../lkl/dce + + # Configure, build and install DCE ./waf configure --with-ns3=$HOME/dce/build --enable-opt \ - --enable-kernel-stack=$HOME/dce/net-next-nuse/arch \ + --with-lkl=../lkl/dce \ --prefix=$HOME/dce/install ./waf build ./waf install + cp ../lkl/dce/liblkl.so build/bin_dce/ + +``utils/build_kernel_deps.sh`` builds the programs the kernel examples +run (ip, iperf, ping, quagga, ...); copy its ``bin_dce`` directory to +``build/bin_dce`` too. Examples diff --git a/doc/source/images/dce-freebsd.png b/doc/source/images/dce-freebsd.png deleted file mode 100644 index f9d18ba1e623923a9ea0552b5c53c7e04ad0a2ef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23309 zcmeFYWl$Ya+pY0LpbwQ{0pQ)sD!UPL{I>^ zv+zezG-N6jBc(2Iv0!3D5wy?#KMWROE|KI1x)p>)_SPaSVb01~uh!FE2Guu~4;mMj zP0zd*TfpR>e?ay)se$`~a@5fN9(QY@ppN$4L0}?*BdCFQZJC%rirouC%ssdBq^1^& z57gfNrS~;zU8U`mCwE+U^NA&97b5)x1}iep#_dgxu?LZ$Slf@q{02F4ok+zT#KerA z1Cv`uywg`{bz8Jg(>k1zOun0gtv-R(2Fi+Pm0wRI+scc8%+X3A+ceGdgfebdl1KyF zyQ7?)z0Z&aV?;~U9XlY;u?l4tLBskRdI~{D@r7r|{Ym+P{A4mqARP95>P|7JYYg3A4qNZ+#$Si8#$2W(qiRp1Xy)Lpu|8q{e%Vi(KABO>cQjsveB2ivEIuB;$OFVTy`0#y*<~5{uu%j{<}F>OwV&GCpS*Hsth3x;_=X zXg$rUHT78du;kU>yT-&JE(awG#b~~OM zcoKEM`r!KDZ$uLebW+5?Xh2&2KqN{cB()+{Lc)eVfPM);?g1(I70Wph2O+ujX&F#e zM=c4oN#K&p#-m9Dkz#x!rAn>yQN*;JCmzymF7BcZB&U5)Hc+fiNuK} ziQtLRzoeCI=h5afe_8*Mn`5*fx6rp(ws@MSFO8fmnsmvU%+v+F?LG$|Rkeh=H$7V1 z^d9jX#bc*ocVJ^-D`O{v23WDuY|@6)_Rm$I)xD3p8pqD!K^DNinD~=vFG66A9lgJqe)&Bnkf+{ z5jiZQxHPw}q}MW8HwkCCZ$=I(Wo;eIt@^F5UM9Nmle3q5$TQC^{ek8)Pq zF6}9;BHb}9fjX7?M>XrZqpHW=mA*7fZ9}iYAd4Nt+KIxo{rd6}x((;fQ?IXS+Er`l zt+K6?w+OeyPvlQbP&~n|!R-ilOHR&|tpQ&1$j5)SnkFtHcAzLcD5_^AiZ(LUC#WXi zC-OeSVVq;k(P27F+ix}DI>gc&(MRaI+U3kCuO{wj96yf9EJhwU_gVL?E%7YBG(FKE zN+HIIg%2ha#UEf#e%1*|u2hXxt!#|DGdQz2E8^wi4dspGMeT6zDD`5v*MEU{aeiF8 zT0B2^Tzry)-$lh-7?$#d!3^Tfb1L~5gHMMg*@m4IA1(SogGT_INZw+27O9SqPML|$m$8$ZWFCI{36ObaLXX@?q0 zR7iZ0*nyA}sNeivqkHMMTaplykVnN*GF>um9#D5*H!Db{A67-9&B8;`&d>|WV-#u` zF`xgdjGD_o9V<=gsBY4l67?#5BOXDTPUf%vxz?g)-BiptdqiOjdH;GhZP#WLZ}@8X z$(z-L)#bZshx2RDb2sZ>ON8>kg$k|JW?2{Q?v|~aj^k)+LTmBm^HI^!NMoE0o3Qmc zg`Wyl9eQT9XZr(!&4LPB?Q|2G2v+@8U#w2oOlr$5R+EW!rf&>Jx5KvO$bQEiZ>W2z zKCF%12AsAR+!j2!Y&t``{9Ze3xl&6tP}fK*$*+n3E7 zArI8jLwT0G=6gsEBF>BL!Fz(UEI96Ew=l0?hZHZ_w+&MoVH!CLzZXIq^4ZKiq&+qc zeP=WACh+L^(%(FR#a82 zkNPE7>RdECl&rQtJu4@dnpSXXfAz}rYCjEniCn;M%}UZe@isrKKeKK>w5xqd+9{9f z_~>=;sQtvSe%s=;7xk8j^CA4x<6rs?k2js)JcnNEFVj#Zh&p^@?i?>>FNP;OxfHN+ z)LFmy%3tR17fnjquTOq1>P^;$m_`RR2{F7SK2*x}ew-|sq!FzXJrXMyn~ZpfSaYww z?mr)mNS@sZ?4){axJ#Ta&GvSAYBx0)?mM@fK2GJ6Trck6d|bG-J}~;TJL_V8R#+6a zUfRKT>-UoLWOzLodw3#uFPE4d$>-t;^L*(_Zf&x^aTjp)#LT7c4aAf%cH-)eU|^q7 z-+#fu(lRiCaC6>VNzF-3Mw-je)|y`5$kxD^-qqR;C=CY2?aBrGX>IJJPv~lGW#h=@ z%0v9G1Q+o4`_~M_g#U^-S@IC8$;cB5**X{#veJX-8Hsse2?+_g9gIx4z6p!|_j2G9 z53!k(lN}cWgNus`y$cJyt%E586DKDp10yp7Gcz4fg3i&+#!26m&c>1CKb8EidW4M~ z4IRwwoXl-)2;b|~H?aNg#6wK{exv{S`OkAYnVbCYJJ~q?w^={~8Q%ZHz(mi;@IQ3} zmvX;<%OzxMZRcR@=m^x$%gX()d)6@BUk!o8i6T|3^psrhzM z)eiF-7#Kg8gs^~;EBI;V#}wTD+uQK)07!AdCP){T;7uF9P4tu8?lnC6zq04yQZahn zME4^8kS@vmD1=?7XU`LFS&tU(9?qt0t?8-Dj#))TrK2rJr&*11nTpyxvg9aQxZq-4 z$aat;#OKj9R9TzQ91u~yBtM}!6doi0ec=BL9kD3r-apLyqY{&P@()k|FQAKKCt{Y| z)FPHL9C~WoYJ;5aLh@flW@3<+`KkCtTe;*3WFy=#uXmkq@K(^~4eA9C&**0!|L!$_ z4j}@0AMNu@f|PBu(2@I`pGYQ&5(yVcdgXDMzdKwVPV#>1`XI2~5KM_0IdEwsAIKuu*Iiq3uCZeGX=q6$nXx2ZZx(eW z6Y|8O7i#)i|20eg4EiVM>%&3z$oFgG-}Zy^TTRF&E1l%Rsf*-9lGINxcN?<~_|A|( zQ-1zM_FW`T(StaJ>Kp!~zh3~O5JxosUdl0dOz2<`xHC{&1U9Q=E0{o9$gzmC5i-{p=knCam6~rwwf>L0fxI~^w2~WQPl;9^J zI$yQz6fs2*vO^PYVqlQb(Fd`yM$T9#xD7oo0b_&x5j=rkbbP!Z#{ zXhwT!G-}7$G&vr2<9iF!Z7il`Ze4!ci*Y=k_x84)b!f)19p=Xez1*%coB$tlJnsII znNwuB8Z=!D;IKyBYCBDfNY3C`RryWi6Y6(=#E5V!W3+@V>v%lc8YJ+UqkMEZ$aLW% z!r|{A_zFGL)F#}goHcC8vy9019OLzPShS2|+ZM}n(TBQh-MBX3!9(Wu^792*&AcEH z&85oLCf(z2b1g&XbB%;QedqI~RSyh>RRiq}te-KnYJ-FuQ8)VN1k)gHDw3weGSA&_ zmLyG=weN;2>mFx#JDg`f1^f-L{S;+63rJsL#qg;l?mme9EJxaW^uIZhVT2~j7WX`2Sjd0 ztLlDq{T1_~$cM9zr@y}FDI-FNJZQiOWnhvS;W}@4-Pv`4Lj`*uP+3L2AHfS)B0P}a7D!L9jqziv zJu)m31gXg)-LA&jS+JOs&GZVDM{uB)4Z`WrXZx~JhLJk*+m^7E;}crtt|EbDv`kak zhH?dcF>q;F+3pN9DDToTbn!1#v$9$?LtE;ARnnwE_!Wq*K?zJ_d;#&!{IP>Tj9gCOq)h z-VvLOE}l<6h@7%9&1U<4>J!2DIRz%0!-8a+22|55Hb$$;e2Y6g}2#3a`=UpR|p?9hmUHl44_E zlV;g)PXkj?Te_0XcfjTOVsOx)^YyVbD7C_Am@K*PUJ`=+x1G@S(A}SD61~;mz=ZjM zlz=Ywgi4{Hue_uu!YPl~7Gpg|vRit)gc%DtHJ2*T)tsd5yd2~0bk_5k0dC1z2I1>T z6(J%fPv4Ud&`-Y>CagX{p7A?32f4wi^EuUM2C10&R7W19J0rQ!AQ zIN7z%n6jJ&-R$D|YLcKH^OB8X%qC_PWaA2k{P!dGg1H9v06i6HhPLNLKNj1xlEw^W zxi`^z+@l=VVSZg&SdYQCknhl;{mj5dVizGM1d?EtR4-!%wu=K=D%7|r<3FV_S%n4I zWEbgHH4-N*4a;b@M3ZXSW6F^?JolUaO!$!+_!cdG;0;pAQ6j#_Y9*rT+!wMIb|pNJ zpAhkJmqtR747y$e^=v^@?431QEs4hJo}dMqiYDSAP#&R>0!^+*bNPHjh}Xq}rmeEf zuRp!gWPx!XK#SQvubiPU0^+5+O>rMm4e~3qnHZeHs0yK%Age30yl$7PIW&x885IfV zoBDImqByog2xoUdUs<417E>|mbb7yNQtufZDbgA~C~~J@5ki(a&wWu*6EOX(#=Eh| z2Eofy+)ybeQf`Pzf&Zs!R`zTEGY5jSut&At7OzaO)eI4(Vwmt(xXO-43#4dwy_C@l zX_j%NZ;N@+yx6k9-V@i9gkUGkG$*&0X3^UA1QQ7(GEmWUl%g0^GZqiX;>Qi1rETV2Hh-Z%Gq}}P)?K+ zB4BvoloW7{?PTwxLP>Gc5?3dU$S;j-oeq4pVH;MfhSYICBqO^^ke+oyTttA7O{jOI z)3Abx!Y-U&3R>vfwXXZw#`t|mc-@97mD=dyL_J!rZJlXkWNUMCa=ea!?RBoiZo)W8 zU9_lOjXPgE^n8L-NM|B)?CV+E7H}r3{@B4BPb}e0gcfqikRq3U5pc|FXG3eK%jG$% zI0$S=8RNbhLyz!qoalpw$v1YL!o2WD^)kjrQj9etPz*cKoSCzKxmjfIW-njuH6#G1 z7^(JwUn%(L^>hp0CSlmqy7{0_Zv>sX#4YqX=W338Vhg8F#K&g8q^3;vq=d7#b}A{1 z`;Dy64mhF?afS97((HJxvgu&lq&@OFq41pKJ5mp97;h#(Z{LPEeld@3#+6t)h^ zN*JUcE{!ISag-8lCW+=|Hp$~bg6ZjVBp=8ln=!#9mF{OMlAuk1#576gcbJO~1%pSN z0p)ZMce@E^H>^2pW|H?o5Rse3=YN6eH4e!vQBJ`%YlKmZM^aJE#dH3vD958bI`pid z-5n}%Np3bz&l=&&9VyjU1eMsuw17BsDDa?)Nn`#SI2+PK3o(m|hsxD3b%0QX#a=K$ ziOT0(+R7*J;p*nR#gKZh5Q?c_cx0sYI5kF7gZArA1d9RJgHf?YTo&IZtF(?w9oL@F zZK{k;w);_u;YOgmxc*f9 zJ|)bP1d$(CP|+S_@qKRnh?Nb)Il=fVeI65*UeZ?VTZtpY=|2muu=mH=H*hgl_&snq z9rE2PsufrA9mn8YGp>jcQ8IsK-DV#0nvKxnek0Al3M(GVq?HiO8|Y55=#u`G+?+!e zo{|z2$`-K@5s6Z!ZGquuzj;>YhPjj~Hdy706D6YI=HYvY@Ef}M#9RS!kpqH5Ld|#x z{cnF^a4$JtN76Bb@CSXSwgUX=g*sZ{1Qem>%Tv6`mk~xhTTa_iX|`n}V>uXb!3t>7 z$q2vcJTa)Ydk+0Bk-Cn!%S{s&7|b{vH-9LE91(Q&{AEYx==dA%)Uf_%1$;)@j4?@h zdP691d|B?c=1*fzn>nK&;+0$M8VVfs)as$l;9K<*oCG97(~Mk&k@nEU(`SALR_4U__V&gfU|A*&76HRsv~b%q zoXF=nE(dLY*Dc?xfB?!+cU_jf2k)H7cDW}Ri)$XIfYMMV1a`xZYFYDu9O0q>_MptG z7LQ6iKor12rXdxyJ6d??sMnQ;x$~6%({8NM{3F(OSr=KSA(CVLo1-wHLZ^ zpZwD;c_i{ll#$wz_E&JLf4W@X-sQywvbbAC^LkRI9fAO{Xo5;8Y9Z0qyJ7@dw#lB< z?!Opu940yyC~R(IW@MfS7S=!im4E$pMTD?@!=f>9y~bV4_i3zY;<2Z?9K*u?BpuKF zD^;Pq+dl~f4^A?oR9e?mR~aY_Bk(PzoyjVeD@|or|DRAgf(Lh5NW5DPd2#hc09R5g++} zpxpx|ruoa`P51@A9VD*b0*T%mtA#J^70DmLT+kgyAo4AG5i)2xh? zd+SyZ&Jjcf80S_1>VRD7Wuj!qKD%C*ZwkSWRbavQuy1(g=~C~~czfoDS8Ct`T44(a zrbFGX`~XG5>Aigo1ea06o;ryP#Da5|j)jc%R%nwe4bO3Gk1;Vxt)@G$lU_;hsVWnfM@9=5P~gmmxaI| z(gO$lW3%p&Nipxs^{nONvT1htQ!W87dl!2S^2${os$4*hP;ijX+tUWN7KGuFl68GvzI(}g-OQ_=demT7;?V{n65ziA!DC!fF6J{37C%~F z=U&H+u{!Jh9(b$D7)&Q;Cz7Nw^DtZ@PAaeFZ8UdmqLRj>_YX$iuW0>iAz|Nn-t#$_ zELrPYG!XpYtxdA6TD5eZ)QrB19aJ5+y)R@fk?iXn+~-{%wg5pHi9w-~09loYt)I92 zE)LV&0lAytnZZM~-OVQKd+cBCo)d^ZW9!X=vSgBOoq)yx-{%IflVD&zx<^d91JbqF zKYCiIe5Cmxs}%7Wh>{f2p#?7O8-~f#rOy;kUOWR%f$=P#rw5s1><*cPXHCAHMHH*@6K+0CANhnORe1g_fUAL+ozwUOv=*KwD&bI7s zuio%}G9ORTz6Ba=H;CsF(VhBQNvW{{4A}Y!v|)rCbI_NvsvcNyK`|=+PYAfhCKOgz z+|S8C#AP=^Rn&LQ#H}5S7Q7L$W2Nh~T_dcX&^*1PqG?^<6||ry+yG2~Uy=6Lf2OOS zP^B3K;EKS*8{5KZYs6iF5U&Ri6G(+=7S!Wmiy8V{>()mV!glK^{7d87@}mb&OhRBD z%)P_L$=WKWeAlO|>^~(0;>=Q0lhdWNAA)>#yH?ZJz;lwVHoX&@8DoyS@ zAzcXD?-=GNeg)LjfK9PtAh8TcAd3BhNGpfg*G^0P1qIr*JW-6tkURcl7#MjDo z=;QK6op>XBYSH{$5wpj?t!G)o@n1avG$467ipmpTBBFsI9Epa!Hxx_4V&hI<4EIz6 z&$h(hyNx_qoTm`@y6}Vh^?oPSEJ4z{E>x!DtUO=*$#n>sLZgsP6;vS|beL+UhNX`c zXYp84l$92V$s~B33IGj~WxXVCV4WKk$u!>YFZ#g7P^kQyiG5 z9coo|y;%J>d7saPtb^BB>cIV)h8V~*PWav)qJxZIE7ygXx-A=lDJw2G*lWp*+|l3>BoUf#Fw%JN*Zq#G3wLnALFs1n{a&Z z^r?%fr%jwZ!qBv*N(M7FotgXX`HHK};C`mwQZ+J}A{jx7gVOYjG?I!R&d}~iPkgY% z2>=9#Xe5P1OfN$PFGyY7Jz8pGWI{o33Z~dXTGJgrxa-m~QuNQm89FclsR~Wkd%hZA zIT53Q-|*N2b}^PXKI)IiTdcsGt4Q&ocX7_G2^f)JDf(tofbob0@1@u_@d5E*e2gr+ z;n7lA?ZmI|`Vjn>%%4*&Ql4U|%t`=xkHM6=#nscM&$bJwi3Kk*zQo(cxE2ed=vB(n zCfMKk`{LayWGjIyzUM(hj#fkP*;F<3pg;x;R+3u;0C=Lac<2wsj6Z5bcj01W%XtdC1Xl7#xXq^|Re z3j0DWr-F(-Oo|{OTW{C1Xd#*%%Vok|$s{VS;ak+#znGT4&GI7W{=iE(2!-K^u_C;h znL%)XyW%l1FcT|4{V*>!0Z|b6AXVd(`11$dl8N4X@_7Gthny<5%kbktfXA2SuyuX# zJ%%ayxkWNo8&DTWQ1K=geG86ojd%^S&lSFZ~aF z&$)z9e*9X>yn+ zTs|oorzty{`*LWi)>JOC^15i|RYhY%Z8Di1)rk$qxZ^Dm-_XB%okZ?Yn)|0rM5j3q zM?ha|NrNO-nlZDITs%*P5dSM$b4s;i$eJ9q!l@mw1lzigmVSy{`bHdbfQW`F;p7pd*a99?DE!EmO-{+OnlDHg0%qB!7+#O%YysSQZRnCcW zB19%aDD29W3w_>yy#N^&^009z*%#vO)(koTA>axuc>43I%1xvMkD33{Ehf)TszPV6 zqrJDTKFd0(patkzGd1XbkZk1h2I3pT%pu9G`EB_ZQK~v?K=UCAMpe@nGF)sJ3t+IV z#4bV~Vw`q$NW6M}wGy9DB3}TRmDd^jXs!l1?ER@lGEQ+Aq56{jcda!zlp;&b3&!k0 z&v1JJSE%PMiXR*T&aa=QOjDo+1&(b@>7?HuG|}_?;F?}Q2boTy>1e<6=S)jk%}hEqi%mUaxEdUSw1DB(j+)q#cI=yNmf(AZ4(MAXPbxTnq!5rrA+Rj;$6P zX;eU;Bi`?jJ=ggun66AY2(T^3o{rw#c<|N zB2NSDB$-?G<$U!Qo^-dK;Yct*g}Tytq@}8d59?PnbYge6M$w3qe*}8qq5ablu757i zZ`@|zTo^zZqQmUg_sqnocIuSAB4qcEMColZiuZ_{?HV(u%8a9ZOc#f+3$$_kxY_o4 zdf)j@NN(OuvD>n&oy4c1_lHo|ap$ ziJ`7kLYJ^yQCEr{R1OAXFsq`3b!4#4(XA7IQWDolZ%fPzU#od+hdFbP-%wdP@bX4K z(BmbwG-PWOw`=}RT9V$+Xk<*7sWrZh_j^^Entf|tdP1o6;n7TIr@RnP#Z>5YFI0i} z<{36G#-AR841==c2U$E2fa>5SoQ<2|9>Wk{PBZc(vExcs#WZ4pbdnktT^E89VCt|b zP@TU|HXBuMNZXvm)i@K%?MJYZ&$6kjsrXG>?Z$r5G03X3S-?d@nfJ^P3SryN!^grm zz7+KvSiZC=H!rGhM8CZ*)e*rraHms)h`f=pKIBnm2J*=**?W|nt)*c}Ves|qqn71`p zO@-deS`mGr_!r)T^ZX?frqUyZaj@7}oUR%dxY4m3?8BkmzZsZgyX zs1)MjlCM;zwn-p%NW-f|gUeR%yai3pS+ijzsh=1z`zIk{IOh2XrBv|+_LouT<)b-I z5?s1~K+qJgx_X)jnh{U0>(N33i^`^-3JGEUPi)>w@|qeR4a~f>1z4JdJoTF2_@B+I z+OQAwvN8VR>;k!~nMvjIVOUHqG>(f@W2FTH&7t7b+jg(5=FUa<E<6^$bTj_@^@(!*a+HA} zTrCAe6J7$QPe!XL8)NuYm+X|#heWcsEwvX_aqN?OjH<4_^Pq9=G^qB z1Bj~~4xc)g{AMP{S0Yn%)8h>5R9a)z;l40IkVS}we~R1>U3!34ot3?8jlhC1i{svP z4at4jTC=gK*5u_5t^$jMxvLKKYQ&2~cp?^Tcg?%%5kiNL?YFG5uZ<$rbK^vrKfyM}Cg#7fmR0Ey%?>Q%PBQ(y0wV3g87 zvrr4I(q|>WL})0<>A4i6iFJ40PeFE1R92Zr%n$Vr3NyX`5}#6mr%t=V*hw@kK6M?cD@}+}W;W53cce;pN{(huniM$oHQw zAs4R{*?mx^nLz~X<%wiELHjHCmaT5>nzIu<1JMQIkJ`~aCm$cis=ZUf8Hr27k6DR3 zGS24}J9v_TenlN=8>KYDMlxCwmmo>Qe zd5;r>hx?H{qGWA1LnkN%W+$gF6FK`pwN4tJ6a&-}vCoa}V?&D+isbC9gono|CUV4o zy zVvw#klcfxA|FpyU>B8a1DM~LV8|8`gp;DNGah03FZV_#kM2CkWX|~sGsykLH<0SR+ zH4(%0hhA=3qFUYkE4$xm&gk>A)ArZuuix;zR@gWRW(V#&qRe6}KJS)>wd5;hDs>0W`c zl9XLvhyG)ca8iLBVg6ZN%4CzJ|CL5>-4;0F&2S;VWm#6J$^@+^Vf_$}prl)enTEG3c$AkkRx`5Du3JBKPLwokzR%vwoxj900v4y^I#+{u)L-vxF>*M3D-}Q zAFN_Dp7e#9RC3fxco`bZmEd3hQU-&`W;pcPjG&>A!BDBAGts*J$5&Kr3K_9_IU8{4 zWg(cLGIfhoX#LRhueDM9JS4v4!vGaZ#f*F_vwerew0Wm=>;Ey7fM6lc)9o`(!yi?W zD8rdOGXC{JCh|MOcAamZ1fEs?3lPXDckiYCYbEif@G@4VJJgNaQkDLDN*(Sp^8$g( zaJ1;3X7PODsha$pjkfPtYvuEDqsr@6Mu1W8wL28+Cx;6pi#80~z3}G29>=_H0DS8* zM>Q&kcvOY1sjrM;0~m_fz0r^P-|9Nhdq8nX+?|V zSrJ{sk{}=UJz#TcJZ>((Y?-A5538iBnU6aU-jlck{3>;twOqf>tf!L!!;2j2@S5|g zOlQzC{?DJ_6?HR`ezr}B_}m4~~ zNZqMto;RSa=PM&;b^*Ruw#G;g1v6_l)$&#Hs=?7-2Rc4@4e+&~An@LAI>=bf-nWp9 zu${K6F3>}B1B7J@P=e4qk?cB~!y7sd0K)q7Cs!(&eoOc5G1(%t-Q#XyHs_)ZVAy0u z9Y9QQ?Ye5+NJpCVo+u7QvyQawUX0saeFaE}#T9@xYtMJ0`GNpIP@9yt`qz5d{kZ)5 z>kiNj1X*tTsm^@5ttfi}Ps2Jr{4z}dVU|BD0`>kTdss)gLvnFdKLo#+DmWMxaJHnD(8@!-4E}e z^Btu+wH_9xUk;NE0_Fy(&1f#*`Ho|@eLo;TBXXbfk11B1C2au!7!j}`38Yw7H20uL z(INwTWHw3}EkUo#W5fM+c(A%jDp^_02sKxRxdcmbA%mIg(28YcxsZBKisaPw5-)&C z>;Q=t{LU9ydjMAopXbwQqfxlv*SCjBpQCdJAe%b-^ux0+ed*j4$UG>GL&$v2kiPF| zg?ZVL3)DlPLQh-rpRw&!K2NlD=v+*&i=QWZ{G+q+%H8*p-1$n9eLeiSzKG8JzJ zhR%|EadQ8rz*^U$q~Q8779)rE=n-IRy+9HGYm&vFFU0Psr0A$!?~Mp#cdl;Jya%Ci z1B8EA?}U^5m_sRT@{!&1{@iZU59+Ab7yx*x&zEUYWfc|rR zt94xNP_5K)Kl7ai=(geXX=5^KVu%wX>W&9FX>4dF6>1`%$95=1&8kgnWMyRHuJs{5 zwAd(Z!;-;r0IF=U{!x|85Rr;K;6pOa{Wb_1odxFLyZ2_+Qzg5gdc)`Kg@q_lEy|h1 zJmZ;)RW{<9X#-$@CHA@Bjeu;6OoZHVQh01=x7yVO%hO4%B`?ucgmdissLwk|#S{7A zk7kpjMoL~mgHso#$)A(3TW&p+=z>V-6e`Y0XOipI%+>IrFCyOE18n_hifH?Z!K!2I z749~O~L=;+qM_Y+=#BthEQY2(JkE>ZDa z39Al1riKk*W*pxIH=BNUjpbd8($+={3=2^%Fa(s^1KASWh@C94U*?UJkkxHSDM<$h zfXyrXBAV~DX<)RMZD5%ve!UhT?k=?14LsIzVc|>aEN(~uA7MXgf`_eLEz)NejqrU{BWX zcWRf*Q?w1#2OaEg6}M3o7Qx|57rD*N>m6uFaz1m*qi z6B?sfu`XML0Bov;`h`sgS-kzo(KKO6>y>seBWL7IcvXTW|A@U-WGXMblCJhNEwGNp7Eqo zaHtF~ObZ>z9tY+^wJg4dc)=oz7xX#*&MVS57mL=6^S82 zsNZ>=4S;=l4q+u5(|U#;)5wAuNbzCXYV1i+WTNfRZU8Bhtna5Q$~!7OyW$9a23;>K2S>=Eh# znS*RYbb@3gH4KhNmrjPY=3FiX-L0;nw&X}A=&j#`QDyS&x#{2y?d!~N`OJ*Wu&r)m z%&m2c5#cu`{GAQrUSaOzQ$y%j_SxaDr-TrJ{EK=}Us`4`rfa@N$2>{zl3EZ{H*h!@ zO8l}uH{HQ!vaVwlUjEivN+yX<%Es(R+ieYa9p{4!Sn*}3Hk@brSSlaxRxBAtH3Sn3 zaH8(uvHErk9%S;!qk`*riu)R*l=>gOt4YBGvv7Rdwe%$Ue-GF~X42#@vT71hj9XVI3y7$faQ61=h3cd2eCZ%14urP)dtxTYeY|B322W_2 zh+*``FivF+vwU73fCWGRc|z`>SR5%DB5SRgPXriYqyCDN&hhXqHGWeE$li{ATD15{ znY0%b3Knx9uyEDl8LJFGTgZ~98llwF#X)JJJGTP^rPS{3&|j}DZAe8S7$ERUD=?&s zhrRTjGCP|h#UR3KMtDg5Do!_u2fh5s;z^LGO4Q6BcvmIkO`=-rqWTG3sC^h&3^ykV zmMDf$i1bFquy^9~`u7K+5^J0%Odb7`h7hxKfuI|U&Uh%Z4zvD2y871bu-9aJz_kl) znqeP=zcnrrU^O3J4s|)Z(y$|3`xMkttxx>Se)87u)%6Cs^`>mjOBeH8)^<5^Auh@e z{zTN%93xp-RIh+m!U%YFE!Dk&lwmuj?1w`hlRNFS^`tEXBFUkjoUui1Z{#v%#MCa-^6+DUHFJ zKz|>x$l$J8=11kV`wGv95L2S(OHvTA{vE~EZNjCCqem92Y9rhYC}={^fjjgP2mfHd zFXS8CAUGm3ghCAwcWSDr-|cWVtVEI0(3H;32?dYrRjduh;ANo)J@dL+mNWFarA?QE zAa?!_3{2+DwqOEpSN>K0*OxONF}m_%gldVSI?S1Khi!!*!b_u}d?Qp32gk8^OqmFH zL<{Qq+rXMa8DG}LTvkEO3S_U|r?C%Ua?Cwi%7w3lzuBLuE|k*GigLy>(;icHObyZ!@m8~-7_Y!(n%+_m^=+9WC01uWlui!2hk{EK9^TLgw=q8_ z?$_R~;M=ysEUt;_F_$g=0VV|x$Q8T`ze%Y}JXbtLT~BTKt$f@) zFL&W@C2BZSj-n;$x*@=7pkyF8wAh<~Qm4vR$g3}i)S3~8@8(9WD5g_(c8G>K1X;cd zIK@AIfI^rfzRmyQVdr@Xylt>2x&AxCG~25I1%9?*iuzkg14BZ6dZyh9TNKTg^%`(Q zX^Q9WX~^j@tOcU&j42-5ZGJWb(=bJG=vkkDoSJbyBWhJgjS6(k-fSl{h=DqIx=>Z} z@PJ@STSoiw6dnJm#=Iyl$zJPaXK7k!M1}Mg;nkG50iUr6U31>@6CXbACd`*<-8y#? zP8L&Tcb-)Gpr5C?ON4X7{x#w=Pc-dkVV6<%8PBnj!b(?c+4x2}re9;>yr$#lH7)I3 zJwpw9ABY{Fw=aTlY|NJDP)#0oLlZNBOe+(7bZtTCEz*wtFE#GDA%&PvzQMSrlCgR*=*%J3UDF?R*doQU5{@!*Xhj_@({J#gg8LQF zi_n6+9T7Gu<8V~v=+Z34XnKd@t_rgj7H$LM8I1>5;<48iO{122>v*0+BuMw!Z5zx6 zm}K!?u@IQCu=+S!_e)Uatu)B+Ju}>-ZEH((_mOH)8wXNnzqQ(<`IX*eGcNia+uG-n zS)$r$Xk^4R4ZtXv;TjvQ+b=i7H30CMlR_mch#1 ze~%raB!H5N#JNGuQ~?W!mx{=|YMiI!+Gfc29-~whf>zw=u^f!0DI{!-_P2<}6cduc za#iE5&MClm=7g+YKr=;F1zto9weZF1ZhqwRNM!43E2-HYquHVvUa|Kj>A}G=mCV0W zWWd2!+r(4#tj9y1ZV4r+gQ2Z|;hgHHC7X#|@Dgf+3diu*ks?y~>a>3YY;r^ZF-viYnIvJ5YKgrq|QQ?(LV+J$CGsLCtb%^oc z4aylgB4@Xn>kb|jp9`_w{uIj)IO4I{xDl2LG4j}sBO5o%x_Q13`ugFu(l;<#RV@{3 z;rcZQgW+(qxuIcah%uv-w>(;e)R1RzD&I|go&!N}$6j=iG*ljb^4K{-0M{(_YhEZP z8n*b#tyBMzyOTs72I6w$mwP)F!jATr10}M4?r4%JVhCVUDTq=_Nz|Vm(4YkcVP#Mb zBJnfm;QIkux5G2Hhy$Wr*mpkP>PU&Ew74Y&vAr~C?DP98K&C=np*6MI_Ggs?B*9>=PSczW_W+IDii*lqd51scfNBJ)_0B~wEywl zgI7y56+n5~uC#h3TVS0DkNdAu5E}$lW4GIYM?>Q}U_z48lOsc;bT{W6KlqP(4jap_ zM%etsA|l;%N%T|y)BgJ{KKLaZq#fHYt6~y~_l3+*+xLJ-JhN{Qn8r|7SvG*)|DSpUNj|iZ3kZmeW+5 z+~rL_bU&?fK|Wp|q6Wd^sHioWPG;!?SOGDdzQt?_a104H+N{A|sqX^zHPi~g%y#~G zTp670{U~w8(6sqOA4vTp0InQOHM5*RTFc*OHKX+ULoq}bP0d|(v$8nATL!6hhehxT@epJxZ?GC-L0v{ zv02O&pZ|88Jam4CCxG*a?VX>Ix$y#Vp~+;n$@}YkvqFIFQkiH|=>b%NAwnlhITT}Gn`A8s)o}_LOGVj5 zWvlFoh%$*1DM#+}s&ha0e(vA*U$c0Bzj=S}_j$g{Q(SMD4AjpxprM|f%o+p@x*8rA zyQJ({d4q`$RXyxJaegS82G6CJ+Cu$ObLiCi>%!Q{FOJ6xY^_&-Fx%5dT3e>yX!!`& zw0Dz51HT#@SWD`%nuuZVuEkMy{qc8YF}o&YB8bh0OTN;Z4N0Kt^%`0n>9*hEwg$5>cN?C2TtVAF-uKiM?NfXAx*G#J7t5JD={%V zA01ZaR9XG$o$cJww@Pz&bBfoPk;J2L;J4Mg6+q9lr~B z%|84g)F0uOBr!v}5I;G^Yi&Nalg>7ssum#QcwB%{8 zyn4&`?NKB2F?=T>Pj)t>^HUV#mS%VwojGTjQgy}AEkqF>!4C3eIeQgE1yot19))i56083(00ct5iRx(uZH zZEso$9sKRgR=>>9&evy`o;8kj0Xr-C6k2mPptI$**qr`MBK8sxNMXAoQxoy;Mykv1 z@OsK@vpw=yNgX2;6~)n@$>6CI3&8ebA8fkibL;iX09n0^=tB&TBUu3EZU%tmgTD4X zjE;nWEqHLPGo-I`BV>CGZyWf!yflEk+Du?!A7&I;lSVlA5TW4$efIVlfJSy*K+r9^ zD>evD&k+FuJ9ZuyoX?ndeA89){o|jt|<8 zP;hJO+FJx=Oq(cpde~4=%Gc%r-zzr%FjgAucM2+M^NzJ zYZ$E-uO$DBy`g}6Dezk*+zCkTmU5Rmm9&+tL(UnOdHzF+Thfa@n%J+naR9C{xZQuT zKsqk$F1&&6mpi<>LK&%Ka7Pw!*5(o!Z4ZG>>?k@+O>ZuVmg(2!|)8;Jwd_4&Y5hF)ZW^F z*9ySH59-qEnPG=lBhVVO!u{(IhyR7wo<= zXDaJO{0(2-0$PororBxf6z}^T5VLi2gi49V0tb{`(@d=H1Om==34tL72KHDq3qZi1 zmT?lm)io4o@)ih0b9sX@t1wS{PC?#t$Y*lJx@xZsdAPl_wYL>`6FQZ$qcqz+vEogl z3k!0yv?eQn2fd&__*NKbYQjR1qcZ0Fit{O zyeSV5bIqq6PBWj;M)~dF9_<@p$Rr*JjF~Urg7PFlu?kcS+V|HX(?8j7CeH?z{a^<$ zVS20^GSWIAgRLr|mx})ghF^XZP2_{9_y*q!JY4+n51lXMUM8P!gmi%570|jA;FKc& zB#5}#$3axe-T=;@IbHNFm<=gVhP$*ta$-A^aH#7GOVI~e@z&cVbkhB=?mw5AjTc#j zI;Wh}y}2piY~fYxITqiDVg#m{tnV$?29N#0cHJn&dd*Ek3klu_7RDP|?$hs-bzg=+ zw|G7rv*OE2`#u9RFOx9knqkOmb-T(`U+xXqyxF(Q)ww!=tP&(0`3`pkZO^w3Vcp*N zYzTE6w3dWOXr_mz0}$>n`<4Lw$uE%RGL#M{{!OFMr-~gh$iF-VruLG+KEoYkwRN}n zo<%L5`~%awRQ(*HL~_oi7KG&a*D$Mvz{rF1$`9g%q7w;VnrpKRYo9^qvTtywA{(a7 zu?nQCIRRxwJq*-F_7>kU+xOsIz(1#at4JYOYKOC5Wd)exDX^clvZ589X$ljyp(%-u z@}=5Fy>%Sj#z<>(K6y_ZBqaGs&_G$z_J8+r4etX&>ZYV?kyEq9NvJ)V+*rnK3<2VO+A?;tvYx#`0c^9YeY+R zK$uG&Z`DT`7)uum^~8oaa`KUm#uU$fyT=psj%wFo?Iq%Dez1A9sr%(t?@ohY<>wII zQ^{(H>=N4vIPVLv;wKyAa>o{j<_nyN`x`2&kP-a2uU&S{rIK6JeW%&gCZZDObmnee zM*)Gtg>Vi&1(Avp{a1$w?h9{lG+xF>5l%_@mwkCKL%fw{rO)H6GY!@rOr>fzLxEy% zwH``Cr9idu%ytF8bZ zg?G6E3FfRBIuKG{1Y{VtGrrY2l#k{0pA)mnxWYgKN{%HS~J5S8U_)=zG8rIZgqQMOn_tY@{DS+Ce z{8r^XaIo{U!x=_-P&_V*yc53gJN>_2iIy;cISldhVP1VT6nev%$MD}ZRQ~B(_XR>o zZ4>KvV=jBKTi3f;gWmwBScg7h%ub>iR)`-DH>5dXf&3w?fdeu8_+&ppH z#RXKB5C{rIhpG=InXNgSQ1PU#!vZb5x2}^7e+?TM$4AX3k4|XXurGN!notLZm8)!B z$q%78fzOT;6byU6XJnSPRmz)DrA;}1FFNG5gj^vZ64)+IFX0%f+7kJftD-?HJxG8> zw1X_T>C(vdHg8#GiLV{qL|HLx@{-Agf6`uc8N`Sr8I6Qr!#Z%d#?xV$#m6B2mXjhS zz{mg&SVR`Q#WyC{oE$owqhCZ46@MZ|5l{O$DP%c0>XdV2uH{1uFmJGo2shJYj}4{4 z)j~~n9NSA8KBTD3b^X)T<=g*6e%5{#Qbu|VkxHpm=d564ikFplTJYv&D*d!@-2XZZ zYyDFnO0wN`8pmh$pZgvI!mmXw2=6p8JTh1yXZ9;3&L?(-N*EW~5u}cK-?kH=&+}{I z&H^?}LVeqR5j5PdJSy%lZdy&E2wi>5Z-UWKW+dFbQtrZ5+hCj~?1nlGUetN1j9CZd z5_|We#>Qh>-6^^e4k!L0V0w&DZAiG|&bER=ZOkCqC)M~to5=ysMIHsrA#;IH? z!vnQab&$_P{p=2-RKcCRM+2Jz!LAoK-Vo8@at_=EZt&)pQ!s2qo>fpyp_I@l{2c&{ zJO+*3P86@z-VarX$3wtw;$M~R{NaB?+{q`xPvbG<0R>RVrlx{xlk{ zk)`qxG1=J<0qe+rTbs%9M8XOe0S@1N>b-Vg%P2^Y_*9L#nINxt5kf*S zkg4_&4DdV(nwDEpf&~9Xo3LT?C;&9cy2sTcZD0&n(?V1Ms6cE`NkphYN9S(y?1?bZ ziJ&jtOxUHp4kSC=Rd59~YB0?uIVT*ABx^2m~Ruc>x` zAnEIqL^fPs`gM$_R{6NqLz9WC7z1YyKTvfN0z(4$F{ee3}*GZ+JwNDmOR>MPG3Yx2Q$F2H-#m zYQhq!vjV<21@XsNYmn@9<}JS>TB7sRkEmOQaQi)yf6&Vx=|75GZ+_Qz-6J2Wtr=)u z{Mdyo%9yeJqU!Xm>;kHLYUq$Plu=J*-q?N=^siGIoB=pAOdz*exdcE#?&_;Fp=f>V zkB#g16^4mI(biZLoe2aE<&u$vF-)s*$sZ~(NV7_BQ^^kQG&to6rVdsn!N8~!3HkEn zZX=y&{7KaD(P=1ynL|g;414WJ(SS-4vpsSTbbVQEz>J8(a}iq+>Zy)(;J$t@R&mj; zw%XNZTJ{RKh3v{Aff;I^$qNv+xWJ_Oa<43-J}p*#26l9#v!+^*97Hu1p$@_doL6#d z0Jg>MDSRUrWR+iGIny8w+q(_X9hsezJ~{Ogoia&bP0#%Uwq>*n9+s$8>J+>^6jOWp zEbgSw>O(=y)MID7W3Eqo&`iarkMoO@EvMs`;KT0)xHq%inwa@ihOd$r`$V-Rs5=_^ zhd!n(aw%gm$w&TyvAO%VfZ(uy&RV6I&C~R*r8RDx60<}JQLEy_?n96wxl+vm$sa0Zf0Q8#~GE~cyBcSXGqtS`UWXPU1gPRB)I)oTT~;oQKqtd!j?=1G0zU81Fh(Jd7V(m4CefJ>O0#SINR6 zDoNA;M{Se+mMLU5r)k9>gdi~5Nw1N*f0d=An|4syOf|`@x=Rc?+b&v=CZiK^w~W4^ zAFB*kPlZ5H$WKdW7G9xWsf5El=0UR5qyl$Sy~N9B6) zOTFJwL;V7SnfXhv=|ayCpQu0@LA8h^uXXt{tcmpyqlepb-e=xqY=UZM#i&XD%D0QF z=p&n4ht1-I645NakMin6ij-`?l|6ZQ@q)^yN0N1;c#UJzud4@&SoIf>cy@2VvZDD7 z8fgdBo^k8evCeaTrlp)Uv^Jr77Dr`V%_vs9%hu-qc;3JfdmDkVuB-^dAH&I8{Fvug4@6qgh_!W-@-H z9AW37U$eQyFw$0>-2t?F(25Me8=6{8s>UkYeu1%*v50v0!4W4#D?96)*`GI%K#&qn v-{)@8V6oYO(>PHlikWRg)yR^_;s!%O<(@b>))(Dq#-OR8xxpQM*Yp1Z=VRA> diff --git a/example/dce-cradle-mptcp.cc b/example/dce-cradle-mptcp.cc index c3154d3d..d0f396ab 100644 --- a/example/dce-cradle-mptcp.cc +++ b/example/dce-cradle-mptcp.cc @@ -45,9 +45,8 @@ int main (int argc, char *argv[]) nodes.Create (2); routers.Create (nRtrs); - // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only - // with LKL; libos has no MPTCP). - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP). + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); DceManagerHelper dceManager; dceManager.SetTaskManagerAttribute ("FiberManagerType", @@ -177,10 +176,6 @@ int main (int argc, char *argv[]) std::cout << "received from " << addr.str () << ": " << bytes << " bytes" << std::endl; ok = ok && bytes > 0; } -#ifdef LKL_LINUX // With MPTCP, every path carries data. return ok ? 0 : 1; -#else - return 0; -#endif } diff --git a/example/dce-freebsd.cc b/example/dce-freebsd.cc deleted file mode 100644 index 02aad5e7..00000000 --- a/example/dce-freebsd.cc +++ /dev/null @@ -1,158 +0,0 @@ -#include "ns3/core-module.h" -#include "ns3/network-module.h" -#include "ns3/dce-module.h" -#include "ns3/point-to-point-module.h" -#include "ns3/csma-module.h" -#include "ns3/wifi-module.h" -#include "ns3/mobility-module.h" -#include "ns3/internet-module.h" -#include - -using namespace ns3; -NS_LOG_COMPONENT_DEFINE ("DceFreeBSD"); - -void -PrintTcpFlags (std::string key, std::string value) -{ - NS_LOG_INFO (key << "=" << value); -} - -int main (int argc, char *argv[]) -{ - CommandLine cmd; - char linkType = 'p'; // P2P - bool reliable = true; - - // for the moment: not supported quagga for freebsd - std::string filePath = SearchExecFile ("DCE_PATH", "libfreebsd.so", 0); - if (filePath.length () <= 0) - { - NS_LOG_UNCOND ("no libfreebsd.so found. exit."); - Simulator::Run (); - Simulator::Destroy (); - return (0); - } - - cmd.AddValue ("linkType", "Link type: ie : C for CSMA, P for Point to Point and w for Wifi, default to P2P", linkType); - cmd.AddValue ("reliable", "If true use TCP transport else UDP, default is TCP", reliable); - cmd.Parse (argc, argv); - linkType = tolower (linkType); - switch (linkType) - { - case 'c': - case 'p': - case 'w': - break; - default: - std::cout << "Unknown link type : " << linkType << " ?" << std::endl; - return 1; - } - - NodeContainer nodes; - nodes.Create (2); - - NetDeviceContainer devices; - - switch (linkType) - { - case 'c': - { - CsmaHelper csma; - csma.SetChannelAttribute ("DataRate", StringValue ("5Mbps")); - csma.SetChannelAttribute ("Delay", StringValue ("2ms")); - devices = csma.Install (NodeContainer (nodes, nodes.Get (0))); - csma.EnablePcapAll ("process-freebsd"); - } - break; - - case 'p': - { - PointToPointHelper p2p; - p2p.SetDeviceAttribute ("DataRate", StringValue ("5Gbps")); - p2p.SetChannelAttribute ("Delay", StringValue ("1ms")); - devices = p2p.Install (nodes); - p2p.EnablePcapAll ("process-freebsd"); - } - break; - - case 'w': - { - MobilityHelper mobility; - Ptr positionAlloc = CreateObject (); - positionAlloc->Add (Vector (0.0, 0.0, 0.0)); - positionAlloc->Add (Vector (5.0, 0.0, 0.0)); - mobility.SetPositionAllocator (positionAlloc); - mobility.SetMobilityModel ("ns3::ConstantPositionMobilityModel"); - mobility.Install (nodes); - - WifiHelper wifi; - YansWifiPhyHelper phy; - YansWifiChannelHelper phyChannel = YansWifiChannelHelper::Default (); - WifiMacHelper mac; - phy.SetChannel (phyChannel.Create ()); - mac.SetType ("ns3::AdhocWifiMac"); - wifi.SetStandard (WIFI_STANDARD_80211a); - devices = wifi.Install (phy, mac, nodes); - phy.EnablePcapAll ("process-freebsd"); - } - break; - - default: - break; - } - - DceManagerHelper processManager; - processManager.SetNetworkStack ("ns3::FreeBSDSocketFdFactory", "Library", StringValue ("libfreebsd.so")); - processManager.Install (nodes); - FreeBSDStackHelper stack; - stack.Install (nodes); - - Ipv4AddressHelper address; - address.SetBase ("10.0.0.0", "255.255.255.0"); - Ipv4InterfaceContainer interfaces = address.Assign (devices); - - Ipv4GlobalRoutingHelper::PopulateRoutingTables (); - FreeBSDStackHelper::PopulateRoutingTables (); - - DceApplicationHelper process; - ApplicationContainer apps; - - if (reliable) - { - process.SetBinary ("tcp-server"); - process.ResetArguments (); - process.SetStackSize (1 << 31); - apps = process.Install (nodes.Get (0)); - apps.Start (Seconds (1.0)); - - process.SetBinary ("tcp-client"); - process.ResetArguments (); - process.ParseArguments ("10.0.0.1"); - apps = process.Install (nodes.Get (1)); - apps.Start (Seconds (1.5)); - } - else - { - process.SetBinary ("udp-server"); - process.ResetArguments (); - process.SetStackSize (1 << 16); - apps = process.Install (nodes.Get (0)); - apps.Start (Seconds (1.0)); - - process.SetBinary ("udp-client"); - process.ResetArguments (); - process.ParseArguments ("10.0.0.1"); - apps = process.Install (nodes.Get (1)); - apps.Start (Seconds (1.5)); - } - - // print tcp sysctl value - // LinuxStackHelper::SysctlGet (nodes.Get (0), Seconds (1.0), - // ".net.ipv4.tcp_available_congestion_control", &PrintTcpFlags); - - Simulator::Stop (Seconds (200.0)); - Simulator::Run (); - Simulator::Destroy (); - - return 0; -} diff --git a/example/dce-iperf-emulation.cc b/example/dce-iperf-emulation.cc index 9c53b3f6..79123576 100644 --- a/example/dce-iperf-emulation.cc +++ b/example/dce-iperf-emulation.cc @@ -81,7 +81,7 @@ int main (int argc, char *argv[]) Config::SetDefault ("ns3::TcpSocket::InitialSlowStartThreshold", UintegerValue (0xfffff)); CommandLine cmd; - cmd.AddValue ("stack", "Name of DCE IP stack: ns3/linux/freebsd. Default:", stack); + cmd.AddValue ("stack", "Name of DCE IP stack: ns3/linux. Default:", stack); cmd.AddValue ("udp", "Use UDP. Default: ", useUdp); cmd.AddValue ("bw", "UDP bandwidth limit. Default: ", bandWidth); cmd.AddValue ("emulation", "Use emulation. Default: ", useEmulation); @@ -142,19 +142,6 @@ int main (int argc, char *argv[]) NS_LOG_ERROR ("Linux kernel stack for DCE is not available. build with dce-linux module."); // silently exit return 0; -#endif - } - else if (stack == "freebsd") - { -#ifdef LINUX_STACK - dceManager.SetNetworkStack ("ns3::FreeBSDSocketFdFactory", "Library", StringValue ("libfreebsd.so")); - dceManager.Install (nodes); - FreeBSDStackHelper stack; - stack.Install (nodes); -#else - NS_LOG_ERROR ("FreeBSD kernel stack for DCE is not available. build with dce-freebsd module."); - // silently exit - return 0; #endif } diff --git a/example/dce-iperf-mptcp.cc b/example/dce-iperf-mptcp.cc index 75e205f5..3e8eb2e8 100644 --- a/example/dce-iperf-mptcp.cc +++ b/example/dce-iperf-mptcp.cc @@ -45,9 +45,8 @@ int main (int argc, char *argv[]) nodes.Create (2); routers.Create (nRtrs); - // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only - // with LKL; libos has no MPTCP). - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP). + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); DceManagerHelper dceManager; dceManager.SetTaskManagerAttribute ("FiberManagerType", @@ -190,10 +189,6 @@ int main (int argc, char *argv[]) std::cout << "received from " << addr.str () << ": " << bytes << " bytes" << std::endl; ok = ok && bytes > 0; } -#ifdef LKL_LINUX // With MPTCP, every path carries data. return ok ? 0 : 1; -#else - return 0; -#endif } diff --git a/example/dce-iperf.cc b/example/dce-iperf.cc index f527884f..11220391 100644 --- a/example/dce-iperf.cc +++ b/example/dce-iperf.cc @@ -37,7 +37,7 @@ int main (int argc, char *argv[]) bool useUdp = 0; std::string bandWidth = "1m"; CommandLine cmd; - cmd.AddValue ("stack", "Name of IP stack: ns3/linux/freebsd.", stack); + cmd.AddValue ("stack", "Name of IP stack: ns3/linux.", stack); cmd.AddValue ("udp", "Use UDP. Default false (0)", useUdp); cmd.AddValue ("bw", "BandWidth. Default 1m.", bandWidth); cmd.Parse (argc, argv); @@ -72,19 +72,6 @@ int main (int argc, char *argv[]) NS_LOG_ERROR ("Linux kernel stack for DCE is not available. build with dce-linux module."); // silently exit return 0; -#endif - } - else if (stack == "freebsd") - { -#ifdef LINUX_STACK - dceManager.SetNetworkStack ("ns3::FreeBSDSocketFdFactory", "Library", StringValue ("libfreebsd.so")); - dceManager.Install (nodes); - FreeBSDStackHelper stack; - stack.Install (nodes); -#else - NS_LOG_ERROR ("FreeBSD kernel stack for DCE is not available. build with dce-freebsd module."); - // silently exit - return 0; #endif } diff --git a/example/dce-lkl-p2p.cc b/example/dce-lkl-p2p.cc index 3975e42b..88fcdf87 100644 --- a/example/dce-lkl-p2p.cc +++ b/example/dce-lkl-p2p.cc @@ -6,8 +6,7 @@ * with status 0. * * With --loopback, a single node runs the TCP client and server over its - * loopback device instead (like dce-linux-simple). --stack=linux runs the - * same programs on the libos (Linux 4.4) stack, for comparison. + * loopback device instead (like dce-linux-simple). */ #include "ns3/core-module.h" #include "ns3/network-module.h" @@ -70,9 +69,7 @@ main (int argc, char *argv[]) { bool loopback = false; bool pcap = false; - std::string stack = "lkl"; CommandLine cmd; - cmd.AddValue ("stack", "Kernel: lkl (Linux 6.12) or linux (libos, Linux 4.4)", stack); cmd.AddValue ("loopback", "Use one node and its loopback device", loopback); cmd.AddValue ("pcap", "Write pcap traces of the link", pcap); cmd.Parse (argc, argv); @@ -91,14 +88,7 @@ main (int argc, char *argv[]) } } DceManagerHelper dceManager; - if (stack == "linux") - { - dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); - } - else - { - dceManager.SetNetworkStack ("ns3::LklSocketFdFactory"); - } + dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory"); dceManager.Install (nodes); for (uint32_t i = 0; i < nodes.GetN (); i++) diff --git a/example/dce-mptcp-handoff-v4v6.cc b/example/dce-mptcp-handoff-v4v6.cc index c3a8950a..dd3e3a91 100644 --- a/example/dce-mptcp-handoff-v4v6.cc +++ b/example/dce-mptcp-handoff-v4v6.cc @@ -168,9 +168,8 @@ int main (int argc, char *argv[]) wifi.Install (phy, mac, ar.Get (1)); - // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only - // with LKL; libos has no MPTCP). - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP). + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); DceManagerHelper dceMng; DceApplicationHelper dce; diff --git a/example/dce-mptcp-handoff-v6.cc b/example/dce-mptcp-handoff-v6.cc index 850688db..e7a98519 100644 --- a/example/dce-mptcp-handoff-v6.cc +++ b/example/dce-mptcp-handoff-v6.cc @@ -167,9 +167,8 @@ int main (int argc, char *argv[]) wifi.Install (phy, mac, ar.Get (1)); - // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only - // with LKL; libos has no MPTCP). - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP). + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); DceManagerHelper dceMng; DceApplicationHelper dce; diff --git a/example/dce-mptcp-lte-wifi-v6.cc b/example/dce-mptcp-lte-wifi-v6.cc index eb21af2a..fd81211c 100644 --- a/example/dce-mptcp-lte-wifi-v6.cc +++ b/example/dce-mptcp-lte-wifi-v6.cc @@ -160,9 +160,8 @@ int main (int argc, char *argv[]) DceManagerHelper dceMng; DceApplicationHelper dce; LinuxStackHelper stack; - // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only - // with LKL; libos has no MPTCP). - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP). + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); dceMng.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); // dceMng.SetLoader ("ns3::DlmLoaderFactory"); diff --git a/example/dce-mptcp-lte-wifi.cc b/example/dce-mptcp-lte-wifi.cc index e6d664da..00175541 100644 --- a/example/dce-mptcp-lte-wifi.cc +++ b/example/dce-mptcp-lte-wifi.cc @@ -84,9 +84,8 @@ int main (int argc, char *argv[]) routers.Create (1); DceManagerHelper dceManager; - // The applications' TCP sockets are MPTCP sockets (Linux MPTCP v1: only - // with LKL; libos has no MPTCP). - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + // The applications' TCP sockets are MPTCP sockets (Linux MPTCP). + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); dceManager.SetNetworkStack ("ns3::LinuxSocketFdFactory", "Library", StringValue ("liblinux.so")); LinuxStackHelper stack; diff --git a/example/examples-to-run.py b/example/examples-to-run.py index 545accb1..9228e99d 100644 --- a/example/examples-to-run.py +++ b/example/examples-to-run.py @@ -88,7 +88,6 @@ ("dce-httpd", "True", "True"), # ("dce-wifi-ccnx", "True", "True"), ("dce-sctp-simple", "True", "True"), - ("dce-freebsd", "True", "True"), # Linux kernels from LKL (configure --with-lkl) ("dce-lkl-boot", "True", "True"), ("dce-lkl-p2p --loopback", "True", "True"), diff --git a/example/freebsd-iproute.cc b/example/freebsd-iproute.cc deleted file mode 100644 index 066f681a..00000000 --- a/example/freebsd-iproute.cc +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -struct sockaddr_in_f { - unsigned char sin_len; - unsigned char sin_family; - unsigned short sin_port; - struct in_addr sin_addr; - char sin_zero[8]; -}; -struct sockaddr_f { - unsigned char sa_len; /* total length */ - unsigned char sa_family; /* address family */ - char sa_data[14]; /* actually longer; address value */ -}; -#define IFNAMSIZ 16 -struct ifaliasreq { - char ifra_name[IFNAMSIZ]; /* if name, e.g. "en0" */ - struct sockaddr_f ifra_addr; - struct sockaddr_f ifra_broadaddr; - struct sockaddr_f ifra_mask; - int ifra_vhid; -}; - -#undef _IOC -#undef _IOW -#undef IOC_IN -#undef IOC_OUT - -#define IOCPARM_SHIFT 13 /* number of bits for ioctl size */ -#define IOCPARM_MASK ((1 << IOCPARM_SHIFT) - 1) /* parameter length mask */ -#define IOCPARM_MAX (1 << IOCPARM_SHIFT) /* max size of ioctl */ -#define IOC_VOID 0x20000000 /* no parameters */ -#define IOC_OUT 0x40000000 /* copy out parameters */ -#define IOC_IN 0x80000000 /* copy in parameters */ - -#define _IOC(inout,group,num,len) ((unsigned long) \ - ((inout) | (((len) & IOCPARM_MASK) << 16) | ((group) << 8) | (num))) -#define _IOW(g,n,t) _IOC(IOC_IN, (g), (n), sizeof(t)) -#define SIOCAIFADDR _IOW('i', 43, struct ifaliasreq)/* add/chg IF alias */ - -int -main (int argc, char *argv[]) -{ - struct ifaliasreq ifra; - struct sockaddr_in_f *sinp; - int s; - - s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - if (s < 0) { - perror ("ifconfig_inet: socket(PF_INET, SOCK_RAW, 0)"); - return (-1); - } - - memset (&ifra, 0, sizeof(ifra)); - strcpy (ifra.ifra_name, argv[1]); - - sinp = (struct sockaddr_in_f *)&ifra.ifra_addr; - sinp->sin_family = AF_INET; - sinp->sin_len = sizeof(ifra.ifra_addr); - sinp->sin_addr.s_addr = inet_addr(argv[2]); - - sinp = (struct sockaddr_in_f *)&ifra.ifra_mask; - sinp->sin_family = AF_INET; - sinp->sin_len = sizeof(ifra.ifra_addr); - sinp->sin_addr.s_addr = inet_addr(argv[3]); - - if (ioctl(s, SIOCAIFADDR, &ifra) < 0) { - perror("ifconfig_inet: ioctl(SIOCAIFADDR)"); - close(s); - return (-1); - } - - close(s); - return 0; -} diff --git a/helper/freebsd-stack-helper.cc b/helper/freebsd-stack-helper.cc deleted file mode 100644 index 7d8222f2..00000000 --- a/helper/freebsd-stack-helper.cc +++ /dev/null @@ -1,153 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -/* - * Copyright (c) 2014 Hajime Tazaki - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation; - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Author: Hajime Tazaki - */ -#include "freebsd-stack-helper.h" -#include "ipv4-freebsd.h" -#include "freebsd-socket-fd-factory.h" -#include "dce-application-helper.h" -#include "ns3/node.h" -#include "ns3/node-container.h" -#include "ns3/names.h" -#include "ns3/simulator.h" -#include "ns3/make-event.h" - -namespace ns3 { -void -FreeBSDStackHelper::Install (Ptr node) -{ -#ifdef KERNEL_STACK - Ipv4FreeBSD::InstallNode (node); -#endif -} -void -FreeBSDStackHelper::Install (std::string nodeName) -{ -#ifdef KERNEL_STACK - Ptr node = Names::Find (nodeName); - Install (node); -#endif -} -void -FreeBSDStackHelper::Install (NodeContainer c) -{ -#ifdef KERNEL_STACK - for (NodeContainer::Iterator i = c.Begin (); i != c.End (); ++i) - { - Install (*i); - } -#endif -} -void -FreeBSDStackHelper::InstallAll (void) -{ -#ifdef KERNEL_STACK - Install (NodeContainer::GetGlobal ()); -#endif -} - -void -FreeBSDStackHelper::PopulateRoutingTables () -{ -#ifdef KERNEL_STACK - NodeContainer c = NodeContainer::GetGlobal (); - for (NodeContainer::Iterator i = c.Begin (); i != c.End (); ++i) - { - Ptr node = *i; - Ptr ipv4 = node->GetObject (); - ipv4->PopulateRoutingTable (); - } -#endif -} - -void -FreeBSDStackHelper::RunIp (Ptr node, Time at, std::string str) -{ -#ifdef KERNEL_STACK - DceApplicationHelper process; - ApplicationContainer apps; - process.SetBinary ("freebsd-iproute"); - process.SetStackSize (1 << 16); - process.ResetArguments (); - process.ParseArguments (str.c_str ()); - apps = process.Install (node); - apps.Start (at); -#endif -} - -void -FreeBSDStackHelper::SysctlGetCallback (Ptr node, std::string path, - void (*callback)(std::string, std::string)) -{ -#ifdef KERNEL_STACK - Ptr sock = node->GetObject (); - if (!sock) - { - callback ("error", "no socket factory"); - NS_ASSERT_MSG (0, "No FreeBSDSocketFdFactory is installed. " - "You may need to do it via DceManagerHelper::Install ()"); - return; - } - - std::string value = sock->Get (path); - callback (path, value); - return; -#endif -} - -void -FreeBSDStackHelper::SysctlGet (Ptr node, Time at, std::string path, - void (*callback)(std::string, std::string)) -{ -#ifdef KERNEL_STACK - Ptr sock = node->GetObject (); - if (!sock) - { - callback ("error", "no socket factory"); - NS_ASSERT_MSG (0, "No FreeBSDSocketFdFactory is installed. " - "You may need to do it via DceManagerHelper::Install ()"); - return; - } - Simulator::ScheduleWithContext (node->GetId (), at, - &FreeBSDSocketFdFactory::ScheduleTask, sock, - MakeEvent (&FreeBSDStackHelper::SysctlGetCallback, - node, path, callback)); - return; -#endif -} -void -FreeBSDStackHelper::SysctlSet (NodeContainer c, std::string path, std::string value) -{ -#ifdef KERNEL_STACK - for (NodeContainer::Iterator i = c.Begin (); i != c.End (); ++i) - { - Ptr node = *i; - Ptr sock = node->GetObject (); - if (!sock) - { - NS_ASSERT_MSG (0, "No FreeBSDSocketFdFactory is installed. " - "You may need to do it via DceManagerHelper::Install ()"); - } - Simulator::ScheduleWithContext (node->GetId (), Seconds (0.1), - MakeEvent (&FreeBSDSocketFdFactory::Set, sock, - path, value)); - } -#endif -} - -} // namespace ns3 diff --git a/helper/freebsd-stack-helper.h b/helper/freebsd-stack-helper.h deleted file mode 100644 index 8885ba1c..00000000 --- a/helper/freebsd-stack-helper.h +++ /dev/null @@ -1,130 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -/* - * Copyright (c) 2014 Hajime Tazaki - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation; - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Author: Hajime Tazaki - */ -#ifndef FREEBSD_STACK_HELPER_H -#define FREEBSD_STACK_HELPER_H - -#include "ns3/object.h" - -namespace ns3 { - -class Node; -class NodeContainer; -class Time; - -/** - * \brief aggregate Ipv4FreeBSD to nodes - * - * Today the first goal of the Ipv4FreeBSD is to be able to use the standards NS-3 Helpers for two tasks: - * 1 - assign address to devices using Ipv4AddressHelper, - * 2 - create the static routes using Ipv4GlobalRoutingHelper - * - * - */ -class FreeBSDStackHelper -{ -public: - /** - * Aggregate ns3::Ipv4FreeBSD classe onto the provided node. - * This method will assert if called on a node that - * already has an Ipv4 object aggregated to it. - * - * \param nodeName The name of the node on which to install the stack. - */ - static void Install (std::string nodeName); - - /** - * Aggregate ns3::Ipv4FreeBSD classe onto the provided node. - * This method will assert if called on a node that - * already has an Ipv4 object aggregated to it. - * - * \param node The node on which to install the stack. - */ - static void Install (Ptr node); - - /** - * Aggregate ns3::Ipv4FreeBSD class onto the provided node. - * This method will assert if called on a node that - * already has an Ipv4 object aggregated to it. - * - * \param c NodeContainer that holds the set of nodes on which to install the - * new stacks. - */ - static void Install (NodeContainer c); - - /** - * Aggregate ns3::Ipv4FreeBSD to all nodes in the simulation - */ - static void InstallAll (void); - - /** - * Configure FreeBSD kernel parameters with traditional 'sysctl' interface. - * (NOTE: not supported yet as of March 2014) - * - * \param c NodeContainer that holds the set of nodes to configure this parameter. - * \param path a string value for sysctl parameter. it starts from '.' following the name of parameter. - * e.g., ".net.ipv4.conf.default.forwarding" - * \param value a string value to set for sysctl parameter. - */ - void SysctlSet (NodeContainer c, std::string path, std::string value); - - /** - * Obtain FreeBSD kernel state with traditional 'sysctl' interface. - * (NOTE: not supported yet as of March 2014) - * - * \param node The node pointer Ptr that will ask the status. - * \param at the delta from the begining of simulation to ask this query. - * \param path a string value for sysctl parameter. it starts from '.' following the name of parameter. - * e.g., ".net.ipv4.conf.default.forwarding" - * \param callback a callback function to parse the result of sysctl query. - */ - static void SysctlGet (Ptr node, Time at, std::string path, - void (*callback)(std::string, std::string)); - - /** - * Populate routing information to all the nodes in network from GlobalRoutingTable. - * (NOTE: not supported yet as of March 2014) - * - * Limitation: - * 1) This method SHOULD call after Ipv4GlobalRoutingHelper::PopulateRoutingTables () so that - * FreeBSDStackHelper can obtain the route information. - * 2) This feature is only available for IPv4 route information. IPv6 is not implemented. - * - */ - static void PopulateRoutingTables (); - - /** - * Execute "freebsd-iproute" command (of FreeBSD) on a specific node to configure the ip address/route/etc information. - * - * \param node The node pointer Ptr to configure. - * \param at the delta from the begining of simulation to execute this command. - * \param str a string for the command line argument of the command. e.g., "freebsd-iproute sim0 10.0.1.1 255.255.255.0" - */ - static void RunIp (Ptr node, Time at, std::string str); - -private: - static void SysctlGetCallback (Ptr node, std::string path, - void (*callback)(std::string, std::string)); - - -}; - -} // namespace ns3 - -#endif // FREEBSD_STACK_HELPER_H diff --git a/model/dce.cc b/model/dce.cc index 54810624..4d304ac0 100644 --- a/model/dce.cc +++ b/model/dce.cc @@ -697,7 +697,7 @@ unsigned dce_if_nametoindex (const char *ifname) // Kernel stacks know their interfaces; ask them. TypeId tid = factory->GetInstanceTypeId (); if (tid == TypeId::LookupByName ("ns3::LinuxSocketFdFactory") - || tid.GetName () == "ns3::LklSocketFdFactory") + || tid.GetName () == "ns3::KernelSocketFdFactory") { struct ifreq ifr; int fd = dce_socket (AF_INET, SOCK_DGRAM, 0); diff --git a/model/freebsd-socket-fd-factory.cc b/model/freebsd-socket-fd-factory.cc deleted file mode 100644 index 70d0f81d..00000000 --- a/model/freebsd-socket-fd-factory.cc +++ /dev/null @@ -1,174 +0,0 @@ -#include "freebsd-socket-fd-factory.h" -#include "kernel-socket-fd-factory.h" -#include "loader-factory.h" -#include "kernel-socket-fd.h" -#include "utils.h" -#include "include/sim-init.h" -#include "ns3/log.h" -#include "ns3/string.h" -#include "ns3/simulator.h" -#include "ns3/node.h" - -NS_LOG_COMPONENT_DEFINE ("DceFreeBSDSocketFdFactory"); - -namespace ns3 { -NS_OBJECT_ENSURE_REGISTERED (FreeBSDSocketFdFactory); - -TypeId -FreeBSDSocketFdFactory::GetTypeId (void) -{ - static TypeId tid = TypeId ("ns3::FreeBSDSocketFdFactory") - .SetParent () - .AddConstructor () - .AddAttribute ("Library", "File to load in memory", - StringValue ("libfreebsd.so"), - MakeStringAccessor (&FreeBSDSocketFdFactory::m_library), - MakeStringChecker ()) - ; - return tid; -} -FreeBSDSocketFdFactory::FreeBSDSocketFdFactory () -{ -} - -FreeBSDSocketFdFactory::~FreeBSDSocketFdFactory () -{ -} - -void -FreeBSDSocketFdFactory::NotifyNewAggregate (void) -{ - NS_LOG_FUNCTION (this); - Ptr node = this->GetObject (); - Ptr loaderFactory = this->GetObject (); - Ptr taskManager = this->GetObject (); - if (node && loaderFactory && taskManager && m_loader == 0) - { - m_manager = taskManager; - m_loader = loaderFactory->Create (0, 0, 0); - UtilsEnsureDirectoryExists (UtilsGetAbsRealFilePath (node->GetId (), "/var")); - UtilsEnsureDirectoryExists (UtilsGetAbsRealFilePath (node->GetId (), "/var/log")); - std::string path = UtilsGetAbsRealFilePath (node->GetId (), "/var/log/messages"); - m_logFile = fopen (path.c_str (), "w"); - setlinebuf (m_logFile); - // must use ScheduleWithContext to ensure that the initialization task gets - // a node context to be able to retrieve the task manager when it runs. - // i.e., TaskManager::Current() needs it. - Simulator::ScheduleWithContext (node->GetId (), Seconds (0.0), - &FreeBSDSocketFdFactory::ScheduleTask, this, - MakeEvent (&FreeBSDSocketFdFactory::InitializeStack, this)); - } -} - -std::vector > -FreeBSDSocketFdFactory::GetSysFileList (void) -{ - struct MyIterator - { - struct SimSysIterator head; - static void ReportStartDir (const struct SimSysIterator *iter, const char *dirname) - { - struct MyIterator *self = (struct MyIterator *)iter; - self->m_stack.push_back (self->m_currentPath); - self->m_currentPath += "." + std::string (dirname); - } - static void ReportEndDir (const struct SimSysIterator *iter) - { - struct MyIterator *self = (struct MyIterator *)iter; - self->m_currentPath = self->m_stack.back (); - self->m_stack.pop_back (); - } - static void ReportFile (const struct SimSysIterator *iter, const char *filename, int flags, struct SimSysFile *file) - { - struct MyIterator *self = (struct MyIterator *)iter; - std::string path = self->m_currentPath + "." + filename; - self->m_list.push_back (std::make_pair (path, file)); - } - std::vector m_stack; - std::vector > m_list; - std::string m_currentPath; - } iter; - iter.head.report_start_dir = &MyIterator::ReportStartDir; - iter.head.report_end_dir = &MyIterator::ReportEndDir; - iter.head.report_file = &MyIterator::ReportFile; -#ifdef FIXME - m_loader->NotifyStartExecute (); - m_exported->sys_iterate_files ((struct SimSysIterator *)&iter); - m_loader->NotifyEndExecute (); -#endif - return iter.m_list; -} - -void -FreeBSDSocketFdFactory::SetTask (std::string path, std::string value) -{ - NS_LOG_FUNCTION (path << value); - std::vector > files = GetSysFileList (); - for (uint32_t i = 0; i < files.size (); i++) - { - if (files[i].first == path) - { - const char *s = value.c_str (); - int toWrite = value.size (); - int written; - written = m_exported->sys_file_write (files[i].second, s, toWrite, 0); - break; - } - } -} - -void -FreeBSDSocketFdFactory::Set (std::string path, std::string value) -{ - if (!m_manager) - { - m_earlySysfs.push_back (std::make_pair (path,value)); - } - else - { - ScheduleTask (MakeEvent (&FreeBSDSocketFdFactory::SetTask, this, path, value)); - } -} - -std::string -FreeBSDSocketFdFactory::Get (std::string path) -{ - NS_LOG_FUNCTION (path); - std::string ret; - std::vector > files = GetSysFileList (); - for (uint32_t i = 0; i < files.size (); i++) - { - if (files[i].first == path) - { - char buffer[512]; - memset (buffer, 0, sizeof(buffer)); - m_exported->sys_file_read (files[i].second, buffer, sizeof(buffer), 0); - NS_LOG_FUNCTION ("sysctl read: " << buffer); - ret = std::string (buffer); - break; - } - } - return ret; -} - - -void -FreeBSDSocketFdFactory::InitializeStack (void) -{ - KernelSocketFdFactory::InitializeStack (); - -#ifdef NOT_YET_SUPPORTED - Set (".net.ipv4.conf.all.forwarding", "1"); - Set (".net.ipv4.conf.all.log_martians", "1"); - Set (".net.ipv6.conf.all.forwarding", "0"); - - while (!m_earlySysfs.empty ()) - { - std::pair op = m_earlySysfs.front (); - Set (op.first, op.second); - m_earlySysfs.pop_front (); - } -#endif -} - -} // namespace ns3 diff --git a/model/freebsd-socket-fd-factory.h b/model/freebsd-socket-fd-factory.h deleted file mode 100644 index cbfc16a9..00000000 --- a/model/freebsd-socket-fd-factory.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef FREEBSD_SOCKET_FD_FACTORY_H -#define FREEBSD_SOCKET_FD_FACTORY_H - -#include "kernel-socket-fd-factory.h" -#include - -extern "C" { -struct SimExported; -struct SimDevice; -struct SimSocket; -struct SimTask; -struct SimKernel; -struct SimSysFile; -} - -namespace ns3 { - -class FreeBSDSocketFdFactory : public KernelSocketFdFactory -{ -public: - static TypeId GetTypeId (void); - FreeBSDSocketFdFactory (); - virtual ~FreeBSDSocketFdFactory (); - - void Set (std::string path, std::string value); - std::string Get (std::string path); - -private: - virtual void NotifyNewAggregate (void); - void InitializeStack (void); - std::vector > GetSysFileList (void); - void SetTask (std::string path, std::string value); - - std::list > m_earlySysfs; -}; - -} // namespace ns3 - -#endif /* FREEBSD_SOCKET_FD_FACTORY_H */ diff --git a/model/freebsd/ipv4-freebsd.cc b/model/freebsd/ipv4-freebsd.cc deleted file mode 100644 index 1574878a..00000000 --- a/model/freebsd/ipv4-freebsd.cc +++ /dev/null @@ -1,637 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -/* - * Copyright (c) 2014 Hajime Tazaki - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation; - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Author: Hajime Tazaki - */ - -#include "ipv4-freebsd.h" -#include "ns3/log.h" -#include "ns3/node.h" -#include "ns3/ipv4-list-routing-helper.h" -#include "ns3/ipv4-static-routing-helper.h" -#include "ns3/ipv4-global-routing-helper.h" -#include "ns3/ipv4-interface.h" -#include "ns3/arp-l3-protocol.h" -#include "ns3/global-routing.h" -#include "ns3/ipv4-routing-table-entry.h" -#include "dce-application-helper.h" -#include "freebsd-socket-fd-factory.h" -#include "kernel-socket-fd.h" -#ifdef NOT_SUPPORTED_YET -#include "freebsd-ipv4-raw-socket-factory-impl.h" -#include "freebsd-udp-socket-factory-impl.h" -#include "freebsd-tcp-socket-factory-impl.h" -#include "freebsd-dccp-socket-factory-impl.h" -#endif - -NS_LOG_COMPONENT_DEFINE ("DceIpv4FreeBSD"); - - -namespace ns3 { - -NS_OBJECT_ENSURE_REGISTERED (Ipv4FreeBSD); - -static void -RunIp (Ptr node, Time at, std::string str) -{ - DceApplicationHelper process; - ApplicationContainer apps; - process.SetBinary ("freebsd-iproute"); - process.SetStackSize (1 << 16); - process.ResetArguments (); - process.ParseArguments (str.c_str ()); - apps = process.Install (node); - apps.Start (at); -} - -TypeId -Ipv4FreeBSD::GetTypeId (void) -{ - static TypeId tid = TypeId ("ns3::Ipv4FreeBSD") - .SetParent () - .AddConstructor (); - - return tid; -} - -Ipv4FreeBSD::Ipv4FreeBSD () - : m_nanoSec (0) -{ - NS_LOG_FUNCTION (this); -} - -Ipv4FreeBSD::~Ipv4FreeBSD () -{ - NS_LOG_FUNCTION (this); -} - -void -Ipv4FreeBSD::SetRoutingProtocol (Ptr routingProtocol) -{ - NS_LOG_FUNCTION (this); - m_routingProtocol = routingProtocol; - m_routingProtocol->SetIpv4 (this); -} - -Ptr -Ipv4FreeBSD::GetRoutingProtocol (void) const -{ - NS_LOG_FUNCTION (this); - return m_routingProtocol; -} - -uint32_t -Ipv4FreeBSD::AddInterface (Ptr device) -{ - NS_LOG_FUNCTION (this << &device); - - Ptr node = GetObject (); - Ptr interface = CreateObject (); - interface->SetNode (node); - interface->SetDevice (device); - interface->SetForwarding (m_ipForward); - return AddIpv4Interface (interface); -} -uint32_t -Ipv4FreeBSD::AddIpv4Interface (Ptrinterface) -{ - NS_LOG_FUNCTION (this << interface); - uint32_t index = m_interfaces.size (); - m_interfaces.push_back (interface); - return index; -} - -uint32_t -Ipv4FreeBSD::GetNInterfaces (void) const -{ - NS_LOG_FUNCTION (this); - return m_interfaces.size (); -} - -int32_t -Ipv4FreeBSD::GetInterfaceForAddress (Ipv4Address address) const -{ - NS_LOG_FUNCTION (this); - - int32_t interface = 0; - for (Ipv4InterfaceList::const_iterator i = m_interfaces.begin (); - i != m_interfaces.end (); - i++, interface++) - { - for (uint32_t j = 0; j < (*i)->GetNAddresses (); j++) - { - if ((*i)->GetAddress (j).GetLocal () == address) - { - return interface; - } - } - } - - return -1; -} - -void -Ipv4FreeBSD::Send (Ptr packet, Ipv4Address source, - Ipv4Address destination, uint8_t protocol, Ptr route) -{ - NS_LOG_FUNCTION (this << "empty method."); -} - -void -Ipv4FreeBSD::SendWithHeader (Ptr packet, Ipv4Header ipHeader, Ptr route) -{ - NS_LOG_FUNCTION (this << "empty method."); -} - -void -Ipv4FreeBSD::Insert (Ptr protocol) -{ - NS_LOG_FUNCTION (this << "empty method."); -} - -void -Ipv4FreeBSD::Insert (Ptr protocol, uint32_t interfaceIndex) -{ - NS_LOG_FUNCTION (this << "empty method."); -} - -void -Ipv4FreeBSD::Remove (Ptr protocol) -{ - NS_LOG_FUNCTION (this << "empty method."); -} - -void -Ipv4FreeBSD::Remove (Ptr protocol, uint32_t interfaceIndex) -{ - NS_LOG_FUNCTION (this << "empty method."); -} - -bool -Ipv4FreeBSD::IsDestinationAddress (Ipv4Address address, uint32_t iif) const -{ - // First check the incoming interface for a unicast address match - for (uint32_t i = 0; i < GetNAddresses (iif); i++) - { - Ipv4InterfaceAddress iaddr = GetAddress (iif, i); - if (address == iaddr.GetLocal ()) - { - NS_LOG_LOGIC ("For me (destination " << address << " match)"); - return true; - } - if (address == iaddr.GetBroadcast ()) - { - NS_LOG_LOGIC ("For me (interface broadcast address)"); - return true; - } - } - - if (address.IsMulticast ()) - { -#ifdef NOTYET - if (MulticastCheckGroup (iif, address)) -#endif - if (true) - { - NS_LOG_LOGIC ("For me (Ipv4Addr multicast address"); - return true; - } - } - - if (address.IsBroadcast ()) - { - NS_LOG_LOGIC ("For me (Ipv4Addr broadcast address)"); - return true; - } - - if (GetWeakEsModel ()) // Check other interfaces - { - for (uint32_t j = 0; j < GetNInterfaces (); j++) - { - if (j == uint32_t (iif)) - { - continue; - } - for (uint32_t i = 0; i < GetNAddresses (j); i++) - { - Ipv4InterfaceAddress iaddr = GetAddress (j, i); - if (address == iaddr.GetLocal ()) - { - NS_LOG_LOGIC ("For me (destination " << address << " match) on another interface"); - return true; - } - // This is a small corner case: match another interface's broadcast address - if (address == iaddr.GetBroadcast ()) - { - NS_LOG_LOGIC ("For me (interface broadcast address on another interface)"); - return true; - } - } - } - } - return false; -} - -int32_t -Ipv4FreeBSD::GetInterfaceForPrefix (Ipv4Address address, Ipv4Mask mask) const -{ - NS_LOG_FUNCTION (this); - int32_t interface = 0; - for (Ipv4InterfaceList::const_iterator i = m_interfaces.begin (); - i != m_interfaces.end (); - i++, interface++) - { - for (uint32_t j = 0; j < (*i)->GetNAddresses (); j++) - { - if ((*i)->GetAddress (j).GetLocal ().CombineMask (mask) == address.CombineMask (mask)) - { - return interface; - } - } - } - - return -1; -} - -Ptr -Ipv4FreeBSD::GetNetDevice (uint32_t i) -{ - NS_LOG_FUNCTION (this); - return GetInterface (i)->GetDevice (); -} - -int32_t -Ipv4FreeBSD::GetInterfaceForDevice (Ptr device) const -{ - NS_LOG_FUNCTION (this); - int32_t interface = 0; - for (Ipv4InterfaceList::const_iterator i = m_interfaces.begin (); - i != m_interfaces.end (); - i++, interface++) - { - if ((*i)->GetDevice () == device) - { - return interface; - } - } - - return -1; -} - -static void -Ipv4Conf (Ptr node, uint32_t i, Ipv4InterfaceAddress address) -{ - -} - -bool -Ipv4FreeBSD::AddAddress (uint32_t i, Ipv4InterfaceAddress address) -{ - NS_LOG_FUNCTION (this << i << address); - Ptr interface = GetInterface (i); - bool retVal = interface->AddAddress (address); - if (m_routingProtocol) - { - m_routingProtocol->NotifyAddAddress (i, address); - } - - std::ostringstream oss; - Ptr node = this->GetObject (); - - oss << "sim" << i << " "; - address.GetLocal ().Print (oss); - oss << " "; - address.GetMask ().Print (oss); - RunIp (node, NanoSeconds (++m_nanoSec), oss.str ()); - - return retVal; -} - -uint32_t -Ipv4FreeBSD::GetNAddresses (uint32_t interface) const -{ - Ptr iface = GetInterface (interface); - return iface->GetNAddresses (); -} - -Ipv4InterfaceAddress -Ipv4FreeBSD::GetAddress (uint32_t interfaceIndex, uint32_t addressIndex) const -{ - NS_LOG_FUNCTION (this); - Ptr interface = GetInterface (interfaceIndex); - return interface->GetAddress (addressIndex); -} - -bool -Ipv4FreeBSD::RemoveAddress (uint32_t i, uint32_t addressIndex) -{ - NS_LOG_FUNCTION (this << i << addressIndex); - Ptr interface = GetInterface (i); - Ipv4InterfaceAddress address = interface->RemoveAddress (addressIndex); - if (address != Ipv4InterfaceAddress ()) - { - if (m_routingProtocol) - { - m_routingProtocol->NotifyRemoveAddress (i, address); - } - return true; - } - return false; -} - -bool -Ipv4FreeBSD::RemoveAddress (uint32_t i, Ipv4Address address) -{ - NS_LOG_FUNCTION (this << i << address); - - if (address == Ipv4Address::GetLoopback()) - { - NS_LOG_WARN ("Cannot remove loopback address."); - return false; - } - Ptr interface = GetInterface (i); - Ipv4InterfaceAddress ifAddr = interface->RemoveAddress (address); - if (ifAddr != Ipv4InterfaceAddress ()) - { - if (m_routingProtocol) - { - m_routingProtocol->NotifyRemoveAddress (i, ifAddr); - } - return true; - } - return false; -} - -Ipv4Address -Ipv4FreeBSD::SourceAddressSelection (uint32_t interfaceIdx, Ipv4Address dest) -{ - return 0; -} - -Ipv4Address -Ipv4FreeBSD::SelectSourceAddress (Ptr device, - Ipv4Address dst, Ipv4InterfaceAddress::InterfaceAddressScope_e scope) -{ - NS_LOG_FUNCTION (this << device << dst << scope); - Ipv4Address addr ("0.0.0.0"); - Ipv4InterfaceAddress iaddr; - bool found = false; - - if (device) - { - int32_t i = GetInterfaceForDevice (device); - NS_ASSERT_MSG (i >= 0, "No device found on node"); - for (uint32_t j = 0; j < GetNAddresses (i); j++) - { - iaddr = GetAddress (i, j); - if (iaddr.GetScope () > scope) - { - continue; - } - if (dst.CombineMask (iaddr.GetMask ()) == iaddr.GetLocal ().CombineMask (iaddr.GetMask ()) ) - { - return iaddr.GetLocal (); - } - if (!found) - { - addr = iaddr.GetLocal (); - found = true; - } - } - } - if (found) - { - return addr; - } - - // Iterate among all interfaces - for (uint32_t i = 0; i < GetNInterfaces (); i++) - { - for (uint32_t j = 0; j < GetNAddresses (i); j++) - { - iaddr = GetAddress (i, j); - if (iaddr.GetScope () != Ipv4InterfaceAddress::LINK - && iaddr.GetScope () <= scope) - { - return iaddr.GetLocal (); - } - } - } - NS_LOG_WARN ("Could not find source address for " << dst << " and scope " - << scope << ", returning 0"); - return addr; -} - -void -Ipv4FreeBSD::SetMetric (uint32_t i, uint16_t metric) -{ - NS_LOG_FUNCTION (this << i << metric); - Ptr interface = GetInterface (i); - interface->SetMetric (metric); -} - -uint16_t -Ipv4FreeBSD::GetMetric (uint32_t i) const -{ - Ptr interface = GetInterface (i); - return interface->GetMetric (); -} - -uint16_t -Ipv4FreeBSD::GetMtu (uint32_t i) const -{ - Ptr interface = GetInterface (i); - return interface->GetDevice ()->GetMtu (); -} - -bool -Ipv4FreeBSD::IsUp (uint32_t i) const -{ - Ptr interface = GetInterface (i); - return interface->IsUp (); -} - -void -Ipv4FreeBSD::SetUp (uint32_t i) -{ - NS_LOG_FUNCTION (this << i); - Ptr interface = GetInterface (i); - interface->SetUp (); - - if (m_routingProtocol) - { - m_routingProtocol->NotifyInterfaceUp (i); - } -} - -void -Ipv4FreeBSD::SetDown (uint32_t ifaceIndex) -{ - NS_LOG_FUNCTION (this << ifaceIndex); - Ptr interface = GetInterface (ifaceIndex); - interface->SetDown (); - - if (m_routingProtocol) - { - m_routingProtocol->NotifyInterfaceDown (ifaceIndex); - } -} - -bool -Ipv4FreeBSD::IsForwarding (uint32_t i) const -{ - NS_LOG_FUNCTION (this << i); - Ptr interface = GetInterface (i); - NS_LOG_LOGIC ("Forwarding state: " << interface->IsForwarding ()); - return interface->IsForwarding (); -} - -void -Ipv4FreeBSD::SetForwarding (uint32_t i, bool val) -{ - NS_LOG_FUNCTION (this << i); - Ptr interface = GetInterface (i); - interface->SetForwarding (val); -} - -void -Ipv4FreeBSD::SetIpForward (bool forward) -{ - NS_LOG_FUNCTION (this << forward); - m_ipForward = forward; - for (Ipv4InterfaceList::const_iterator i = m_interfaces.begin (); i != m_interfaces.end (); i++) - { - (*i)->SetForwarding (forward); - } -} - -bool -Ipv4FreeBSD::GetIpForward (void) const -{ - return m_ipForward; -} - -void -Ipv4FreeBSD::SetWeakEsModel (bool model) -{ - m_weakEsModel = model; -} - -bool -Ipv4FreeBSD::GetWeakEsModel (void) const -{ - return m_weakEsModel; -} - -void -Ipv4FreeBSD::SetStrongEndSystemModel(bool model) -{ - m_strongEsModel = model; -} - -bool -Ipv4FreeBSD::GetStrongEndSystemModel() const -{ - return m_strongEsModel; -} - -void -Ipv4FreeBSD::InstallNode (Ptr node) -{ - ObjectFactory factory; - factory.SetTypeId ("ns3::Ipv4FreeBSD"); - Ptr protocol = factory.Create (); - node->AggregateObject (protocol); - // ns3::Ipv4Interface creates an ARP cache for devices that need ARP once it - // knows its node. The FreeBSD kernel does ARP itself, so this ArpL3Protocol - // is never registered as a protocol handler and stays inert. - if (!node->GetObject ()) - { - node->AggregateObject (CreateObject ()); - } - Ipv4GlobalRoutingHelper globalRouting; - // Set routing - Ptr ipv4 = node->GetObject (); - Ptr ipv4Routing = globalRouting.Create (node); - ipv4->SetRoutingProtocol (ipv4Routing); -#ifdef NOT_SUPPORTED_YET - // Socket related stuff - Ptr ipv4rawFactory = CreateObject (); - node->AggregateObject (ipv4rawFactory); - Ptr udpFactory = CreateObject (); - node->AggregateObject (udpFactory); - Ptr tcpFactory = CreateObject (); - node->AggregateObject (tcpFactory); - Ptr dccpFactory = CreateObject (); - node->AggregateObject (dccpFactory); -#endif -} -Ptr -Ipv4FreeBSD::GetInterface (uint32_t index) const -{ - if (index < m_interfaces.size ()) - { - return m_interfaces[index]; - } - return 0; -} - -Ptr -Ipv4FreeBSD::GetProtocol (int protocolNumber) const -{ - return 0; -} - -Ptr -Ipv4FreeBSD::GetProtocol (int protocolNumber, int32_t interfaceIndex) const -{ - return 0; -} - -Ptr -Ipv4FreeBSD::CreateRawSocket (void) -{ - return 0; -} - -/** - * Do nothing - */ -void -Ipv4FreeBSD::DeleteRawSocket (Ptr socket) -{ - -} - -void -Ipv4FreeBSD::PopulateRoutingTable () -{ - // Only support Ipv4GlobalRouting - Ptr node = this->GetObject (); - Ptr globalRouting = DynamicCast (GetRoutingProtocol ()); - NS_ASSERT_MSG (globalRouting, "No global routing"); - - for (uint32_t i = 0; i < globalRouting->GetNRoutes (); i++) - { - Ipv4RoutingTableEntry route = globalRouting->GetRoute (i); - std::ostringstream oss; - oss << "route add to " << route.GetDest () << '/' << route.GetDestNetworkMask () << " via " << route.GetGateway (); - RunIp (node, NanoSeconds (++m_nanoSec), oss.str ()); - } -} - -} diff --git a/model/freebsd/ipv4-freebsd.h b/model/freebsd/ipv4-freebsd.h deleted file mode 100644 index e1c39d46..00000000 --- a/model/freebsd/ipv4-freebsd.h +++ /dev/null @@ -1,373 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -/* - * Copyright (c) 2014 Hajime Tazaki - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation; - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Author: Hajime Tazaki - */ - -#ifndef IPV4_FREEBSD_H -#define IPV4_FREEBSD_H - -#include "ns3/ipv4.h" -#include "ns3/ipv4-routing-protocol.h" -#include "ns3/ip-l4-protocol.h" - -namespace ns3 { -class Ipv4Interface; - -/** - * \brief This implementation of Ipv4 for nodes using a real FreeBSD Stack. - * - * The main goal of this class is not to do the stack job which is done by the real code .... - * but it is used to not redo the wheel and in particular to be able to reuse these 2 standards NS-3 Helpers for two tasks: - * 1 - assign address to devices using Ipv4AddressHelper, - * 2 - create the static routes using Ipv4GlobalRoutingHelper - * - */ -class Ipv4FreeBSD : public Ipv4 -{ -public: - static TypeId GetTypeId (void); - Ipv4FreeBSD (); - virtual ~Ipv4FreeBSD (); - - /** - * \brief Register a new routing protocol to be used by this Ipv4 stack - * - * This call will replace any routing protocol that has been previously - * registered. If you want to add multiple routing protocols, you must - * add them to a Ipv4ListRoutingProtocol directly. - * - * \param routingProtocol smart pointer to Ipv4RoutingProtocol object - */ - virtual void SetRoutingProtocol (Ptr routingProtocol); - - /** - * \brief Get the routing protocol to be used by this Ipv4 stack - * - * \returns smart pointer to Ipv4RoutingProtocol object, or null pointer if none - */ - virtual Ptr GetRoutingProtocol (void) const; - - /** - * \param device device to add to the list of Ipv4 interfaces - * which can be used as output interfaces during packet forwarding. - * \returns the index of the Ipv4 interface added. - * - * Once a device has been added, it can never be removed: if you want - * to disable it, you can invoke Ipv4::SetDown which will - * make sure that it is never used during packet forwarding. - */ - virtual uint32_t AddInterface (Ptr device); - - /** - * \returns the number of interfaces added by the user. - */ - virtual uint32_t GetNInterfaces (void) const; - - /** - * \brief Return the interface number of the interface that has been - * assigned the specified IP address. - * - * \param address The IP address being searched for - * \returns The interface number of the Ipv4 interface with the given - * address or -1 if not found. - * - * Each IP interface has one or more IP addresses associated with it. - * This method searches the list of interfaces for one that holds a - * particular address. This call takes an IP address as a parameter and - * returns the interface number of the first interface that has been assigned - * that address, or -1 if not found. There must be an exact match; this - * method will not match broadcast or multicast addresses. - */ - virtual int32_t GetInterfaceForAddress (Ipv4Address address) const; - - /** - * Do nothing - */ - virtual void Send (Ptr packet, Ipv4Address source, - Ipv4Address destination, uint8_t protocol, Ptr route); - - /** - * Do nothing - */ - virtual void SendWithHeader (Ptr packet, Ipv4Header ipHeader, Ptr route); - - /** - * Do nothing - */ - virtual void Insert (Ptr protocol); - - /** - * Do nothing - */ - virtual void Insert (Ptr protocol, uint32_t interfaceIndex); - - /** - * Do nothing - */ - virtual void Remove (Ptr protocol); - - /** - * Do nothing - */ - virtual void Remove (Ptr protocol, uint32_t interfaceIndex); - - /** - * \brief Determine whether address and interface corresponding to - * received packet can be accepted for local delivery - * - * \param address The IP address being considered - * \param iif The incoming Ipv4 interface index - * - * This method can be used to determine whether a received packet has - * an acceptable address for local delivery on the host. The address - * may be a unicast, multicast, or broadcast address. This method will - * return true if address is an exact match of a unicast address on - * one of the host's interfaces (see below), if address corresponds to - * a multicast group that the host has joined (and the incoming device - * is acceptable), or if address corresponds to a broadcast address. - * - * If the Ipv4 attribute WeakEsModel is true, the unicast address may - * match any of the Ipv4 addresses on any interface. If the attribute is - * false, the address must match one assigned to the incoming device. - */ - virtual bool IsDestinationAddress (Ipv4Address address, uint32_t iif) const; - - /** - * \brief Return the interface number of first interface found that - * has an Ipv4 address within the prefix specified by the input - * address and mask parameters - * - * \param address The IP address assigned to the interface of interest. - * \param mask The IP prefix to use in the mask - * \returns The interface number of the Ipv4 interface with the given - * address or -1 if not found. - * - * Each IP interface has one or more IP addresses associated with it. - * This method searches the list of interfaces for the first one found - * that holds an address that is included within the prefix - * formed by the input address and mask parameters. The value -1 is - * returned if no match is found. - */ - virtual int32_t GetInterfaceForPrefix (Ipv4Address address, - Ipv4Mask mask) const; - - /** - * \param interface The interface number of an Ipv4 interface. - * \returns The NetDevice associated with the Ipv4 interface number. - */ - virtual Ptr GetNetDevice (uint32_t interface); - - /** - * \param device The NetDevice for an Ipv4Interface - * \returns The interface number of an Ipv4 interface or -1 if not found. - */ - virtual int32_t GetInterfaceForDevice (Ptr device) const; - - /** - * \param interface Interface number of an Ipv4 interface - * \param address Ipv4InterfaceAddress address to associate with the underlying Ipv4 interface - * \returns true if the operation succeeded - */ - virtual bool AddAddress (uint32_t interface, Ipv4InterfaceAddress address); - - /** - * \param interface Interface number of an Ipv4 interface - * \returns the number of Ipv4InterfaceAddress entries for the interface. - */ - virtual uint32_t GetNAddresses (uint32_t interface) const; - - /** - * Because addresses can be removed, the addressIndex is not guaranteed - * to be static across calls to this method. - * - * \param interface Interface number of an Ipv4 interface - * \param addressIndex index of Ipv4InterfaceAddress - * \returns the Ipv4InterfaceAddress associated to the interface and addressIndex - */ - virtual Ipv4InterfaceAddress GetAddress (uint32_t interface, uint32_t addressIndex) const; - - /** - * Remove the address at addressIndex on named interface. The addressIndex - * for all higher indices will decrement by one after this method is called; - * so, for example, to remove 5 addresses from an interface i, one could - * call RemoveAddress (i, 0); 5 times. - * - * \param interface Interface number of an Ipv4 interface - * \param addressIndex index of Ipv4InterfaceAddress to remove - * \returns true if the operation succeeded - */ - virtual bool RemoveAddress (uint32_t interface, uint32_t addressIndex); - - /** - * \brief Remove the given address on named Ipv4 interface - * - * \param interface Interface number of an Ipv4 interface - * \param address The address to remove - * \returns true if the operation succeeded - */ - virtual bool RemoveAddress (uint32_t interface, Ipv4Address address); - - /** - * \brief Return the first primary source address with scope less than - * or equal to the requested scope, to use in sending a packet to - * destination dst out of the specified device. - * - * This method mirrors the behavior of FreeBSD inet_select_addr() and is - * provided because interfaces may have multiple IP addresses configured - * on them with different scopes, and with a primary and secondary status. - * Secondary addresses are never returned. - * \see Ipv4InterfaceAddress - * - * If a non-zero device pointer is provided, the method first tries to - * return a primary address that is configured on that device, and whose - * subnet matches that of dst and whose scope is less than or equal to - * the requested scope. If a primary address does not match the - * subnet of dst but otherwise matches the scope, it is returned. - * If no such address on the device is found, the other devices are - * searched in order of their interface index, but not considering dst - * as a factor in the search. Because a loopback interface is typically - * the first one configured on a node, it will be the first alternate - * device to be tried. Addresses scoped at LINK scope are not returned - * in this phase. - * - * If no device pointer is provided, the same logic as above applies, only - * that there is no preferred device that is consulted first. This means - * that if the device pointer is null, input parameter dst will be ignored. - * - * If there are no possible addresses to return, a warning log message - * is issued and the all-zeroes address is returned. - * - * \param device output NetDevice (optionally provided, only to constrain the search) - * \param dst Destination address to match, if device is provided - * \param scope Scope of returned address must be less than or equal to this - * \returns the first primary Ipv4Address that meets the search criteria - */ - virtual Ipv4Address SelectSourceAddress (Ptr device, - Ipv4Address dst, Ipv4InterfaceAddress::InterfaceAddressScope_e scope); - - /** - * \param interface The interface number of an Ipv4 interface - * \param metric routing metric (cost) associated to the underlying - * Ipv4 interface - */ - virtual void SetMetric (uint32_t interface, uint16_t metric); - - /** - * \param interface The interface number of an Ipv4 interface - * \returns routing metric (cost) associated to the underlying - * Ipv4 interface - */ - virtual uint16_t GetMetric (uint32_t interface) const; - - /** - * \param interface Interface number of Ipv4 interface - * \returns the Maximum Transmission Unit (in bytes) associated - * to the underlying Ipv4 interface - */ - virtual uint16_t GetMtu (uint32_t interface) const; - - /** - * \param interface Interface number of Ipv4 interface - * \returns true if the underlying interface is in the "up" state, - * false otherwise. - */ - virtual bool IsUp (uint32_t interface) const; - - /** - * \param interface Interface number of Ipv4 interface - * - * Set the interface into the "up" state. In this state, it is - * considered valid during Ipv4 forwarding. - */ - virtual void SetUp (uint32_t interface); - - /** - * \param interface Interface number of Ipv4 interface - * - * Set the interface into the "down" state. In this state, it is - * ignored during Ipv4 forwarding. - */ - virtual void SetDown (uint32_t interface); - - /** - * \param interface Interface number of Ipv4 interface - * \returns true if IP forwarding enabled for input datagrams on this device - */ - virtual bool IsForwarding (uint32_t interface) const; - - /** - * \param interface Interface number of Ipv4 interface - * \param val Value to set the forwarding flag - * - * If set to true, IP forwarding is enabled for input datagrams on this device - */ - virtual void SetForwarding (uint32_t interface, bool val); - - /** - * Do nothing - */ - virtual Ipv4Address SourceAddressSelection (uint32_t interface, Ipv4Address dest); - - /** - * Do nothing - */ - virtual Ptr GetProtocol (int protocolNumber) const ; - - /** - * Do nothing - */ - virtual Ptr GetProtocol (int protocolNumber, int32_t interfaceIndex) const; - - /** - * Do nothing - */ - virtual Ptr CreateRawSocket (void); - - /** - * Do nothing - */ - virtual void DeleteRawSocket (Ptr socket); - - static void InstallNode (Ptr node); - void PopulateRoutingTable (); - -private: - // Indirect the Ipv4 attributes through private pure virtual methods - Ipv4FreeBSD (Ipv4FreeBSD const &); - Ipv4FreeBSD& operator= (Ipv4FreeBSD const &); - - - virtual void SetIpForward (bool forward); - virtual bool GetIpForward (void) const; - virtual void SetWeakEsModel (bool model); - virtual bool GetWeakEsModel (void) const; - virtual void SetStrongEndSystemModel(bool model); - virtual bool GetStrongEndSystemModel() const; - uint32_t AddIpv4Interface (Ptr interface); - Ptr GetInterface (uint32_t i) const; - - typedef std::vector > Ipv4InterfaceList; - Ptr m_routingProtocol; - bool m_ipForward; - bool m_weakEsModel; - bool m_strongEsModel; - Ipv4InterfaceList m_interfaces; - uint64_t m_nanoSec; -}; -} -#endif // IPV4_FREEBSD_H diff --git a/model/kernel-socket-fd-factory.cc b/model/kernel-socket-fd-factory.cc index 5346cb4b..5ed82811 100644 --- a/model/kernel-socket-fd-factory.cc +++ b/model/kernel-socket-fd-factory.cc @@ -1,964 +1,636 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ #include "kernel-socket-fd-factory.h" #include "kernel-socket-fd.h" -#include "loader-factory.h" -#include "dce-manager.h" -#include "process.h" -#include "utils.h" -#include "wait-queue.h" +#include "lkl-kernel.h" #include "task-manager.h" -#include "kingsley-alloc.h" -#include "file-usage.h" -#include "dce-unistd.h" -#include "dce-stdlib.h" -#include "sys/dce-stat.h" -#include "dce-fcntl.h" -#include "dce-stdio.h" -#include "include/sim-init.h" #include "ns3/log.h" -#include "ns3/string.h" -#include "ns3/double.h" -#include "ns3/pointer.h" #include "ns3/node.h" -#include "ns3/net-device.h" -#include "ns3/random-variable-stream.h" -#include "ns3/event-id.h" +#include "ns3/packet.h" #include "ns3/simulator.h" +#include "ns3/string.h" +#include "ns3/boolean.h" #include "ns3/mac48-address.h" -#include "ns3/mac64-address.h" -#include "ns3/packet.h" -#include "exec-utils.h" -#include -#include -#include -#include +#include "ns3/loopback-net-device.h" +#include "ns3/make-event.h" +#include +#include +#include +#include #include -#include #include +#include +#include +#include +#include +#include +#include NS_LOG_COMPONENT_DEFINE ("DceKernelSocketFdFactory"); -namespace ns3 { - -// Sadly NetDevice Callback add by method AddLinkChangeCallback take no parameters .. -// .. so we need to create the following class to link NetDevice and KernelSocketFdFactory together -// in order to do Warn the factory about which NetDevice is changing . -class KernelDeviceStateListener : public SimpleRefCount +/* + * LKL's network device interface (tools/lkl/include/lkl_host.h and lkl.h). + * Those headers are not valid C++, so the few definitions needed here are + * repeated; they must match the LKL version built by utils/build_lkl.sh. + */ +extern "C" { +struct lkl_netdev; +struct lkl_dev_net_ops +{ + int (*tx)(struct lkl_netdev *nd, struct iovec *iov, int cnt); + int (*rx)(struct lkl_netdev *nd, struct iovec *iov, int cnt); + int (*poll)(struct lkl_netdev *nd); + void (*poll_hup)(struct lkl_netdev *nd); + void (*free)(struct lkl_netdev *nd); +}; +struct lkl_netdev { -public: - KernelDeviceStateListener (Ptr, Ptr); - - void NotifyDeviceStateChange (); - -private: - Ptr m_netDevice; - Ptr m_factory; + struct lkl_dev_net_ops *ops; + int id; + uint8_t has_vnet_hdr : 1; + uint8_t mac[6]; }; - -KernelDeviceStateListener::KernelDeviceStateListener (Ptr d, - Ptr f) - : m_netDevice (d), - m_factory (f) +struct lkl_netdev_args { + void *mac; + unsigned int offload; +}; } +#define LKL_DEV_NET_POLL_RX 1 +#define LKL_DEV_NET_POLL_HUP 4 -void -KernelDeviceStateListener::NotifyDeviceStateChange () -{ - m_factory->NotifyDeviceStateChange (m_netDevice); -} +namespace ns3 { NS_OBJECT_ENSURE_REGISTERED (KernelSocketFdFactory); -TypeId -KernelSocketFdFactory::GetTypeId (void) +/** + * A kernel network device backed by an ns-3 NetDevice. LKL drives it + * through the network device host operations below. + */ +struct LklNetDevice +{ + struct lkl_netdev nd; // must stay first: LKL passes &nd to the operations + KernelSocketFdFactory *factory; + Ptr device; + std::deque > rx; // received Ethernet frames + Task *poller; + bool hup; + // The kernel had no receive buffers for the queued frames: it takes them + // when it adds buffers, so do not report them again. + bool stalled; +}; + +namespace { + +const uint32_t LKL_TASK_STACK = 1 << 18; +const uint32_t ETH_HEADER = 14; +// Received frames waiting for the kernel, like the ring of a network card. +const uint32_t RX_BACKLOG = 1024; + +LklNetDevice * +DeviceOf (struct lkl_netdev *nd) { - static TypeId tid = TypeId ("ns3::KernelSocketFdFactory") - .SetParent () - .AddConstructor () - .AddAttribute ("ErrorRate", "The error rate of malloc().", - DoubleValue (DoubleValue (0.0)), - MakeDoubleAccessor (&KernelSocketFdFactory::m_rate), - MakeDoubleChecker ()) - .AddAttribute ("RanVar", "The decision variable attached to this error model.", - StringValue ("ns3::UniformRandomVariable[Min=0.0|Max=1.0]"), - MakePointerAccessor (&KernelSocketFdFactory::m_ranvar), - MakePointerChecker ()) - ; - return tid; + return (LklNetDevice *)nd; } -KernelSocketFdFactory::KernelSocketFdFactory () - : m_loader (0), - m_exported (0), - m_alloc (new KingsleyAlloc ()), - m_logFile (0) + +void +SendMain (Ptr device, Ptr p, Mac48Address dest, uint16_t protocol) { - TypeId::LookupByNameFailSafe ("ns3::LteUeNetDevice", &m_lteUeTid); - m_variable = CreateObject (); + device->Send (p, dest, protocol); } -KernelSocketFdFactory::~KernelSocketFdFactory () +// The kernel sends a frame: hand it to the ns-3 device. +int +NetTx (struct lkl_netdev *nd, struct iovec *iov, int cnt) { - for (uint32_t i = 0; i < m_devices.size (); i++) + LklNetDevice *dev = DeviceOf (nd); + std::vector frame; + for (int i = 0; i < cnt; i++) { - // Note: we don't really destroy devices from here - // because calling destroy requires a task context - // m_exported->dev_destroy(m_devices[i].second); + const uint8_t *base = (const uint8_t *)iov[i].iov_base; + frame.insert (frame.end (), base, base + iov[i].iov_len); } - delete m_exported; - delete m_loader; - delete m_alloc; - if (m_logFile != 0) + if (frame.size () < ETH_HEADER) { - fclose (m_logFile); + return frame.size (); } - m_exported = 0; - m_loader = 0; - m_alloc = 0; - m_logFile = 0; + Mac48Address dest; + dest.CopyFrom (&frame[0]); + uint16_t protocol = (frame[12] << 8) | frame[13]; + NS_LOG_DEBUG ("tx " << dev->device << " " << dest << " protocol " << protocol + << " " << frame.size () << " bytes"); + Ptr p = Create (&frame[ETH_HEADER], frame.size () - ETH_HEADER); + TaskManager::Current ()->ExecOnMain (MakeEvent (&SendMain, dev->device, p, dest, protocol)); + return frame.size (); } -void -KernelSocketFdFactory::DoDispose (void) +// The kernel reads the next received frame, if any. +int +NetRx (struct lkl_netdev *nd, struct iovec *iov, int cnt) { - for (std::list::const_iterator i = m_kernelTasks.begin (); i != m_kernelTasks.end (); ++i) + LklNetDevice *dev = DeviceOf (nd); + if (dev->rx.empty ()) { - m_manager->Stop (*i); + return -1; } - m_kernelTasks.clear (); - m_manager = 0; - m_listeners.clear (); + std::vector &frame = dev->rx.front (); + size_t done = 0; + for (int i = 0; i < cnt && done < frame.size (); i++) + { + size_t n = std::min (iov[i].iov_len, frame.size () - done); + memcpy (iov[i].iov_base, &frame[done], n); + done += n; + } + dev->rx.pop_front (); + dev->stalled = false; + return done; } +// Blocks the device's poll task until the kernel can take a received frame. +// Sending never blocks, so transmission readiness is never reported. int -KernelSocketFdFactory::Vprintf (struct SimKernel *kernel, const char *str, va_list args) +NetPoll (struct lkl_netdev *nd) { - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - return vfprintf (self->m_logFile, str, args); -} - -void * -KernelSocketFdFactory::Malloc (struct SimKernel *kernel, unsigned long size) -{ - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - if (self->m_ranvar->GetValue () < self->m_rate) + LklNetDevice *dev = DeviceOf (nd); + TaskManager *manager = TaskManager::Current (); + while ((dev->rx.empty () || dev->stalled) && !dev->hup) { - NS_LOG_DEBUG ("return null"); - // Inject fault - return NULL; + dev->poller = manager->RunningTask (); + manager->Sleep (); } - - size += sizeof (size_t); - uint8_t *buffer = self->m_alloc->Malloc (size); - memcpy (buffer, &size, sizeof (size_t)); - buffer += sizeof (size_t); - return buffer; -} -void -KernelSocketFdFactory::Free (struct SimKernel *kernel, void *ptr) -{ - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - uint8_t *buffer = (uint8_t*)ptr; - size_t size; - buffer -= sizeof (size_t); - memcpy (&size, buffer, sizeof (size_t)); - self->m_alloc->Free (buffer, size); -} -void * -KernelSocketFdFactory::Memcpy (struct SimKernel *kernel, void *dst, const void *src, unsigned long size) -{ - return memcpy (dst, src, size); -} -void * -KernelSocketFdFactory::Memset (struct SimKernel *kernel, void *dst, char value, unsigned long size) -{ - return memset (dst, value, size); + dev->poller = 0; + if (dev->hup) + { + return LKL_DEV_NET_POLL_HUP; + } + // Until the kernel takes a frame or another one arrives. + dev->stalled = true; + return LKL_DEV_NET_POLL_RX; } -int -KernelSocketFdFactory::AtExit (struct SimKernel *kernel, void (*function)(void)) -{ - NS_LOG_FUNCTION (kernel << function); - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - Ptr manager = self->GetObject (); - Process *p = manager->SearchProcess (self->m_pid); - // Register process-level atexit store - struct AtExitHandler handler; - handler.type = AtExitHandler::NORMAL; - handler.value.normal = function; - p->atExitHandlers.push_back (handler); - return 0; -} -int -KernelSocketFdFactory::Access (struct SimKernel *kernel, const char *pathname, int mode) -{ - return dce_access (pathname, mode); -} -char* -KernelSocketFdFactory::Getenv (struct SimKernel *kernel, const char *name) -{ - return dce_getenv (name); -} -int -KernelSocketFdFactory::Mkdir (struct SimKernel *kernel, const char *pathname, mode_t mode) -{ - return dce_mkdir (pathname, mode); -} -int -KernelSocketFdFactory::Open (struct SimKernel *kernel, const char *pathname, int flags) -{ - return dce_open (pathname, flags, 0666); -} -int -KernelSocketFdFactory::__Fxstat (struct SimKernel *kernel, int ver, int fd, void *buf) -{ - return dce_fstat (fd, (struct stat *)buf); -} -int -KernelSocketFdFactory::Fseek (struct SimKernel *kernel, FILE *stream, long offset, int whence) -{ - return dce_fseek (stream, offset, whence); -} void -KernelSocketFdFactory::Setbuf (struct SimKernel *kernel, FILE *stream, char *buf) -{ - return dce_setbuf (stream, buf); -} -long -KernelSocketFdFactory::Ftell (struct SimKernel *kernel, FILE *stream) +NetPollHup (struct lkl_netdev *nd) { - return dce_ftell (stream); -} -FILE* -KernelSocketFdFactory::FdOpen (struct SimKernel *kernel, int fd, const char *mode) -{ - return dce_fdopen (fd, mode); -} -size_t -KernelSocketFdFactory::Fread (struct SimKernel *kernel, void *ptr, size_t size, size_t nmemb, FILE *stream) -{ - return dce_fread (ptr, size, nmemb, stream); -} -size_t -KernelSocketFdFactory::Fwrite (struct SimKernel *kernel, const void *ptr, size_t size, size_t nmemb, FILE *stream) -{ - return dce_fwrite (ptr, size, nmemb, stream); -} -int -KernelSocketFdFactory::Fclose (struct SimKernel *kernel, FILE *fp) -{ - return dce_fclose (fp); -} -unsigned long -KernelSocketFdFactory::Random (struct SimKernel *kernel) -{ - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - union - { - uint8_t buffer[sizeof(unsigned long)]; - unsigned long v; - } u; - for (uint8_t i = 0; i < sizeof (u.buffer); i++) + LklNetDevice *dev = DeviceOf (nd); + dev->hup = true; + if (dev->poller) { - u.buffer[i] = self->m_variable->GetInteger (0,255); + TaskManager::Current ()->Wakeup (dev->poller); } - return u.v; } + void -KernelSocketFdFactory::EventTrampoline (void (*fn)(void *context), - void *context, void (*pre_fn)(void), - Ptr event) +NetFree (struct lkl_netdev *nd) { - m_loader->NotifyStartExecute (); - pre_fn (); - fn (context); - m_loader->NotifyEndExecute (); + // The factory owns the device. } -void * -KernelSocketFdFactory::EventScheduleNs (struct SimKernel *kernel, __u64 ns, void (*fn)(void *context), void *context, - void (*pre_fn)(void)) -{ - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - Ptr ev = Create (); - TaskManager *manager = TaskManager::Current (); - ev->id = manager->ScheduleMain (NanoSeconds (ns), - MakeEvent (&KernelSocketFdFactory::EventTrampoline, self, fn, context, pre_fn, ev)); +struct lkl_dev_net_ops g_netOps = { + &NetTx, &NetRx, &NetPoll, &NetPollHup, &NetFree +}; - return &ev->id; -} -void -KernelSocketFdFactory::EventCancel (struct SimKernel *kernel, void *ev) -{ - EventId *event = (EventId *)ev; - Simulator::Remove (*event); -} -static __u64 CurrentNs (struct SimKernel *kernel) -{ - return Simulator::Now ().GetNanoSeconds (); -} +} // namespace -void -KernelSocketFdFactory::TaskSwitch (enum Task::SwitchType type, void *context) +TypeId +KernelSocketFdFactory::GetTypeId (void) { - NS_LOG_FUNCTION (type << context); - Loader *loader = (Loader *)context; - switch (type) - { - case Task::TO: - loader->NotifyStartExecute (); - break; - case Task::FROM: - loader->NotifyEndExecute (); - break; - } + static TypeId tid = TypeId ("ns3::KernelSocketFdFactory") + .SetParent () + .AddConstructor () + .AddAttribute ("Library", "The LKL shared library to load.", + StringValue ("liblkl.so"), + MakeStringAccessor (&KernelSocketFdFactory::m_library), + MakeStringChecker ()) + .AddAttribute ("CommandLine", "The kernel command line.", + StringValue ("mem=64M loglevel=4"), + MakeStringAccessor (&KernelSocketFdFactory::m_cmdline), + MakeStringChecker ()) + .AddAttribute ("Mptcp", + "Create MPTCP sockets for the TCP sockets of applications, " + "like mptcpize: Linux uses MPTCP only when asked for.", + BooleanValue (false), + MakeBooleanAccessor (&KernelSocketFdFactory::m_mptcp), + MakeBooleanChecker ()) + ; + return tid; } -struct SimTask * -KernelSocketFdFactory::TaskStart (struct SimKernel *kernel, void (*callback)(void *), void *context) +KernelSocketFdFactory::KernelSocketFdFactory () + : m_mptcp (false), + m_booted (false), + m_nextDevice (0), + m_epollFd (-1) { - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - Task *task = self->m_manager->Start (callback, context, 1 << 17); - struct SimTask *simTask = self->m_exported->task_create (task, 0); - task->SetExtraContext (simTask); - task->SetSwitchNotifier (&KernelSocketFdFactory::TaskSwitch, self->m_loader); - self->m_kernelTasks.push_back (task); - return (struct SimTask *)task->GetExtraContext (); } -struct SimTask * -KernelSocketFdFactory::TaskCurrent (struct SimKernel *kernel) + +KernelSocketFdFactory::~KernelSocketFdFactory () { - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - TaskManager *manager = TaskManager::Current (); - Task *current = manager->CurrentTask (); - if (current->GetExtraContext () == 0) + for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) { - uint32_t pid = 0; - struct Thread *thread = (struct Thread *)current->GetContext (); - if (thread != 0) - { - pid = thread->process->pid; - } - struct SimTask *simTask = self->m_exported->task_create (current, pid); - current->SetExtraContext (simTask); + delete *i; } - return (struct SimTask *)current->GetExtraContext (); } + void -KernelSocketFdFactory::TaskWait (struct SimKernel *kernel) -{ - // force initialization of 'current' - TaskCurrent (kernel); - // now, sleep. - TaskManager::Current ()->Sleep (); -} -int -KernelSocketFdFactory::TaskWakeup (struct SimKernel *kernel, struct SimTask *task) +KernelSocketFdFactory::DoDispose (void) { - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - TaskManager *manager = TaskManager::Current (); - if (!manager) + // The devices refer to their node, which refers to this factory. + for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) { - return 1; + (*i)->device = 0; } - Task *other = (Task *)self->m_exported->task_get_private (task); - bool isBlocked = other->IsBlocked (); - manager->Wakeup (other); - return isBlocked ? 1 : 0; + m_kernel = 0; + SocketFdFactory::DoDispose (); } -void -KernelSocketFdFactory::TaskYield (struct SimKernel *kernel) + +Ptr +KernelSocketFdFactory::GetKernel (void) const { - // force initialization of 'current' - TaskCurrent (kernel); - // now, yield. - TaskManager::Current ()->Yield (); + return m_kernel; } + void -KernelSocketFdFactory::SendMain (bool *r, NetDevice *dev, Ptr p, const Address& d, uint16_t pro) +KernelSocketFdFactory::NotifyNewAggregate (void) { - *r = dev->Send (p, d, pro); -} -void -KernelSocketFdFactory::DevXmit (struct SimKernel *kernel, struct SimDevice *dev, unsigned char *data, int len) -{ - NS_LOG_FUNCTION (dev); - KernelSocketFdFactory *self = (KernelSocketFdFactory *)kernel; - NetDevice *nsDev = (NetDevice *)self->m_exported->dev_get_private (dev); - NS_ASSERT (len >= 14); - - struct ethhdr - { - unsigned char h_dest[6]; - unsigned char h_source[6]; - uint16_t h_proto; - } *hdr = (struct ethhdr *)data; - data += 14; - len -= 14; - Ptr p = Create (data, len); - uint16_t protocol = ntohs (hdr->h_proto); - Mac48Address dest; - dest.CopyFrom (hdr->h_dest); - TaskManager *manager = TaskManager::Current (); - bool r = false; - - manager->ExecOnMain (MakeEvent (&KernelSocketFdFactory::SendMain, &r, nsDev, p, dest, protocol)); + Ptr node = GetObject (); + Ptr manager = GetObject (); + if (node && manager && !m_kernel) + { + m_kernel = CreateObject (); + // Scripts written for libos name its library; LKL replaces it. + std::string library = m_library == "liblinux.so" ? "liblkl.so" : m_library; + m_kernel->SetAttribute ("Library", StringValue (library)); + m_kernel->SetAttribute ("CommandLine", StringValue (m_cmdline)); + node->AggregateObject (m_kernel); + Simulator::ScheduleWithContext (node->GetId (), Seconds (0), + &TaskManager::Start, PeekPointer (manager), + &KernelSocketFdFactory::BootTrampoline, (void *)this, + LKL_TASK_STACK); + } + SocketFdFactory::NotifyNewAggregate (); } void -KernelSocketFdFactory::SignalRaised (struct SimKernel *kernel, struct SimTask *task, int signalNumber) +KernelSocketFdFactory::BootTrampoline (void *context) { - NS_LOG_FUNCTION ("XXX: Not Yet Implemented " << signalNumber); + ((KernelSocketFdFactory *)context)->Boot (); + TaskManager::Current ()->Exit (); } -struct SimDevice * -KernelSocketFdFactory::DevToDev (Ptr device) +void +KernelSocketFdFactory::Boot (void) { - for (uint32_t i = 0; i < m_devices.size (); i++) + NS_LOG_FUNCTION (this); + m_kernel->Boot (); + // procfs, for sysctls. + long ret = Call (__lkl__NR_mkdirat, AT_FDCWD, (long)"/proc", 0555); + NS_ASSERT_MSG (ret == 0, "mkdir /proc failed: " << ret); + ret = Call (__lkl__NR_mount, (long)"proc", (long)"/proc", (long)"proc", 0, 0); + NS_ASSERT_MSG (ret == 0, "mount /proc failed: " << ret); + m_epollFd = Call (__lkl__NR_epoll_create1, 0); + NS_ASSERT_MSG (m_epollFd >= 0, "epoll_create1 failed: " << m_epollFd); + m_kernel->GetTaskManager ()->Start (&KernelSocketFdFactory::WatcherTrampoline, this, LKL_TASK_STACK); + // Called right away for the devices the node already has. + GetObject ()->RegisterDeviceAdditionListener (MakeCallback (&KernelSocketFdFactory::NotifyAddDevice, this)); + m_booted = true; + for (std::list::iterator i = m_bootWaiters.begin (); i != m_bootWaiters.end (); ++i) { - if (m_devices[i].first == device) - { - struct SimDevice *dev = m_devices[i].second; - return dev; - } + m_kernel->GetTaskManager ()->Wakeup (*i); + } + m_bootWaiters.clear (); + for (std::list >::iterator i = m_earlySysctls.begin (); + i != m_earlySysctls.end (); ++i) + { + SetTask (i->first, i->second); } - return 0; + m_earlySysctls.clear (); } -void -KernelSocketFdFactory::RxFromDevice (Ptr device, Ptr p, - uint16_t protocol, const Address & from, - const Address &to, NetDevice::PacketType type) +namespace { +std::string +SysctlFile (std::string path) { - struct SimDevice *dev = DevToDev (device); - if (dev == 0) + if (!path.empty () && path[0] == '/') { - return; + return path; // any kernel file, e.g. /proc/net/netstat } - m_loader->NotifyStartExecute (); // Restore the memory of the kernel before access it ! - struct SimDevicePacket packet = m_exported->dev_create_packet (dev, p->GetSize () + 14); - p->CopyData (((unsigned char *)packet.buffer) + 14, p->GetSize ()); - struct ethhdr - { - unsigned char h_dest[6]; - unsigned char h_source[6]; - uint16_t h_proto; - } *hdr = (struct ethhdr *)packet.buffer; - if (device->GetInstanceTypeId () != m_lteUeTid) - { - Mac48Address realFrom; - if (Mac48Address::IsMatchingType (from)) - { - realFrom = Mac48Address::ConvertFrom (from); - } - else - { - // Only handling Mac64Address conversion at this time - // See DCE issue #122 - NS_ASSERT_MSG (Mac64Address::IsMatchingType (from), "Not a Mac64Address"); - Mac64Address addr = Mac64Address::ConvertFrom (from); - uint8_t bytes[8]; - addr.CopyTo (bytes); - uint8_t* newBytes = bytes + 2; // discard first two bytes - realFrom.CopyFrom (newBytes); - NS_LOG_DEBUG ("Converting 'from' address from " << from << " to " << realFrom); - } - realFrom.CopyTo (hdr->h_source); - } - - Mac48Address realTo; - if (Mac48Address::IsMatchingType (to)) + std::string file = "/proc/sys"; + if (path.empty () || path[0] != '.') { - realTo = Mac48Address::ConvertFrom (to); + file += '/'; } - else + for (std::string::iterator i = path.begin (); i != path.end (); ++i) { - // Only handling Mac64Address conversion at this time - // See DCE issue #122 - NS_ASSERT_MSG (Mac64Address::IsMatchingType (to), "Not a Mac64Address"); - Mac64Address addr = Mac64Address::ConvertFrom (to); - uint8_t bytes[8]; - addr.CopyTo (bytes); - uint8_t* newBytes = bytes + 2; // discard first two bytes - realTo.CopyFrom (newBytes); - NS_LOG_DEBUG ("Converting 'to' address from " << from << " to " << realTo); + file += *i == '.' ? '/' : *i; } - realTo.CopyTo (hdr->h_dest); - hdr->h_proto = ntohs (protocol); - m_exported->dev_rx (dev, packet); - m_loader->NotifyEndExecute (); + return file; } +} // namespace void -KernelSocketFdFactory::NotifyDeviceStateChange (Ptr device) +KernelSocketFdFactory::Set (std::string path, std::string value) { - ScheduleTask (MakeEvent (&KernelSocketFdFactory::NotifyDeviceStateChangeTask, - this, device)); + NS_LOG_FUNCTION (this << path << value); + if (!m_booted) + { + m_earlySysctls.push_back (std::make_pair (path, value)); + return; + } + ScheduleTask (MakeEvent (&KernelSocketFdFactory::SetTask, this, path, value)); } + void -KernelSocketFdFactory::NotifyDeviceStateChangeTask (Ptr device) +KernelSocketFdFactory::SetTask (std::string path, std::string value) { - NS_LOG_FUNCTION (device); - struct SimDevice *dev = DevToDev (device); - if (dev == 0) + std::string file = SysctlFile (path); + long fd = Call (__lkl__NR_openat, AT_FDCWD, (long)file.c_str (), O_WRONLY); + if (fd < 0) { + NS_LOG_WARN ("sysctl " << path << ": cannot open " << file << ": " << fd); return; } - - Mac48Address ad; - if (Mac48Address::IsMatchingType (device->GetAddress())) + long ret = Call (__lkl__NR_write, fd, (long)value.c_str (), value.size ()); + if (ret < 0) { - ad = Mac48Address::ConvertFrom (device->GetAddress ()); + NS_LOG_WARN ("sysctl " << path << " = " << value << ": " << ret); } - else - { - // Only handling Mac64Address conversion at this time - // See DCE issue #122 - NS_ASSERT_MSG (Mac64Address::IsMatchingType (device->GetAddress ()), "Not a Mac64Address"); - - Mac64Address addr = Mac64Address::ConvertFrom (device->GetAddress ()); - uint8_t bytes[8]; - addr.CopyTo (bytes); - uint8_t* newBytes = bytes + 2; // discard first two bytes - ad.CopyFrom (newBytes); - NS_LOG_DEBUG ("Converting device address from " << device->GetAddress () << " to " << ad); - } - - uint8_t buffer[6]; - ad.CopyTo (buffer); - m_loader->NotifyStartExecute (); // Restore the memory of the kernel before access it ! - m_exported->dev_set_address (dev, buffer); - m_exported->dev_set_mtu (dev, device->GetMtu ()); - m_loader->NotifyEndExecute (); + Call (__lkl__NR_close, fd); } -void -KernelSocketFdFactory::ScheduleTaskTrampoline (void *context) +std::string +KernelSocketFdFactory::Get (std::string path) { - Task *current = TaskManager::Current ()->CurrentTask (); - KernelSocketFdFactory *self = (KernelSocketFdFactory *) current->GetExtraContext (); - current->SetExtraContext (0); - EventImpl *event = (EventImpl *)context; - event->Invoke (); - event->Unref (); - self->m_kernelTasks.remove (current); - TaskManager::Current ()->Exit (); + NS_LOG_FUNCTION (this << path); + WaitBooted (); + std::string file = SysctlFile (path); + long fd = Call (__lkl__NR_openat, AT_FDCWD, (long)file.c_str (), O_RDONLY); + if (fd < 0) + { + NS_LOG_WARN ("sysctl " << path << ": cannot open " << file << ": " << fd); + return ""; + } + std::string value; + char buffer[4096]; + long n; + while ((n = Call (__lkl__NR_read, fd, (long)buffer, sizeof (buffer))) > 0) + { + value.append (buffer, n); + } + Call (__lkl__NR_close, fd); + return value; } void KernelSocketFdFactory::ScheduleTask (EventImpl *event) { - Task *task = m_manager->Start (&KernelSocketFdFactory::ScheduleTaskTrampoline, - event, 1 << 17); - task->SetExtraContext (this); - task->SetSwitchNotifier (&KernelSocketFdFactory::TaskSwitch, m_loader); - m_kernelTasks.push_back (task); + m_kernel->GetTaskManager ()->Start (&KernelSocketFdFactory::ScheduleTaskTrampoline, + event, LKL_TASK_STACK); } void -KernelSocketFdFactory::NotifyAddDevice (Ptr device) +KernelSocketFdFactory::ScheduleTaskTrampoline (void *context) { - ScheduleTask (MakeEvent (&KernelSocketFdFactory::NotifyAddDeviceTask, this, device)); + EventImpl *event = (EventImpl *)context; + event->Invoke (); + event->Unref (); + TaskManager::Current ()->Exit (); } + void -KernelSocketFdFactory::NotifyAddDeviceTask (Ptr device) +KernelSocketFdFactory::WaitBooted (void) { - NS_LOG_FUNCTION (device); - int flags = 0; - //NS_ASSERT (!device->IsPointToPoint ()); - //NS_ASSERT (device->NeedsArp ()); - //NS_ASSERT (device->IsMulticast ()); - //NS_ASSERT (device->IsBroadcast ()); - if (device->IsMulticast ()) - { - flags |= SIM_DEV_MULTICAST; - } - if (device->IsBroadcast ()) - { - flags |= SIM_DEV_BROADCAST; - } - if (!device->NeedsArp ()) - { - flags |= SIM_DEV_NOARP; - } - m_loader->NotifyStartExecute (); // Restore the memory of the kernel before access it ! -#if ((LIBOS_API_VERSION == 2)) - struct SimDevice *dev = m_exported->dev_create ("sim%d", PeekPointer (device), (enum SimDevFlags)flags); -#else - struct SimDevice *dev = m_exported->dev_create (PeekPointer (device), (enum SimDevFlags)flags); -#endif // LIBOS_API_VERSION - m_loader->NotifyEndExecute (); - Ptr listener = Create (device, this); - m_listeners.push_back (listener); - device->AddLinkChangeCallback (MakeCallback (&KernelDeviceStateListener::NotifyDeviceStateChange, listener)); - - m_devices.push_back (std::make_pair (device,dev)); - Ptr node = GetObject (); - if (device->GetInstanceTypeId () == m_lteUeTid) - { - node->RegisterProtocolHandler (MakeCallback (&KernelSocketFdFactory::RxFromDevice, this), - 0, device, false); - } - else + TaskManager *manager = TaskManager::Current (); + while (!m_booted) { - node->RegisterProtocolHandler (MakeCallback (&KernelSocketFdFactory::RxFromDevice, this), - 0, device, true); + m_bootWaiters.push_back (manager->RunningTask ()); + manager->Sleep (); } - NotifyDeviceStateChangeTask (device); } - -void -KernelSocketFdFactory::InitializeStack (void) -{ - std::string filePath = SearchExecFile ("DCE_PATH", m_library, 0); - if (filePath.length () <= 0) - { - std::string line = "Stack file '"; - line += m_library; - line += "' not found ! Please check your DCE_PATH environment variable."; - NS_ASSERT_MSG (filePath.length () > 0, line.c_str ()); - return ; - } - NS_LOG_INFO ("loading " + filePath); - void *handle = m_loader->Load (filePath, RTLD_LOCAL); - void *symbol = m_loader->Lookup (handle, "sim_init"); - SimInit init = (SimInit) symbol; - if (init == 0) - { - NS_FATAL_ERROR ("Oops. Can't find initialization function"); - } - m_exported = new struct SimExported (); - struct SimImported imported; - imported.vprintf = &KernelSocketFdFactory::Vprintf; - imported.malloc = &KernelSocketFdFactory::Malloc; - imported.free = &KernelSocketFdFactory::Free; - imported.memcpy = &KernelSocketFdFactory::Memcpy; - imported.memset = &KernelSocketFdFactory::Memset; - imported.atexit = &KernelSocketFdFactory::AtExit; - imported.access = &KernelSocketFdFactory::Access; - imported.getenv = &KernelSocketFdFactory::Getenv; - imported.mkdir = &KernelSocketFdFactory::Mkdir; - imported.open = &KernelSocketFdFactory::Open; - imported.__fxstat = &KernelSocketFdFactory::__Fxstat; - imported.fseek = &KernelSocketFdFactory::Fseek; - imported.setbuf = &KernelSocketFdFactory::Setbuf; - imported.ftell = &KernelSocketFdFactory::Ftell; - imported.fdopen = &KernelSocketFdFactory::FdOpen; - imported.fread = &KernelSocketFdFactory::Fread; - imported.fwrite = &KernelSocketFdFactory::Fwrite; - imported.fclose = &KernelSocketFdFactory::Fclose; - imported.random = &KernelSocketFdFactory::Random; - imported.event_schedule_ns = &KernelSocketFdFactory::EventScheduleNs; - imported.event_cancel = &KernelSocketFdFactory::EventCancel; - imported.current_ns = &CurrentNs; - imported.task_start = &KernelSocketFdFactory::TaskStart; - imported.task_wait = &KernelSocketFdFactory::TaskWait; - imported.task_current = &KernelSocketFdFactory::TaskCurrent; - imported.task_wakeup = &KernelSocketFdFactory::TaskWakeup; - imported.task_yield = &KernelSocketFdFactory::TaskYield; - imported.dev_xmit = &KernelSocketFdFactory::DevXmit; - imported.signal_raised = &KernelSocketFdFactory::SignalRaised; - imported.poll_event = &KernelSocketFdFactory::PollEvent; - // create internal process - Ptr manager = this->GetObject (); - m_pid = manager->StartInternalTask (); - - init (m_exported, &imported, (struct SimKernel *)this); - - // update the kernel device list with simulation device list - Ptr node = GetObject (); - node->RegisterDeviceAdditionListener (MakeCallback (&KernelSocketFdFactory::NotifyAddDevice, - this)); - NS_LOG_FUNCTION (this << "m_exported " << m_exported); -} - -UnixFd * -KernelSocketFdFactory::CreateSocket (int domain, int type, int protocol) +long +KernelSocketFdFactory::Call (long no, long a0, long a1, long a2, long a3, long a4, long a5) { - GET_CURRENT (this << domain << type << protocol); - struct SimSocket *socket; - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_socket (domain, type, protocol, &socket); - m_loader->NotifyEndExecute (); - if (retval < 0) - { - return 0; - } - UnixFd *fd = new KernelSocketFd (this, socket); - return fd; + long params[6] = { a0, a1, a2, a3, a4, a5 }; + return m_kernel->Syscall (no, params); } -int -KernelSocketFdFactory::Close (struct SimSocket *socket) +namespace { +struct AddDeviceContext { - GET_CURRENT (socket); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_close (socket); - m_loader->NotifyEndExecute (); - if (retval < 0) - { - current->err = -retval; - return -1; - } - return retval; -} -ssize_t -KernelSocketFdFactory::Recvmsg (struct SimSocket *socket, struct msghdr *msg, int flags) + KernelSocketFdFactory *factory; + Ptr device; +}; +} // namespace + +void +KernelSocketFdFactory::NotifyAddDevice (Ptr device) { - GET_CURRENT (socket << msg << flags); - m_loader->NotifyStartExecute (); - ssize_t retval = m_exported->sock_recvmsg (socket, msg, flags); - m_loader->NotifyEndExecute (); - if (retval < 0) + TaskManager *manager = m_kernel->GetTaskManager (); + if (TaskManager::Current () == manager && manager->RunningTask ()) { - current->err = -retval; - return -1; + AddDevice (device); + return; } - return retval; + // Adding a kernel device needs a task. + AddDeviceContext *context = new AddDeviceContext (); + context->factory = this; + context->device = device; + manager->Start (&KernelSocketFdFactory::AddDeviceTrampoline, context, LKL_TASK_STACK); } -ssize_t -KernelSocketFdFactory::Sendmsg (struct SimSocket *socket, const struct msghdr *msg, int flags) + +void +KernelSocketFdFactory::AddDeviceTrampoline (void *c) { - GET_CURRENT (socket << msg << flags); - m_loader->NotifyStartExecute (); - ssize_t retval = m_exported->sock_sendmsg (socket, msg, flags); - m_loader->NotifyEndExecute (); - if (retval < 0) - { - current->err = -retval; - return -1; - } - return retval; + AddDeviceContext *context = (AddDeviceContext *)c; + context->factory->AddDevice (context->device); + delete context; + TaskManager::Current ()->Exit (); } -int -KernelSocketFdFactory::Getsockname (struct SimSocket *socket, struct sockaddr *name, socklen_t *namelen) -{ - GET_CURRENT (socket << name << namelen); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_getsockname (socket, name, (int*)namelen); - m_loader->NotifyEndExecute (); - if (retval < 0) + +void +KernelSocketFdFactory::AddDevice (Ptr device) +{ + NS_LOG_FUNCTION (this << device); + if (DynamicCast (device)) + { + return; // the kernel has its own loopback device + } + LklNetDevice *dev = new LklNetDevice (); + memset (&dev->nd, 0, sizeof (dev->nd)); + dev->nd.ops = &g_netOps; + dev->factory = this; + dev->device = device; + dev->poller = 0; + dev->hup = false; + dev->stalled = false; + m_devices.push_back (dev); + + uint8_t mac[6]; + Address address = device->GetAddress (); + Mac48Address mac48 = Mac48Address::IsMatchingType (address) + ? Mac48Address::ConvertFrom (address) : Mac48Address::Allocate (); + mac48.CopyTo (mac); + struct lkl_netdev_args args; + memset (&args, 0, sizeof (args)); + args.mac = mac; + + typedef int (*NetdevAdd)(struct lkl_netdev *, struct lkl_netdev_args *); + NetdevAdd netdevAdd = (NetdevAdd) m_kernel->Lookup ("lkl_netdev_add"); + int id = netdevAdd (&dev->nd, &args); + NS_ASSERT_MSG (id >= 0, "lkl_netdev_add failed: " << id); + + // Find the new device, named eth, by its address (lkl_netdev_get_ifindex + // looks for eth, which the devices renamed below break), and name it + // sim like the libos stack, with the MTU of the ns-3 device. + long s = Call (__lkl__NR_socket, AF_INET, SOCK_DGRAM, 0); + NS_ASSERT (s >= 0); + struct ifreq ifr; + bool found = false; + for (int ifindex = 1; ifindex < 4096 && !found; ifindex++) + { + memset (&ifr, 0, sizeof (ifr)); + ifr.ifr_ifindex = ifindex; + if (Call (__lkl__NR_ioctl, s, SIOCGIFNAME, (long)&ifr) != 0 + || strncmp (ifr.ifr_name, "eth", 3) != 0 + || Call (__lkl__NR_ioctl, s, SIOCGIFHWADDR, (long)&ifr) != 0) + { + continue; + } + found = memcmp (ifr.ifr_hwaddr.sa_data, mac, 6) == 0; + } + NS_ASSERT_MSG (found, "no kernel device for " << mac48); + long ret; + snprintf (ifr.ifr_newname, IFNAMSIZ, "sim%u", m_nextDevice++); + ret = Call (__lkl__NR_ioctl, s, SIOCSIFNAME, (long)&ifr); + NS_ASSERT_MSG (ret == 0, "SIOCSIFNAME failed: " << ret); + memcpy (ifr.ifr_name, ifr.ifr_newname, IFNAMSIZ); + ifr.ifr_mtu = device->GetMtu (); + ret = Call (__lkl__NR_ioctl, s, SIOCSIFMTU, (long)&ifr); + NS_ASSERT_MSG (ret == 0, "SIOCSIFMTU failed: " << ret); + if (!device->NeedsArp ()) { - current->err = -retval; - return -1; + // e.g. point-to-point links, which cannot carry ARP: the kernel then + // sends to neighbors without resolving them. + ret = Call (__lkl__NR_ioctl, s, SIOCGIFFLAGS, (long)&ifr); + NS_ASSERT_MSG (ret == 0, "SIOCGIFFLAGS failed: " << ret); + ifr.ifr_flags |= IFF_NOARP; + ret = Call (__lkl__NR_ioctl, s, SIOCSIFFLAGS, (long)&ifr); + NS_ASSERT_MSG (ret == 0, "SIOCSIFFLAGS failed: " << ret); } - return retval; + Call (__lkl__NR_close, s); + + GetObject ()->RegisterProtocolHandler (MakeCallback (&KernelSocketFdFactory::RxFromDevice, this), + 0, device, true); } -int -KernelSocketFdFactory::Getpeername (struct SimSocket *socket, struct sockaddr *name, socklen_t *namelen) + +void +KernelSocketFdFactory::RxFromDevice (Ptr device, Ptr p, uint16_t protocol, + const Address &from, const Address &to, NetDevice::PacketType type) { - GET_CURRENT (socket << name << namelen); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_getpeername (socket, name, (int*)namelen); - m_loader->NotifyEndExecute (); - if (retval < 0) + LklNetDevice *dev = 0; + for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) { - current->err = -retval; - return -1; + if ((*i)->device == device) + { + dev = *i; + break; + } } - return retval; -} -int -KernelSocketFdFactory::Bind (struct SimSocket *socket, const struct sockaddr *my_addr, socklen_t addrlen) -{ - GET_CURRENT (socket << my_addr << addrlen); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_bind (socket, my_addr, addrlen); - m_loader->NotifyEndExecute (); - if (retval < 0) + if (!dev || dev->rx.size () >= RX_BACKLOG) { - current->err = -retval; - return -1; + return; } - return retval; -} -int -KernelSocketFdFactory::Connect (struct SimSocket *socket, const struct sockaddr *my_addr, - socklen_t addrlen, int flags) -{ - GET_CURRENT (socket << my_addr << addrlen << flags); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_connect (socket, my_addr, addrlen, flags); - m_loader->NotifyEndExecute (); - if (retval < 0) + // Rebuild the Ethernet frame the kernel expects. + std::vector frame (ETH_HEADER + p->GetSize ()); + Mac48Address dest = Mac48Address::IsMatchingType (to) + ? Mac48Address::ConvertFrom (to) : Mac48Address::ConvertFrom (device->GetAddress ()); + Mac48Address source = Mac48Address::IsMatchingType (from) + ? Mac48Address::ConvertFrom (from) : Mac48Address ("00:00:00:00:00:00"); + dest.CopyTo (&frame[0]); + source.CopyTo (&frame[6]); + frame[12] = protocol >> 8; + frame[13] = protocol & 0xff; + p->CopyData (&frame[ETH_HEADER], p->GetSize ()); + NS_LOG_DEBUG ("rx " << device << " protocol " << protocol << " " << frame.size () << " bytes"); + dev->rx.push_back (frame); + dev->stalled = false; + if (dev->poller) { - current->err = -retval; - return -1; + m_kernel->GetTaskManager ()->Wakeup (dev->poller); } - return retval; } -int -KernelSocketFdFactory::Listen (struct SimSocket *socket, int backlog) + +UnixFd * +KernelSocketFdFactory::CreateSocket (int domain, int type, int protocol) { - GET_CURRENT (socket << backlog); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_listen (socket, backlog); - m_loader->NotifyEndExecute (); - if (retval < 0) + NS_LOG_FUNCTION (this << domain << type << protocol); + WaitBooted (); + if (m_mptcp && (domain == AF_INET || domain == AF_INET6) + && (type & 0xf) == SOCK_STREAM && (protocol == 0 || protocol == IPPROTO_TCP)) { - current->err = -retval; - return -1; + protocol = 262; // IPPROTO_MPTCP } - return retval; -} -int -KernelSocketFdFactory::Shutdown (struct SimSocket *socket, int how) -{ - GET_CURRENT (socket << how); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_shutdown (socket, how); - m_loader->NotifyEndExecute (); - if (retval < 0) + long fd = Call (__lkl__NR_socket, domain, type, protocol); + if (fd < 0) { - current->err = -retval; - return -1; + return 0; } - return retval; + return new KernelSocketFd (this, fd); } -int -KernelSocketFdFactory::Accept (struct SimSocket *socket, struct sockaddr *my_addr, socklen_t *addrlen, int flags) -{ - GET_CURRENT (socket << my_addr << addrlen << flags); - struct SimSocket *newSocket; - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_accept (socket, &newSocket, flags); - m_loader->NotifyEndExecute (); - if (retval < 0) - { - current->err = -retval; - return -1; - } - if (my_addr != 0) - { - m_loader->NotifyStartExecute (); - retval = m_exported->sock_getpeername (newSocket, my_addr, (int*)addrlen); - if (retval < 0) - { - current->err = -retval; - m_exported->sock_close (newSocket); - m_loader->NotifyEndExecute (); - return -1; - } - m_loader->NotifyEndExecute (); - } - int fd = UtilsAllocateFd (); - if (fd == -1) - { - m_loader->NotifyStartExecute (); - m_exported->sock_close (newSocket); - current->err = EMFILE; - m_loader->NotifyEndExecute (); - return -1; - } - UnixFd *unixFd = new KernelSocketFd (this, newSocket); - unixFd->IncFdCount (); - current->process->openFiles[fd] = new FileUsage (fd, unixFd); - - return fd; -} -int -KernelSocketFdFactory::Ioctl (struct SimSocket *socket, unsigned long request, char *argp) +void +KernelSocketFdFactory::Register (KernelSocketFd *socket) { - GET_CURRENT (socket << request << argp); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_ioctl (socket, request, argp); - m_loader->NotifyEndExecute (); - if (retval < 0) - { - current->err = -retval; - return -1; - } - return retval; + m_sockets[socket->GetKernelFd ()] = socket; } -int -KernelSocketFdFactory::Setsockopt (struct SimSocket *socket, int level, int optname, - const void *optval, socklen_t optlen) + +void +KernelSocketFdFactory::Unregister (KernelSocketFd *socket) { - GET_CURRENT (socket << level << optname << optval << optlen); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_setsockopt (socket, level, optname, optval, optlen); - m_loader->NotifyEndExecute (); - if (retval < 0) - { - current->err = -retval; - return -1; - } - return retval; + m_sockets.erase (socket->GetKernelFd ()); + m_watched.erase (socket->GetKernelFd ()); } -int -KernelSocketFdFactory::Getsockopt (struct SimSocket *socket, int level, int optname, - void *optval, socklen_t *optlen) + +void +KernelSocketFdFactory::Watch (KernelSocketFd *socket) { - GET_CURRENT (socket << level << optname << optval << optlen); - m_loader->NotifyStartExecute (); - int retval = m_exported->sock_getsockopt (socket, level, optname, optval, (int*)optlen); - m_loader->NotifyEndExecute (); - if (retval < 0) + // Like a kernel wait queue: once watched, every change of the socket's + // state wakes its poll waiters, which check what they wait for. + int fd = socket->GetKernelFd (); + if (m_watched.count (fd)) { - current->err = -retval; - return -1; + return; } - return retval; + struct epoll_event event; + memset (&event, 0, sizeof (event)); + event.events = EPOLLIN | EPOLLOUT | EPOLLPRI | EPOLLRDHUP | EPOLLET; + event.data.fd = fd; + long ret = Call (__lkl__NR_epoll_ctl, m_epollFd, EPOLL_CTL_ADD, fd, (long)&event); + NS_ASSERT_MSG (ret == 0, "epoll_ctl failed: " << ret); + m_watched.insert (fd); } + void -KernelSocketFdFactory::PollEvent (int flag, void *context) +KernelSocketFdFactory::WatcherTrampoline (void *context) { - PollTable* ptable = (PollTable*)context; - ptable->WakeUpCallback (); + ((KernelSocketFdFactory *)context)->Watcher (); } -/** - * Struct used to pass pool table context between DCE and Kernel and back from Kernel to DCE - * - * When calling sock_poll we provide in ret field the wanted eventmask, and in the opaque field - * the DCE poll table - * - * if a corresponding event occurs later, the PollEvent will be called by kernel with the DCE - * poll table in context variable, then we will able to wake up the thread blocked in poll call. - * - * Back from sock_poll method the kernel change ret field with the response from poll return of the - * corresponding kernel socket, and in opaque field there is a reference to the kernel poll table - * we will use this reference to remove us from the file wait queue when ending the DCE poll call or - * when ending the DCE process which is currently polling. - * - */ -struct poll_table_ref -{ - int ret; - void *opaque; -}; -int -KernelSocketFdFactory::Poll (struct SimSocket *socket, PollTable* ptable) +void +KernelSocketFdFactory::Watcher (void) { - struct poll_table_ref kernelInOut = - { - 0 - }; - if (ptable) - { - // Fill Opaque and ptable. - kernelInOut.opaque = ptable; - kernelInOut.ret = ptable->GetEventMask (); - } - - GET_CURRENT (socket); - m_loader->NotifyStartExecute (); - m_exported->sock_poll (socket, &kernelInOut); - m_loader->NotifyEndExecute (); - - if (ptable) + struct epoll_event events[32]; + while (true) { - ptable->PollWait (kernelInOut.opaque, MakeCallback (&KernelSocketFdFactory::PollFreeWait, this)); + long n = Call (__lkl__NR_epoll_pwait, m_epollFd, (long)events, 32, -1, 0, 8); + for (long i = 0; i < n; i++) + { + std::map::iterator s = m_sockets.find (events[i].data.fd); + if (s != m_sockets.end ()) + { + s->second->NotifyEvents (events[i].events & 0xffff); + } + } } - - return kernelInOut.ret; -} -void -KernelSocketFdFactory::PollFreeWait (void *ref) -{ - m_loader->NotifyStartExecute (); - m_exported->sock_pollfreewait (ref); - m_loader->NotifyEndExecute (); } + } // namespace ns3 diff --git a/model/kernel-socket-fd-factory.h b/model/kernel-socket-fd-factory.h index 011ffef6..8c1cf19d 100644 --- a/model/kernel-socket-fd-factory.h +++ b/model/kernel-socket-fd-factory.h @@ -1,39 +1,31 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ #ifndef KERNEL_SOCKET_FD_FACTORY_H #define KERNEL_SOCKET_FD_FACTORY_H #include "socket-fd-factory.h" -#include "task-manager.h" #include "ns3/net-device.h" -#include "ns3/random-variable-stream.h" -#include -#include +#include +#include +#include #include -#include -#include -#include -#include "linux/types.h" - -extern "C" { -struct SimExported; -struct SimDevice; -struct SimSocket; -struct SimTask; -struct SimKernel; -struct SimSysFile; -} - -class KingsleyAlloc; +#include namespace ns3 { -class Loader; -class NetDevice; +class EventImpl; +class LklKernel; +class KernelSocketFd; class Task; -class TaskManager; -class Packet; -class KernelDeviceStateListener; -class PollTable; - +struct LklNetDevice; + +/** + * Network stack of a node backed by a Linux kernel (LKL): select it with + * DceManagerHelper::SetNetworkStack ("ns3::KernelSocketFdFactory"). + * + * The kernel boots when the simulation starts. Every ns-3 NetDevice of the + * node becomes a kernel network device named sim, in device order; + * sockets of DCE applications are kernel sockets. + */ class KernelSocketFdFactory : public SocketFdFactory { public: @@ -43,115 +35,58 @@ class KernelSocketFdFactory : public SocketFdFactory virtual UnixFd * CreateSocket (int domain, int type, int protocol); + Ptr GetKernel (void) const; + + /** + * Write a sysctl, like libos: path ".net.ipv4.tcp_sack" is + * /proc/sys/net/ipv4/tcp_sack. Sysctls set before the kernel booted are + * written right after boot. + */ + void Set (std::string path, std::string value); + /** + * Read a sysctl, or any kernel file given by its absolute path (e.g. + * /proc/net/netstat); must be called from a task of this node. + */ + std::string Get (std::string path); + /** Run the event in a new task of this node. */ void ScheduleTask (EventImpl *event); - std::string m_library; -protected: - void InitializeStack (void); - struct SimExported *m_exported; - Ptr m_manager; - Loader *m_loader; - FILE *m_logFile; + // Used by KernelSocketFd. + void Register (KernelSocketFd *socket); + void Unregister (KernelSocketFd *socket); + /** From now on, wake the socket's poll waiters when its state changes. */ + void Watch (KernelSocketFd *socket); private: - KernelSocketFdFactory (KernelSocketFdFactory const &); - KernelSocketFdFactory& operator= (KernelSocketFdFactory const &); - - friend class KernelSocketFd; - friend class KernelDeviceStateListener; - struct EventIdHolder : public SimpleRefCount - { - EventId id; - }; - - // called from KernelSocketFd - int Close (struct SimSocket *socket); - ssize_t Recvmsg (struct SimSocket *socket, struct msghdr *msg, int flags); - ssize_t Sendmsg (struct SimSocket *socket, const struct msghdr *msg, int flags); - int Getsockname (struct SimSocket *socket, struct sockaddr *name, socklen_t *namelen); - int Getpeername (struct SimSocket *socket, struct sockaddr *name, socklen_t *namelen); - int Bind (struct SimSocket *socket, const struct sockaddr *my_addr, socklen_t addrlen); - int Connect (struct SimSocket *socket, const struct sockaddr *my_addr, socklen_t addrlen, int flags); - int Listen (struct SimSocket *socket, int backlog); - int Shutdown (struct SimSocket *socket, int how); - int Accept (struct SimSocket *socket, struct sockaddr *my_addr, socklen_t *addrlen, int flags); - int Ioctl (struct SimSocket *socket, unsigned long request, char *argp); - int Setsockopt (struct SimSocket *socket, int level, int optname, - const void *optval, socklen_t optlen); - int Getsockopt (struct SimSocket *socket, int level, int optname, - void *optval, socklen_t *optlen); - bool CanRecv (struct SimSocket *socket); - bool CanSend (struct SimSocket *socket); - void* PollWait (struct SimSocket *socket, void *ctxt); - void FreePoll (struct SimSocket *socket, void *ctxt); - int Poll (struct SimSocket *socket, PollTable* ptable); - void PollFreeWait (void *ref); - - // called from kernel. - static int Vprintf (struct SimKernel *kernel, const char *str, va_list args); - static void * Malloc (struct SimKernel *kernel, unsigned long size); - static void Free (struct SimKernel *kernel, void *buffer); - static void * Memcpy (struct SimKernel *kernel, void *dst, const void *src, unsigned long size); - static void * Memset (struct SimKernel *kernel, void *dst, char value, unsigned long size); - static int AtExit (struct SimKernel *kernel, void (*function)(void)); - static int Access (struct SimKernel *kernel, const char *pathname, int mode); - static char* Getenv (struct SimKernel *kernel, const char *name); - static int Mkdir (struct SimKernel *kernel, const char *pathname, mode_t mode); - static int Open (struct SimKernel *kernel, const char *pathname, int flags); - static size_t Fread (struct SimKernel *kernel, void *ptr, size_t size, size_t nmemb, FILE *stream); - static size_t Fwrite (struct SimKernel *kernel, const void *ptr, size_t size, size_t nmemb, FILE *stream); - static FILE* FdOpen (struct SimKernel *kernel, int fd, const char *mode); - static int __Fxstat (struct SimKernel *kernel, int ver, int fd, void *buf); - static int Fseek (struct SimKernel *kernel, FILE *stream, long offset, int whence); - static void Setbuf (struct SimKernel *kernel, FILE *stream, char *buf); - static long Ftell (struct SimKernel *kernel, FILE *stream); - static int Fclose (struct SimKernel *kernel, FILE *fp); - static unsigned long Random (struct SimKernel *kernel); - static void *EventScheduleNs (struct SimKernel *kernel, __u64 ns, void (*fn)(void *context), void *context, - void (*pre_fn)(void)); - static void EventCancel (struct SimKernel *kernel, void *ev); - static struct SimTask *TaskStart (struct SimKernel *kernel, void (*callback)(void *), void *context); - static struct SimTask * TaskCurrent (struct SimKernel *kernel); - static void TaskWait (struct SimKernel *kernel); - static int TaskWakeup (struct SimKernel *kernel, struct SimTask *task); - static void TaskYield (struct SimKernel *kernel); - static void DevXmit (struct SimKernel *kernel, struct SimDevice *dev, unsigned char *data, int len); - static void SignalRaised (struct SimKernel *kernel, struct SimTask *task, int signalNumber); - static void PollEvent (int flag, void *context); - - - // inherited from Object. virtual void DoDispose (void); - // called from Node - void NotifyAddDevice (Ptr device); - // called during initialization with a task context - // to enter the kernel functions. - void StartInitializationTask (void); - void RxFromDevice (Ptr device, Ptr p, - uint16_t protocol, const Address & from, - const Address &to, NetDevice::PacketType type); - struct SimDevice * DevToDev (Ptr dev); - void NotifyDeviceStateChange (Ptr device); - void NotifyDeviceStateChangeTask (Ptr device); - void NotifyAddDeviceTask (Ptr device); - - void DoSet (std::string path, std::string value); - static void TaskSwitch (enum Task::SwitchType type, void *context); + virtual void NotifyNewAggregate (void); + static void BootTrampoline (void *context); static void ScheduleTaskTrampoline (void *context); - void EventTrampoline (void (*fn)(void *context), - void *context, void (*pre_fn)(void), - Ptr event); - static void SendMain (bool *r, NetDevice *d, Ptr packet, const Address& dest, uint16_t protocolNumber); + void SetTask (std::string path, std::string value); + void Boot (void); + void WaitBooted (void); + void NotifyAddDevice (Ptr device); + void AddDevice (Ptr device); + static void AddDeviceTrampoline (void *context); + void RxFromDevice (Ptr device, Ptr p, uint16_t protocol, + const Address &from, const Address &to, NetDevice::PacketType type); + static void WatcherTrampoline (void *context); + void Watcher (void); + long Call (long no, long a0 = 0, long a1 = 0, long a2 = 0, + long a3 = 0, long a4 = 0, long a5 = 0); - std::vector,struct SimDevice *> > m_devices; - std::list m_kernelTasks; - Ptr m_variable; - KingsleyAlloc *m_alloc; - std::vector > m_listeners; - double m_rate; - Ptr m_ranvar; - uint16_t m_pid; - TypeId m_lteUeTid; + std::string m_library; + std::string m_cmdline; + bool m_mptcp; + Ptr m_kernel; + bool m_booted; + std::list m_bootWaiters; + std::list > m_earlySysctls; + std::vector m_devices; + uint32_t m_nextDevice; + int m_epollFd; + std::map m_sockets; + std::set m_watched; }; } // namespace ns3 diff --git a/model/kernel-socket-fd.cc b/model/kernel-socket-fd.cc index fc176ba7..8f9dfee0 100644 --- a/model/kernel-socket-fd.cc +++ b/model/kernel-socket-fd.cc @@ -1,240 +1,287 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ #include "kernel-socket-fd.h" #include "kernel-socket-fd-factory.h" -#include "utils.h" +#include "lkl-kernel.h" #include "process.h" +#include "utils.h" +#include "file-usage.h" +#include "wait-queue.h" #include "ns3/log.h" +#include #include #include -#include // for MMAP_FAILED #include +#include +#include +#include NS_LOG_COMPONENT_DEFINE ("DceKernelSocketFd"); namespace ns3 { -KernelSocketFd::KernelSocketFd (Ptr factory, struct SimSocket *socket) +KernelSocketFd::KernelSocketFd (Ptr factory, int fd) : m_factory (factory), - m_socket (socket), - m_statusFlags (0) + m_fd (fd) { + NS_LOG_FUNCTION (this << fd); + m_factory->Register (this); } KernelSocketFd::~KernelSocketFd () { } +int +KernelSocketFd::GetKernelFd (void) const +{ + return m_fd; +} + +void +KernelSocketFd::NotifyEvents (short events) +{ + WakeWaiters (&events); +} + +long +KernelSocketFd::Call (long no, long a0, long a1, long a2, long a3, long a4, long a5) const +{ + long params[6] = { a0, a1, a2, a3, a4, a5 }; + long ret = m_factory->GetKernel ()->Syscall (no, params); + if (ret < 0) + { + Current ()->err = -ret; + return -1; + } + return ret; +} + int KernelSocketFd::Close (void) { - return m_factory->Close (m_socket); + NS_LOG_FUNCTION (this); + m_factory->Unregister (this); + return Call (__lkl__NR_close, m_fd); } + ssize_t KernelSocketFd::Write (const void *buf, size_t count) { - NS_LOG_FUNCTION (this << buf << count); - struct msghdr msg; - struct iovec iov; - msg.msg_control = 0; - msg.msg_controllen = 0; - msg.msg_iovlen = 1; - msg.msg_iov = &iov; - iov.iov_len = count; - iov.iov_base = (void*)buf; - msg.msg_name = 0; - msg.msg_namelen = 0; - ssize_t retval = Sendmsg (&msg, 0); - return retval; + return Call (__lkl__NR_write, m_fd, (long)buf, count); } + ssize_t KernelSocketFd::Read (void *buf, size_t count) { - NS_LOG_FUNCTION (this << buf << count); - struct msghdr msg; - struct iovec iov; - msg.msg_control = 0; - msg.msg_controllen = 0; - msg.msg_iovlen = 1; - msg.msg_iov = &iov; - iov.iov_len = count; - iov.iov_base = buf; - msg.msg_name = 0; - msg.msg_namelen = 0; - ssize_t retval = Recvmsg (&msg, 0); - return retval; + return Call (__lkl__NR_read, m_fd, (long)buf, count); } + ssize_t KernelSocketFd::Recvmsg (struct msghdr *msg, int flags) { - bool nonBlocking = (m_statusFlags & O_NONBLOCK) == O_NONBLOCK; - flags |= nonBlocking ? MSG_DONTWAIT : 0; - return m_factory->Recvmsg (m_socket, msg, flags); + return Call (__lkl__NR_recvmsg, m_fd, (long)msg, flags); } + ssize_t KernelSocketFd::Sendmsg (const struct msghdr *msg, int flags) { - bool nonBlocking = (m_statusFlags & O_NONBLOCK) == O_NONBLOCK; - flags |= nonBlocking ? MSG_DONTWAIT : 0; - return m_factory->Sendmsg (m_socket, msg, flags); + return Call (__lkl__NR_sendmsg, m_fd, (long)msg, flags); } + bool KernelSocketFd::Isatty (void) const { return false; } + int -KernelSocketFd::Setsockopt (int level, int optname, - const void *optval, socklen_t optlen) +KernelSocketFd::Setsockopt (int level, int optname, const void *optval, socklen_t optlen) { - return m_factory->Setsockopt (m_socket, level, optname, optval, optlen); + return Call (__lkl__NR_setsockopt, m_fd, level, optname, (long)optval, optlen); } + int -KernelSocketFd::Getsockopt (int level, int optname, - void *optval, socklen_t *optlen) +KernelSocketFd::Getsockopt (int level, int optname, void *optval, socklen_t *optlen) { - return m_factory->Getsockopt (m_socket, level, optname, optval, optlen); + return Call (__lkl__NR_getsockopt, m_fd, level, optname, (long)optval, (long)optlen); } + int KernelSocketFd::Getsockname (struct sockaddr *name, socklen_t *namelen) { - return m_factory->Getsockname (m_socket, name, namelen); + return Call (__lkl__NR_getsockname, m_fd, (long)name, (long)namelen); } + int KernelSocketFd::Getpeername (struct sockaddr *name, socklen_t *namelen) { - return m_factory->Getpeername (m_socket, name, namelen); + return Call (__lkl__NR_getpeername, m_fd, (long)name, (long)namelen); } + int KernelSocketFd::Ioctl (unsigned long request, char *argp) { - return m_factory->Ioctl (m_socket, request, argp); + return Call (__lkl__NR_ioctl, m_fd, request, (long)argp); } + int KernelSocketFd::Bind (const struct sockaddr *my_addr, socklen_t addrlen) { - return m_factory->Bind (m_socket, my_addr, addrlen); + return Call (__lkl__NR_bind, m_fd, (long)my_addr, addrlen); } + int KernelSocketFd::Connect (const struct sockaddr *my_addr, socklen_t addrlen) { - return m_factory->Connect (m_socket, my_addr, addrlen, m_statusFlags); + return Call (__lkl__NR_connect, m_fd, (long)my_addr, addrlen); } + int KernelSocketFd::Listen (int backlog) { - return m_factory->Listen (m_socket, backlog); + return Call (__lkl__NR_listen, m_fd, backlog); } + int KernelSocketFd::Shutdown (int how) { - return m_factory->Shutdown (m_socket, how); + return Call (__lkl__NR_shutdown, m_fd, how); } + int KernelSocketFd::Accept (struct sockaddr *my_addr, socklen_t *addrlen) { - return m_factory->Accept (m_socket, my_addr, addrlen, m_statusFlags); + NS_LOG_FUNCTION (this); + long kernelFd = Call (__lkl__NR_accept4, m_fd, (long)my_addr, (long)addrlen, 0); + if (kernelFd < 0) + { + return -1; + } + Thread *current = Current (); + int fd = UtilsAllocateFd (); + if (fd == -1) + { + Call (__lkl__NR_close, kernelFd); + current->err = EMFILE; + return -1; + } + UnixFd *unixFd = new KernelSocketFd (m_factory, kernelFd); + unixFd->IncFdCount (); + current->process->openFiles[fd] = new FileUsage (fd, unixFd); + return fd; } + void * KernelSocketFd::Mmap (void *start, size_t length, int prot, int flags, off64_t offset) { - GET_CURRENT (start << length << prot << flags << offset); - current->err = ENODEV; + Current ()->err = ENODEV; return MAP_FAILED; } + off64_t KernelSocketFd::Lseek (off64_t offset, int whence) { - GET_CURRENT (offset << whence); - current->err = ESPIPE; + Current ()->err = ESPIPE; return -1; } + int KernelSocketFd::Fxstat (int ver, struct ::stat *buf) { - GET_CURRENT (ver << buf); buf->st_mode = S_IFSOCK; buf->st_dev = -1; buf->st_blksize = 0; return 0; } + int KernelSocketFd::Fxstat64 (int ver, struct ::stat64 *buf) { - GET_CURRENT (ver << buf); buf->st_mode = S_IFSOCK; buf->st_dev = -1; buf->st_blksize = 0; return 0; } + int KernelSocketFd::Fcntl (int cmd, unsigned long arg) { switch (cmd) { - case F_GETFL: //XXX this command should also consider the flags O_APPEND and O_ASYNC - return m_statusFlags; - break; + case F_GETFL: case F_SETFL: - m_statusFlags = arg; - return 0; - break; + // The kernel keeps O_NONBLOCK and applies it to every operation. + return Call (__lkl__NR_fcntl, m_fd, cmd, arg); case F_GETFD: return m_fdFlags; - break; case F_SETFD: m_fdFlags = arg; return 0; - break; default: - return UnixFd::Fcntl(cmd, arg ); + return UnixFd::Fcntl (cmd, arg); } } + int -KernelSocketFd::Settime (int flags, - const struct itimerspec *new_value, - struct itimerspec *old_value) +KernelSocketFd::Settime (int flags, const struct itimerspec *new_value, struct itimerspec *old_value) { - NS_LOG_FUNCTION (this << Current () << flags << new_value << old_value); - NS_ASSERT (Current () != 0); - Thread *current = Current (); - current->err = EINVAL; + Current ()->err = EINVAL; return -1; } + int KernelSocketFd::Gettime (struct itimerspec *cur_value) const { - NS_LOG_FUNCTION (this << Current () << cur_value); - NS_ASSERT (Current () != 0); - Thread *current = Current (); - current->err = EINVAL; + Current ()->err = EINVAL; + return -1; +} + +int +KernelSocketFd::Ftruncate (off_t length) +{ + Current ()->err = EINVAL; return -1; } + bool KernelSocketFd::HangupReceived (void) const { - // XXX: TO BE IMPLEMENTED OR NOT :) return false; } + int KernelSocketFd::Poll (PollTable* ptable) { - return m_factory->Poll (m_socket, ptable); -} -int -KernelSocketFd::Ftruncate (off_t length) -{ - NS_LOG_FUNCTION (this << Current ()); - NS_ASSERT (Current () != 0); - Thread *current = Current (); - current->err = EINVAL; - return -1; + short wanted = ptable ? ptable->GetEventMask () : (POLLIN | POLLOUT | POLLPRI | POLLRDHUP); + struct pollfd pfd; + pfd.fd = m_fd; + pfd.events = wanted; + pfd.revents = 0; + struct + { + long long tv_sec; + long long tv_nsec; + } zero = { 0, 0 }; + long params[6] = { (long)&pfd, 1, (long)&zero, 0, 8, 0 }; + long ret = m_factory->GetKernel ()->Syscall (__lkl__NR_ppoll, params); + if (ret < 0) + { + return POLLERR; + } + if (ptable) + { + // Get woken when the kernel reports one of these events. + ptable->PollWait (this); + m_factory->Watch (this); + } + return pfd.revents; } + int KernelSocketFd::Fsync (void) { - Thread *current = Current (); - NS_LOG_FUNCTION (this << current); - NS_ASSERT (current != 0); - current->err = EBADF; + Current ()->err = EINVAL; return -1; } + } // namespace ns3 diff --git a/model/kernel-socket-fd.h b/model/kernel-socket-fd.h index 320f3fce..cd685d6e 100644 --- a/model/kernel-socket-fd.h +++ b/model/kernel-socket-fd.h @@ -1,25 +1,28 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ #ifndef KERNEL_SOCKET_FD_H #define KERNEL_SOCKET_FD_H #include "unix-fd.h" #include "ns3/ptr.h" -extern "C" { -struct SimSocket; -} - namespace ns3 { class KernelSocketFdFactory; -class Waiter; - +/** + * A socket of a node's LKL kernel: every operation is the corresponding + * kernel system call on the kernel's file descriptor. + */ class KernelSocketFd : public UnixFd { public: - KernelSocketFd (Ptr factory, struct SimSocket *socket); + KernelSocketFd (Ptr factory, int fd); virtual ~KernelSocketFd (); + int GetKernelFd (void) const; + /** Wake the tasks waiting in poll or select for these events. */ + void NotifyEvents (short events); + virtual int Close (void); virtual ssize_t Write (const void *buf, size_t count); virtual ssize_t Read (void *buf, size_t count); @@ -47,20 +50,18 @@ class KernelSocketFd : public UnixFd const struct itimerspec *new_value, struct itimerspec *old_value); virtual int Gettime (struct itimerspec *cur_value) const; - + virtual int Ftruncate (off_t length); virtual bool HangupReceived (void) const; - virtual int Poll (PollTable* ptable); - virtual int Ftruncate (off_t length); virtual int Fsync (void); - void PollEvent (int flag); - - private: + // Kernel system call; on error, sets the current thread's errno. + long Call (long no, long a0 = 0, long a1 = 0, long a2 = 0, + long a3 = 0, long a4 = 0, long a5 = 0) const; + Ptr m_factory; - struct SimSocket *m_socket; - int m_statusFlags; + int m_fd; }; } // namespace ns3 diff --git a/model/linux-socket-fd-factory.cc b/model/linux-socket-fd-factory.cc index 441b92df..db53c9b2 100644 --- a/model/linux-socket-fd-factory.cc +++ b/model/linux-socket-fd-factory.cc @@ -1,18 +1,8 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ #include "linux-socket-fd-factory.h" -#include "include/sim-init.h" -#include "kernel-socket-fd-factory.h" -#include "loader-factory.h" -#include "utils.h" -#include "ns3/log.h" -#include "ns3/string.h" -#include "ns3/simulator.h" -#include "ns3/event-id.h" -#include "ns3/node.h" - - -NS_LOG_COMPONENT_DEFINE ("DceLinuxSocketFdFactory"); namespace ns3 { + NS_OBJECT_ENSURE_REGISTERED (LinuxSocketFdFactory); TypeId @@ -21,149 +11,8 @@ LinuxSocketFdFactory::GetTypeId (void) static TypeId tid = TypeId ("ns3::LinuxSocketFdFactory") .SetParent () .AddConstructor () - .AddAttribute ("Library", "File to load in memory", - StringValue ("liblinux.so"), - MakeStringAccessor (&KernelSocketFdFactory::m_library), - MakeStringChecker ()) ; return tid; } -LinuxSocketFdFactory::LinuxSocketFdFactory () -{ -} - -LinuxSocketFdFactory::~LinuxSocketFdFactory () -{ -} - -void -LinuxSocketFdFactory::NotifyNewAggregate (void) -{ - NS_LOG_FUNCTION (this); - Ptr node = this->GetObject (); - Ptr loaderFactory = this->GetObject (); - Ptr taskManager = this->GetObject (); - if (node && loaderFactory && taskManager && m_loader == 0) - { - m_manager = taskManager; - m_loader = loaderFactory->Create (0, 0, 0); - UtilsEnsureDirectoryExists (UtilsGetAbsRealFilePath (node->GetId (), "/var")); - UtilsEnsureDirectoryExists (UtilsGetAbsRealFilePath (node->GetId (), "/var/log")); - std::string path = UtilsGetAbsRealFilePath (node->GetId (), "/var/log/messages"); - m_logFile = fopen (path.c_str (), "w"); - setlinebuf (m_logFile); - // must use ScheduleWithContext to ensure that the initialization task gets - // a node context to be able to retrieve the task manager when it runs. - // i.e., TaskManager::Current() needs it. - Simulator::ScheduleWithContext (node->GetId (), Seconds (0.0), - &LinuxSocketFdFactory::ScheduleTask, this, - MakeEvent (&LinuxSocketFdFactory::InitializeStack, this)); - } -} - -void -LinuxSocketFdFactory::SetTask (std::string path, std::string value) -{ - NS_LOG_FUNCTION (path << value); - std::vector > files = GetSysFileList (); - for (uint32_t i = 0; i < files.size (); i++) - { - if (files[i].first == path) - { - const char *s = value.c_str (); - int toWrite = value.size (); - int written; - written = m_exported->sys_file_write (files[i].second, s, toWrite, 0); - break; - } - } -} - -void -LinuxSocketFdFactory::Set (std::string path, std::string value) -{ - if (!m_manager) - { - m_earlySysfs.push_back (std::make_pair (path,value)); - } - else - { - KernelSocketFdFactory::ScheduleTask (MakeEvent (&LinuxSocketFdFactory::SetTask, this, path, value)); - } -} - -std::string -LinuxSocketFdFactory::Get (std::string path) -{ - NS_LOG_FUNCTION (path); - std::string ret; - std::vector > files = GetSysFileList (); - for (uint32_t i = 0; i < files.size (); i++) - { - if (files[i].first == path) - { - char buffer[512]; - memset (buffer, 0, sizeof(buffer)); - m_exported->sys_file_read (files[i].second, buffer, sizeof(buffer), 0); - NS_LOG_FUNCTION ("sysctl read: " << buffer); - ret = std::string (buffer); - break; - } - } - return ret; -} - -std::vector > -LinuxSocketFdFactory::GetSysFileList (void) -{ - struct MyIterator - { - struct SimSysIterator head; - static void ReportStartDir (const struct SimSysIterator *iter, const char *dirname) - { - struct MyIterator *self = (struct MyIterator *)iter; - self->m_stack.push_back (self->m_currentPath); - self->m_currentPath += "." + std::string (dirname); - } - static void ReportEndDir (const struct SimSysIterator *iter) - { - struct MyIterator *self = (struct MyIterator *)iter; - self->m_currentPath = self->m_stack.back (); - self->m_stack.pop_back (); - } - static void ReportFile (const struct SimSysIterator *iter, const char *filename, int flags, struct SimSysFile *file) - { - struct MyIterator *self = (struct MyIterator *)iter; - std::string path = self->m_currentPath + "." + filename; - self->m_list.push_back (std::make_pair (path, file)); - } - std::vector m_stack; - std::vector > m_list; - std::string m_currentPath; - } iter; - iter.head.report_start_dir = &MyIterator::ReportStartDir; - iter.head.report_end_dir = &MyIterator::ReportEndDir; - iter.head.report_file = &MyIterator::ReportFile; - m_loader->NotifyStartExecute (); - m_exported->sys_iterate_files ((struct SimSysIterator *)&iter); - m_loader->NotifyEndExecute (); - return iter.m_list; -} - -void -LinuxSocketFdFactory::InitializeStack (void) -{ - KernelSocketFdFactory::InitializeStack (); - Set (".net.ipv4.conf.all.forwarding", "1"); - Set (".net.ipv4.conf.all.log_martians", "1"); - Set (".net.ipv6.conf.all.forwarding", "0"); - - while (!m_earlySysfs.empty ()) - { - std::pair op = m_earlySysfs.front (); - Set (op.first, op.second); - m_earlySysfs.pop_front (); - } -} } // namespace ns3 diff --git a/model/linux-socket-fd-factory.h b/model/linux-socket-fd-factory.h index a383c585..917aaace 100644 --- a/model/linux-socket-fd-factory.h +++ b/model/linux-socket-fd-factory.h @@ -1,61 +1,20 @@ #ifndef LINUX_SOCKET_FD_FACTORY_H #define LINUX_SOCKET_FD_FACTORY_H -#ifdef LKL_LINUX - -#include "lkl-socket-fd-factory.h" +#include "kernel-socket-fd-factory.h" namespace ns3 { /** - * Without libos (configure --with-lkl only), the Linux network stack is - * LKL; see LklSocketFdFactory. + * The Linux network stack: the Linux kernel as a library (LKL); see + * KernelSocketFdFactory. */ -class LinuxSocketFdFactory : public LklSocketFdFactory -{ -public: - static TypeId GetTypeId (void); -}; - -} // namespace ns3 - -#else - -#include "kernel-socket-fd-factory.h" -#include - -extern "C" { -struct SimExported; -struct SimDevice; -struct SimSocket; -struct SimTask; -struct SimKernel; -struct SimSysFile; -} - -namespace ns3 { - class LinuxSocketFdFactory : public KernelSocketFdFactory { public: static TypeId GetTypeId (void); - LinuxSocketFdFactory (); - virtual ~LinuxSocketFdFactory (); - - void Set (std::string path, std::string value); - std::string Get (std::string path); - -private: - virtual void NotifyNewAggregate (void); - void InitializeStack (void); - std::vector > GetSysFileList (void); - void SetTask (std::string path, std::string value); - - std::list > m_earlySysfs; }; } // namespace ns3 -#endif /* LKL_LINUX */ - #endif /* LINUX_SOCKET_FD_FACTORY_H */ diff --git a/model/lkl/lkl-linux-socket-fd-factory.cc b/model/lkl/lkl-linux-socket-fd-factory.cc deleted file mode 100644 index f31e9a3c..00000000 --- a/model/lkl/lkl-linux-socket-fd-factory.cc +++ /dev/null @@ -1,18 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -#include "linux-socket-fd-factory.h" - -namespace ns3 { - -NS_OBJECT_ENSURE_REGISTERED (LinuxSocketFdFactory); - -TypeId -LinuxSocketFdFactory::GetTypeId (void) -{ - static TypeId tid = TypeId ("ns3::LinuxSocketFdFactory") - .SetParent () - .AddConstructor () - ; - return tid; -} - -} // namespace ns3 diff --git a/model/lkl/lkl-socket-fd-factory.cc b/model/lkl/lkl-socket-fd-factory.cc deleted file mode 100644 index 62c8d58d..00000000 --- a/model/lkl/lkl-socket-fd-factory.cc +++ /dev/null @@ -1,636 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -#include "lkl-socket-fd-factory.h" -#include "lkl-socket-fd.h" -#include "lkl-kernel.h" -#include "task-manager.h" -#include "ns3/log.h" -#include "ns3/node.h" -#include "ns3/packet.h" -#include "ns3/simulator.h" -#include "ns3/string.h" -#include "ns3/boolean.h" -#include "ns3/mac48-address.h" -#include "ns3/loopback-net-device.h" -#include "ns3/make-event.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -NS_LOG_COMPONENT_DEFINE ("DceLklSocketFdFactory"); - -/* - * LKL's network device interface (tools/lkl/include/lkl_host.h and lkl.h). - * Those headers are not valid C++, so the few definitions needed here are - * repeated; they must match the LKL version built by utils/build_lkl.sh. - */ -extern "C" { -struct lkl_netdev; -struct lkl_dev_net_ops -{ - int (*tx)(struct lkl_netdev *nd, struct iovec *iov, int cnt); - int (*rx)(struct lkl_netdev *nd, struct iovec *iov, int cnt); - int (*poll)(struct lkl_netdev *nd); - void (*poll_hup)(struct lkl_netdev *nd); - void (*free)(struct lkl_netdev *nd); -}; -struct lkl_netdev -{ - struct lkl_dev_net_ops *ops; - int id; - uint8_t has_vnet_hdr : 1; - uint8_t mac[6]; -}; -struct lkl_netdev_args -{ - void *mac; - unsigned int offload; -}; -} -#define LKL_DEV_NET_POLL_RX 1 -#define LKL_DEV_NET_POLL_HUP 4 - -namespace ns3 { - -NS_OBJECT_ENSURE_REGISTERED (LklSocketFdFactory); - -/** - * A kernel network device backed by an ns-3 NetDevice. LKL drives it - * through the network device host operations below. - */ -struct LklNetDevice -{ - struct lkl_netdev nd; // must stay first: LKL passes &nd to the operations - LklSocketFdFactory *factory; - Ptr device; - std::deque > rx; // received Ethernet frames - Task *poller; - bool hup; - // The kernel had no receive buffers for the queued frames: it takes them - // when it adds buffers, so do not report them again. - bool stalled; -}; - -namespace { - -const uint32_t LKL_TASK_STACK = 1 << 18; -const uint32_t ETH_HEADER = 14; -// Received frames waiting for the kernel, like the ring of a network card. -const uint32_t RX_BACKLOG = 1024; - -LklNetDevice * -DeviceOf (struct lkl_netdev *nd) -{ - return (LklNetDevice *)nd; -} - -void -SendMain (Ptr device, Ptr p, Mac48Address dest, uint16_t protocol) -{ - device->Send (p, dest, protocol); -} - -// The kernel sends a frame: hand it to the ns-3 device. -int -NetTx (struct lkl_netdev *nd, struct iovec *iov, int cnt) -{ - LklNetDevice *dev = DeviceOf (nd); - std::vector frame; - for (int i = 0; i < cnt; i++) - { - const uint8_t *base = (const uint8_t *)iov[i].iov_base; - frame.insert (frame.end (), base, base + iov[i].iov_len); - } - if (frame.size () < ETH_HEADER) - { - return frame.size (); - } - Mac48Address dest; - dest.CopyFrom (&frame[0]); - uint16_t protocol = (frame[12] << 8) | frame[13]; - NS_LOG_DEBUG ("tx " << dev->device << " " << dest << " protocol " << protocol - << " " << frame.size () << " bytes"); - Ptr p = Create (&frame[ETH_HEADER], frame.size () - ETH_HEADER); - TaskManager::Current ()->ExecOnMain (MakeEvent (&SendMain, dev->device, p, dest, protocol)); - return frame.size (); -} - -// The kernel reads the next received frame, if any. -int -NetRx (struct lkl_netdev *nd, struct iovec *iov, int cnt) -{ - LklNetDevice *dev = DeviceOf (nd); - if (dev->rx.empty ()) - { - return -1; - } - std::vector &frame = dev->rx.front (); - size_t done = 0; - for (int i = 0; i < cnt && done < frame.size (); i++) - { - size_t n = std::min (iov[i].iov_len, frame.size () - done); - memcpy (iov[i].iov_base, &frame[done], n); - done += n; - } - dev->rx.pop_front (); - dev->stalled = false; - return done; -} - -// Blocks the device's poll task until the kernel can take a received frame. -// Sending never blocks, so transmission readiness is never reported. -int -NetPoll (struct lkl_netdev *nd) -{ - LklNetDevice *dev = DeviceOf (nd); - TaskManager *manager = TaskManager::Current (); - while ((dev->rx.empty () || dev->stalled) && !dev->hup) - { - dev->poller = manager->RunningTask (); - manager->Sleep (); - } - dev->poller = 0; - if (dev->hup) - { - return LKL_DEV_NET_POLL_HUP; - } - // Until the kernel takes a frame or another one arrives. - dev->stalled = true; - return LKL_DEV_NET_POLL_RX; -} - -void -NetPollHup (struct lkl_netdev *nd) -{ - LklNetDevice *dev = DeviceOf (nd); - dev->hup = true; - if (dev->poller) - { - TaskManager::Current ()->Wakeup (dev->poller); - } -} - -void -NetFree (struct lkl_netdev *nd) -{ - // The factory owns the device. -} - -struct lkl_dev_net_ops g_netOps = { - &NetTx, &NetRx, &NetPoll, &NetPollHup, &NetFree -}; - -} // namespace - -TypeId -LklSocketFdFactory::GetTypeId (void) -{ - static TypeId tid = TypeId ("ns3::LklSocketFdFactory") - .SetParent () - .AddConstructor () - .AddAttribute ("Library", "The LKL shared library to load.", - StringValue ("liblkl.so"), - MakeStringAccessor (&LklSocketFdFactory::m_library), - MakeStringChecker ()) - .AddAttribute ("CommandLine", "The kernel command line.", - StringValue ("mem=64M loglevel=4"), - MakeStringAccessor (&LklSocketFdFactory::m_cmdline), - MakeStringChecker ()) - .AddAttribute ("Mptcp", - "Create MPTCP sockets for the TCP sockets of applications, " - "like mptcpize: Linux uses MPTCP only when asked for.", - BooleanValue (false), - MakeBooleanAccessor (&LklSocketFdFactory::m_mptcp), - MakeBooleanChecker ()) - ; - return tid; -} - -LklSocketFdFactory::LklSocketFdFactory () - : m_mptcp (false), - m_booted (false), - m_nextDevice (0), - m_epollFd (-1) -{ -} - -LklSocketFdFactory::~LklSocketFdFactory () -{ - for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) - { - delete *i; - } -} - -void -LklSocketFdFactory::DoDispose (void) -{ - // The devices refer to their node, which refers to this factory. - for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) - { - (*i)->device = 0; - } - m_kernel = 0; - SocketFdFactory::DoDispose (); -} - -Ptr -LklSocketFdFactory::GetKernel (void) const -{ - return m_kernel; -} - -void -LklSocketFdFactory::NotifyNewAggregate (void) -{ - Ptr node = GetObject (); - Ptr manager = GetObject (); - if (node && manager && !m_kernel) - { - m_kernel = CreateObject (); - // Scripts written for libos name its library; LKL replaces it. - std::string library = m_library == "liblinux.so" ? "liblkl.so" : m_library; - m_kernel->SetAttribute ("Library", StringValue (library)); - m_kernel->SetAttribute ("CommandLine", StringValue (m_cmdline)); - node->AggregateObject (m_kernel); - Simulator::ScheduleWithContext (node->GetId (), Seconds (0), - &TaskManager::Start, PeekPointer (manager), - &LklSocketFdFactory::BootTrampoline, (void *)this, - LKL_TASK_STACK); - } - SocketFdFactory::NotifyNewAggregate (); -} - -void -LklSocketFdFactory::BootTrampoline (void *context) -{ - ((LklSocketFdFactory *)context)->Boot (); - TaskManager::Current ()->Exit (); -} - -void -LklSocketFdFactory::Boot (void) -{ - NS_LOG_FUNCTION (this); - m_kernel->Boot (); - // procfs, for sysctls. - long ret = Call (__lkl__NR_mkdirat, AT_FDCWD, (long)"/proc", 0555); - NS_ASSERT_MSG (ret == 0, "mkdir /proc failed: " << ret); - ret = Call (__lkl__NR_mount, (long)"proc", (long)"/proc", (long)"proc", 0, 0); - NS_ASSERT_MSG (ret == 0, "mount /proc failed: " << ret); - m_epollFd = Call (__lkl__NR_epoll_create1, 0); - NS_ASSERT_MSG (m_epollFd >= 0, "epoll_create1 failed: " << m_epollFd); - m_kernel->GetTaskManager ()->Start (&LklSocketFdFactory::WatcherTrampoline, this, LKL_TASK_STACK); - // Called right away for the devices the node already has. - GetObject ()->RegisterDeviceAdditionListener (MakeCallback (&LklSocketFdFactory::NotifyAddDevice, this)); - m_booted = true; - for (std::list::iterator i = m_bootWaiters.begin (); i != m_bootWaiters.end (); ++i) - { - m_kernel->GetTaskManager ()->Wakeup (*i); - } - m_bootWaiters.clear (); - for (std::list >::iterator i = m_earlySysctls.begin (); - i != m_earlySysctls.end (); ++i) - { - SetTask (i->first, i->second); - } - m_earlySysctls.clear (); -} - -namespace { -std::string -SysctlFile (std::string path) -{ - if (!path.empty () && path[0] == '/') - { - return path; // any kernel file, e.g. /proc/net/netstat - } - std::string file = "/proc/sys"; - if (path.empty () || path[0] != '.') - { - file += '/'; - } - for (std::string::iterator i = path.begin (); i != path.end (); ++i) - { - file += *i == '.' ? '/' : *i; - } - return file; -} -} // namespace - -void -LklSocketFdFactory::Set (std::string path, std::string value) -{ - NS_LOG_FUNCTION (this << path << value); - if (!m_booted) - { - m_earlySysctls.push_back (std::make_pair (path, value)); - return; - } - ScheduleTask (MakeEvent (&LklSocketFdFactory::SetTask, this, path, value)); -} - -void -LklSocketFdFactory::SetTask (std::string path, std::string value) -{ - std::string file = SysctlFile (path); - long fd = Call (__lkl__NR_openat, AT_FDCWD, (long)file.c_str (), O_WRONLY); - if (fd < 0) - { - NS_LOG_WARN ("sysctl " << path << ": cannot open " << file << ": " << fd); - return; - } - long ret = Call (__lkl__NR_write, fd, (long)value.c_str (), value.size ()); - if (ret < 0) - { - NS_LOG_WARN ("sysctl " << path << " = " << value << ": " << ret); - } - Call (__lkl__NR_close, fd); -} - -std::string -LklSocketFdFactory::Get (std::string path) -{ - NS_LOG_FUNCTION (this << path); - WaitBooted (); - std::string file = SysctlFile (path); - long fd = Call (__lkl__NR_openat, AT_FDCWD, (long)file.c_str (), O_RDONLY); - if (fd < 0) - { - NS_LOG_WARN ("sysctl " << path << ": cannot open " << file << ": " << fd); - return ""; - } - std::string value; - char buffer[4096]; - long n; - while ((n = Call (__lkl__NR_read, fd, (long)buffer, sizeof (buffer))) > 0) - { - value.append (buffer, n); - } - Call (__lkl__NR_close, fd); - return value; -} - -void -LklSocketFdFactory::ScheduleTask (EventImpl *event) -{ - m_kernel->GetTaskManager ()->Start (&LklSocketFdFactory::ScheduleTaskTrampoline, - event, LKL_TASK_STACK); -} - -void -LklSocketFdFactory::ScheduleTaskTrampoline (void *context) -{ - EventImpl *event = (EventImpl *)context; - event->Invoke (); - event->Unref (); - TaskManager::Current ()->Exit (); -} - -void -LklSocketFdFactory::WaitBooted (void) -{ - TaskManager *manager = TaskManager::Current (); - while (!m_booted) - { - m_bootWaiters.push_back (manager->RunningTask ()); - manager->Sleep (); - } -} - -long -LklSocketFdFactory::Call (long no, long a0, long a1, long a2, long a3, long a4, long a5) -{ - long params[6] = { a0, a1, a2, a3, a4, a5 }; - return m_kernel->Syscall (no, params); -} - -namespace { -struct AddDeviceContext -{ - LklSocketFdFactory *factory; - Ptr device; -}; -} // namespace - -void -LklSocketFdFactory::NotifyAddDevice (Ptr device) -{ - TaskManager *manager = m_kernel->GetTaskManager (); - if (TaskManager::Current () == manager && manager->RunningTask ()) - { - AddDevice (device); - return; - } - // Adding a kernel device needs a task. - AddDeviceContext *context = new AddDeviceContext (); - context->factory = this; - context->device = device; - manager->Start (&LklSocketFdFactory::AddDeviceTrampoline, context, LKL_TASK_STACK); -} - -void -LklSocketFdFactory::AddDeviceTrampoline (void *c) -{ - AddDeviceContext *context = (AddDeviceContext *)c; - context->factory->AddDevice (context->device); - delete context; - TaskManager::Current ()->Exit (); -} - -void -LklSocketFdFactory::AddDevice (Ptr device) -{ - NS_LOG_FUNCTION (this << device); - if (DynamicCast (device)) - { - return; // the kernel has its own loopback device - } - LklNetDevice *dev = new LklNetDevice (); - memset (&dev->nd, 0, sizeof (dev->nd)); - dev->nd.ops = &g_netOps; - dev->factory = this; - dev->device = device; - dev->poller = 0; - dev->hup = false; - dev->stalled = false; - m_devices.push_back (dev); - - uint8_t mac[6]; - Address address = device->GetAddress (); - Mac48Address mac48 = Mac48Address::IsMatchingType (address) - ? Mac48Address::ConvertFrom (address) : Mac48Address::Allocate (); - mac48.CopyTo (mac); - struct lkl_netdev_args args; - memset (&args, 0, sizeof (args)); - args.mac = mac; - - typedef int (*NetdevAdd)(struct lkl_netdev *, struct lkl_netdev_args *); - NetdevAdd netdevAdd = (NetdevAdd) m_kernel->Lookup ("lkl_netdev_add"); - int id = netdevAdd (&dev->nd, &args); - NS_ASSERT_MSG (id >= 0, "lkl_netdev_add failed: " << id); - - // Find the new device, named eth, by its address (lkl_netdev_get_ifindex - // looks for eth, which the devices renamed below break), and name it - // sim like the libos stack, with the MTU of the ns-3 device. - long s = Call (__lkl__NR_socket, AF_INET, SOCK_DGRAM, 0); - NS_ASSERT (s >= 0); - struct ifreq ifr; - bool found = false; - for (int ifindex = 1; ifindex < 4096 && !found; ifindex++) - { - memset (&ifr, 0, sizeof (ifr)); - ifr.ifr_ifindex = ifindex; - if (Call (__lkl__NR_ioctl, s, SIOCGIFNAME, (long)&ifr) != 0 - || strncmp (ifr.ifr_name, "eth", 3) != 0 - || Call (__lkl__NR_ioctl, s, SIOCGIFHWADDR, (long)&ifr) != 0) - { - continue; - } - found = memcmp (ifr.ifr_hwaddr.sa_data, mac, 6) == 0; - } - NS_ASSERT_MSG (found, "no kernel device for " << mac48); - long ret; - snprintf (ifr.ifr_newname, IFNAMSIZ, "sim%u", m_nextDevice++); - ret = Call (__lkl__NR_ioctl, s, SIOCSIFNAME, (long)&ifr); - NS_ASSERT_MSG (ret == 0, "SIOCSIFNAME failed: " << ret); - memcpy (ifr.ifr_name, ifr.ifr_newname, IFNAMSIZ); - ifr.ifr_mtu = device->GetMtu (); - ret = Call (__lkl__NR_ioctl, s, SIOCSIFMTU, (long)&ifr); - NS_ASSERT_MSG (ret == 0, "SIOCSIFMTU failed: " << ret); - if (!device->NeedsArp ()) - { - // e.g. point-to-point links, which cannot carry ARP: the kernel then - // sends to neighbors without resolving them. - ret = Call (__lkl__NR_ioctl, s, SIOCGIFFLAGS, (long)&ifr); - NS_ASSERT_MSG (ret == 0, "SIOCGIFFLAGS failed: " << ret); - ifr.ifr_flags |= IFF_NOARP; - ret = Call (__lkl__NR_ioctl, s, SIOCSIFFLAGS, (long)&ifr); - NS_ASSERT_MSG (ret == 0, "SIOCSIFFLAGS failed: " << ret); - } - Call (__lkl__NR_close, s); - - GetObject ()->RegisterProtocolHandler (MakeCallback (&LklSocketFdFactory::RxFromDevice, this), - 0, device, true); -} - -void -LklSocketFdFactory::RxFromDevice (Ptr device, Ptr p, uint16_t protocol, - const Address &from, const Address &to, NetDevice::PacketType type) -{ - LklNetDevice *dev = 0; - for (std::vector::iterator i = m_devices.begin (); i != m_devices.end (); ++i) - { - if ((*i)->device == device) - { - dev = *i; - break; - } - } - if (!dev || dev->rx.size () >= RX_BACKLOG) - { - return; - } - // Rebuild the Ethernet frame the kernel expects. - std::vector frame (ETH_HEADER + p->GetSize ()); - Mac48Address dest = Mac48Address::IsMatchingType (to) - ? Mac48Address::ConvertFrom (to) : Mac48Address::ConvertFrom (device->GetAddress ()); - Mac48Address source = Mac48Address::IsMatchingType (from) - ? Mac48Address::ConvertFrom (from) : Mac48Address ("00:00:00:00:00:00"); - dest.CopyTo (&frame[0]); - source.CopyTo (&frame[6]); - frame[12] = protocol >> 8; - frame[13] = protocol & 0xff; - p->CopyData (&frame[ETH_HEADER], p->GetSize ()); - NS_LOG_DEBUG ("rx " << device << " protocol " << protocol << " " << frame.size () << " bytes"); - dev->rx.push_back (frame); - dev->stalled = false; - if (dev->poller) - { - m_kernel->GetTaskManager ()->Wakeup (dev->poller); - } -} - -UnixFd * -LklSocketFdFactory::CreateSocket (int domain, int type, int protocol) -{ - NS_LOG_FUNCTION (this << domain << type << protocol); - WaitBooted (); - if (m_mptcp && (domain == AF_INET || domain == AF_INET6) - && (type & 0xf) == SOCK_STREAM && (protocol == 0 || protocol == IPPROTO_TCP)) - { - protocol = 262; // IPPROTO_MPTCP - } - long fd = Call (__lkl__NR_socket, domain, type, protocol); - if (fd < 0) - { - return 0; - } - return new LklSocketFd (this, fd); -} - -void -LklSocketFdFactory::Register (LklSocketFd *socket) -{ - m_sockets[socket->GetKernelFd ()] = socket; -} - -void -LklSocketFdFactory::Unregister (LklSocketFd *socket) -{ - m_sockets.erase (socket->GetKernelFd ()); - m_watched.erase (socket->GetKernelFd ()); -} - -void -LklSocketFdFactory::Watch (LklSocketFd *socket) -{ - // Like a kernel wait queue: once watched, every change of the socket's - // state wakes its poll waiters, which check what they wait for. - int fd = socket->GetKernelFd (); - if (m_watched.count (fd)) - { - return; - } - struct epoll_event event; - memset (&event, 0, sizeof (event)); - event.events = EPOLLIN | EPOLLOUT | EPOLLPRI | EPOLLRDHUP | EPOLLET; - event.data.fd = fd; - long ret = Call (__lkl__NR_epoll_ctl, m_epollFd, EPOLL_CTL_ADD, fd, (long)&event); - NS_ASSERT_MSG (ret == 0, "epoll_ctl failed: " << ret); - m_watched.insert (fd); -} - -void -LklSocketFdFactory::WatcherTrampoline (void *context) -{ - ((LklSocketFdFactory *)context)->Watcher (); -} - -void -LklSocketFdFactory::Watcher (void) -{ - struct epoll_event events[32]; - while (true) - { - long n = Call (__lkl__NR_epoll_pwait, m_epollFd, (long)events, 32, -1, 0, 8); - for (long i = 0; i < n; i++) - { - std::map::iterator s = m_sockets.find (events[i].data.fd); - if (s != m_sockets.end ()) - { - s->second->NotifyEvents (events[i].events & 0xffff); - } - } - } -} - -} // namespace ns3 diff --git a/model/lkl/lkl-socket-fd-factory.h b/model/lkl/lkl-socket-fd-factory.h deleted file mode 100644 index be977a3c..00000000 --- a/model/lkl/lkl-socket-fd-factory.h +++ /dev/null @@ -1,94 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -#ifndef LKL_SOCKET_FD_FACTORY_H -#define LKL_SOCKET_FD_FACTORY_H - -#include "socket-fd-factory.h" -#include "ns3/net-device.h" -#include -#include -#include -#include -#include - -namespace ns3 { - -class EventImpl; -class LklKernel; -class LklSocketFd; -class Task; -struct LklNetDevice; - -/** - * Network stack of a node backed by a Linux kernel (LKL): select it with - * DceManagerHelper::SetNetworkStack ("ns3::LklSocketFdFactory"). - * - * The kernel boots when the simulation starts. Every ns-3 NetDevice of the - * node becomes a kernel network device named sim, in device order; - * sockets of DCE applications are kernel sockets. - */ -class LklSocketFdFactory : public SocketFdFactory -{ -public: - static TypeId GetTypeId (void); - LklSocketFdFactory (); - virtual ~LklSocketFdFactory (); - - virtual UnixFd * CreateSocket (int domain, int type, int protocol); - - Ptr GetKernel (void) const; - - /** - * Write a sysctl, like libos: path ".net.ipv4.tcp_sack" is - * /proc/sys/net/ipv4/tcp_sack. Sysctls set before the kernel booted are - * written right after boot. - */ - void Set (std::string path, std::string value); - /** - * Read a sysctl, or any kernel file given by its absolute path (e.g. - * /proc/net/netstat); must be called from a task of this node. - */ - std::string Get (std::string path); - /** Run the event in a new task of this node. */ - void ScheduleTask (EventImpl *event); - - // Used by LklSocketFd. - void Register (LklSocketFd *socket); - void Unregister (LklSocketFd *socket); - /** From now on, wake the socket's poll waiters when its state changes. */ - void Watch (LklSocketFd *socket); - -private: - virtual void DoDispose (void); - virtual void NotifyNewAggregate (void); - static void BootTrampoline (void *context); - static void ScheduleTaskTrampoline (void *context); - void SetTask (std::string path, std::string value); - void Boot (void); - void WaitBooted (void); - void NotifyAddDevice (Ptr device); - void AddDevice (Ptr device); - static void AddDeviceTrampoline (void *context); - void RxFromDevice (Ptr device, Ptr p, uint16_t protocol, - const Address &from, const Address &to, NetDevice::PacketType type); - static void WatcherTrampoline (void *context); - void Watcher (void); - long Call (long no, long a0 = 0, long a1 = 0, long a2 = 0, - long a3 = 0, long a4 = 0, long a5 = 0); - - std::string m_library; - std::string m_cmdline; - bool m_mptcp; - Ptr m_kernel; - bool m_booted; - std::list m_bootWaiters; - std::list > m_earlySysctls; - std::vector m_devices; - uint32_t m_nextDevice; - int m_epollFd; - std::map m_sockets; - std::set m_watched; -}; - -} // namespace ns3 - -#endif /* LKL_SOCKET_FD_FACTORY_H */ diff --git a/model/lkl/lkl-socket-fd.cc b/model/lkl/lkl-socket-fd.cc deleted file mode 100644 index 40161aba..00000000 --- a/model/lkl/lkl-socket-fd.cc +++ /dev/null @@ -1,287 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -#include "lkl-socket-fd.h" -#include "lkl-socket-fd-factory.h" -#include "lkl-kernel.h" -#include "process.h" -#include "utils.h" -#include "file-usage.h" -#include "wait-queue.h" -#include "ns3/log.h" -#include -#include -#include -#include -#include -#include -#include - -NS_LOG_COMPONENT_DEFINE ("DceLklSocketFd"); - -namespace ns3 { - -LklSocketFd::LklSocketFd (Ptr factory, int fd) - : m_factory (factory), - m_fd (fd) -{ - NS_LOG_FUNCTION (this << fd); - m_factory->Register (this); -} - -LklSocketFd::~LklSocketFd () -{ -} - -int -LklSocketFd::GetKernelFd (void) const -{ - return m_fd; -} - -void -LklSocketFd::NotifyEvents (short events) -{ - WakeWaiters (&events); -} - -long -LklSocketFd::Call (long no, long a0, long a1, long a2, long a3, long a4, long a5) const -{ - long params[6] = { a0, a1, a2, a3, a4, a5 }; - long ret = m_factory->GetKernel ()->Syscall (no, params); - if (ret < 0) - { - Current ()->err = -ret; - return -1; - } - return ret; -} - -int -LklSocketFd::Close (void) -{ - NS_LOG_FUNCTION (this); - m_factory->Unregister (this); - return Call (__lkl__NR_close, m_fd); -} - -ssize_t -LklSocketFd::Write (const void *buf, size_t count) -{ - return Call (__lkl__NR_write, m_fd, (long)buf, count); -} - -ssize_t -LklSocketFd::Read (void *buf, size_t count) -{ - return Call (__lkl__NR_read, m_fd, (long)buf, count); -} - -ssize_t -LklSocketFd::Recvmsg (struct msghdr *msg, int flags) -{ - return Call (__lkl__NR_recvmsg, m_fd, (long)msg, flags); -} - -ssize_t -LklSocketFd::Sendmsg (const struct msghdr *msg, int flags) -{ - return Call (__lkl__NR_sendmsg, m_fd, (long)msg, flags); -} - -bool -LklSocketFd::Isatty (void) const -{ - return false; -} - -int -LklSocketFd::Setsockopt (int level, int optname, const void *optval, socklen_t optlen) -{ - return Call (__lkl__NR_setsockopt, m_fd, level, optname, (long)optval, optlen); -} - -int -LklSocketFd::Getsockopt (int level, int optname, void *optval, socklen_t *optlen) -{ - return Call (__lkl__NR_getsockopt, m_fd, level, optname, (long)optval, (long)optlen); -} - -int -LklSocketFd::Getsockname (struct sockaddr *name, socklen_t *namelen) -{ - return Call (__lkl__NR_getsockname, m_fd, (long)name, (long)namelen); -} - -int -LklSocketFd::Getpeername (struct sockaddr *name, socklen_t *namelen) -{ - return Call (__lkl__NR_getpeername, m_fd, (long)name, (long)namelen); -} - -int -LklSocketFd::Ioctl (unsigned long request, char *argp) -{ - return Call (__lkl__NR_ioctl, m_fd, request, (long)argp); -} - -int -LklSocketFd::Bind (const struct sockaddr *my_addr, socklen_t addrlen) -{ - return Call (__lkl__NR_bind, m_fd, (long)my_addr, addrlen); -} - -int -LklSocketFd::Connect (const struct sockaddr *my_addr, socklen_t addrlen) -{ - return Call (__lkl__NR_connect, m_fd, (long)my_addr, addrlen); -} - -int -LklSocketFd::Listen (int backlog) -{ - return Call (__lkl__NR_listen, m_fd, backlog); -} - -int -LklSocketFd::Shutdown (int how) -{ - return Call (__lkl__NR_shutdown, m_fd, how); -} - -int -LklSocketFd::Accept (struct sockaddr *my_addr, socklen_t *addrlen) -{ - NS_LOG_FUNCTION (this); - long kernelFd = Call (__lkl__NR_accept4, m_fd, (long)my_addr, (long)addrlen, 0); - if (kernelFd < 0) - { - return -1; - } - Thread *current = Current (); - int fd = UtilsAllocateFd (); - if (fd == -1) - { - Call (__lkl__NR_close, kernelFd); - current->err = EMFILE; - return -1; - } - UnixFd *unixFd = new LklSocketFd (m_factory, kernelFd); - unixFd->IncFdCount (); - current->process->openFiles[fd] = new FileUsage (fd, unixFd); - return fd; -} - -void * -LklSocketFd::Mmap (void *start, size_t length, int prot, int flags, off64_t offset) -{ - Current ()->err = ENODEV; - return MAP_FAILED; -} - -off64_t -LklSocketFd::Lseek (off64_t offset, int whence) -{ - Current ()->err = ESPIPE; - return -1; -} - -int -LklSocketFd::Fxstat (int ver, struct ::stat *buf) -{ - buf->st_mode = S_IFSOCK; - buf->st_dev = -1; - buf->st_blksize = 0; - return 0; -} - -int -LklSocketFd::Fxstat64 (int ver, struct ::stat64 *buf) -{ - buf->st_mode = S_IFSOCK; - buf->st_dev = -1; - buf->st_blksize = 0; - return 0; -} - -int -LklSocketFd::Fcntl (int cmd, unsigned long arg) -{ - switch (cmd) - { - case F_GETFL: - case F_SETFL: - // The kernel keeps O_NONBLOCK and applies it to every operation. - return Call (__lkl__NR_fcntl, m_fd, cmd, arg); - case F_GETFD: - return m_fdFlags; - case F_SETFD: - m_fdFlags = arg; - return 0; - default: - return UnixFd::Fcntl (cmd, arg); - } -} - -int -LklSocketFd::Settime (int flags, const struct itimerspec *new_value, struct itimerspec *old_value) -{ - Current ()->err = EINVAL; - return -1; -} - -int -LklSocketFd::Gettime (struct itimerspec *cur_value) const -{ - Current ()->err = EINVAL; - return -1; -} - -int -LklSocketFd::Ftruncate (off_t length) -{ - Current ()->err = EINVAL; - return -1; -} - -bool -LklSocketFd::HangupReceived (void) const -{ - return false; -} - -int -LklSocketFd::Poll (PollTable* ptable) -{ - short wanted = ptable ? ptable->GetEventMask () : (POLLIN | POLLOUT | POLLPRI | POLLRDHUP); - struct pollfd pfd; - pfd.fd = m_fd; - pfd.events = wanted; - pfd.revents = 0; - struct - { - long long tv_sec; - long long tv_nsec; - } zero = { 0, 0 }; - long params[6] = { (long)&pfd, 1, (long)&zero, 0, 8, 0 }; - long ret = m_factory->GetKernel ()->Syscall (__lkl__NR_ppoll, params); - if (ret < 0) - { - return POLLERR; - } - if (ptable) - { - // Get woken when the kernel reports one of these events. - ptable->PollWait (this); - m_factory->Watch (this); - } - return pfd.revents; -} - -int -LklSocketFd::Fsync (void) -{ - Current ()->err = EINVAL; - return -1; -} - -} // namespace ns3 diff --git a/model/lkl/lkl-socket-fd.h b/model/lkl/lkl-socket-fd.h deleted file mode 100644 index d66b7dec..00000000 --- a/model/lkl/lkl-socket-fd.h +++ /dev/null @@ -1,69 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -#ifndef LKL_SOCKET_FD_H -#define LKL_SOCKET_FD_H - -#include "unix-fd.h" -#include "ns3/ptr.h" - -namespace ns3 { - -class LklSocketFdFactory; - -/** - * A socket of a node's LKL kernel: every operation is the corresponding - * kernel system call on the kernel's file descriptor. - */ -class LklSocketFd : public UnixFd -{ -public: - LklSocketFd (Ptr factory, int fd); - virtual ~LklSocketFd (); - - int GetKernelFd (void) const; - /** Wake the tasks waiting in poll or select for these events. */ - void NotifyEvents (short events); - - virtual int Close (void); - virtual ssize_t Write (const void *buf, size_t count); - virtual ssize_t Read (void *buf, size_t count); - virtual ssize_t Recvmsg (struct msghdr *msg, int flags); - virtual ssize_t Sendmsg (const struct msghdr *msg, int flags); - virtual bool Isatty (void) const; - virtual int Setsockopt (int level, int optname, - const void *optval, socklen_t optlen); - virtual int Getsockopt (int level, int optname, - void *optval, socklen_t *optlen); - virtual int Getsockname (struct sockaddr *name, socklen_t *namelen); - virtual int Getpeername (struct sockaddr *name, socklen_t *namelen); - virtual int Ioctl (unsigned long request, char *argp); - virtual int Bind (const struct sockaddr *my_addr, socklen_t addrlen); - virtual int Connect (const struct sockaddr *my_addr, socklen_t addrlen); - virtual int Listen (int backlog); - virtual int Shutdown (int how); - virtual int Accept (struct sockaddr *my_addr, socklen_t *addrlen); - virtual void * Mmap (void *start, size_t length, int prot, int flags, off64_t offset); - virtual off64_t Lseek (off64_t offset, int whence); - virtual int Fxstat (int ver, struct ::stat *buf); - virtual int Fxstat64 (int ver, struct ::stat64 *buf); - virtual int Fcntl (int cmd, unsigned long arg); - virtual int Settime (int flags, - const struct itimerspec *new_value, - struct itimerspec *old_value); - virtual int Gettime (struct itimerspec *cur_value) const; - virtual int Ftruncate (off_t length); - virtual bool HangupReceived (void) const; - virtual int Poll (PollTable* ptable); - virtual int Fsync (void); - -private: - // Kernel system call; on error, sets the current thread's errno. - long Call (long no, long a0 = 0, long a1 = 0, long a2 = 0, - long a3 = 0, long a4 = 0, long a5 = 0) const; - - Ptr m_factory; - int m_fd; -}; - -} // namespace ns3 - -#endif /* LKL_SOCKET_FD_H */ diff --git a/myscripts/btorrent1/wscript b/myscripts/btorrent1/wscript index feeeed3c..3e58867a 100644 --- a/myscripts/btorrent1/wscript +++ b/myscripts/btorrent1/wscript @@ -6,7 +6,7 @@ def configure(conf): ns3waf.check_modules(conf, ['core', 'internet', 'point-to-point' ], mandatory = True) def build(bld): - if bld.env['KERNEL_STACK']: + if bld.env['LINUX_STACK']: bld.build_a_script('dce', needed = ['core', 'internet', 'dce', 'point-to-point' ], target='bin/dce-bt1', source=['dce-bt1.cc'], diff --git a/myscripts/iperf-multi/wscript b/myscripts/iperf-multi/wscript index cb40a404..c8b11810 100644 --- a/myscripts/iperf-multi/wscript +++ b/myscripts/iperf-multi/wscript @@ -6,7 +6,7 @@ def configure(conf): ns3waf.check_modules(conf, ['core', 'internet', 'point-to-point', 'csma' ], mandatory = True) def build(bld): - if bld.env['KERNEL_STACK']: + if bld.env['LINUX_STACK']: bld.build_a_script('dce', needed = ['core', 'internet', 'dce', 'point-to-point', 'csma' ], target='bin/dce-iperf-multi', source=['dce-iperf-multi.cc'], diff --git a/test/dce-cradle-test.cc b/test/dce-cradle-test.cc index 4da99803..ca715f7f 100644 --- a/test/dce-cradle-test.cc +++ b/test/dce-cradle-test.cc @@ -159,13 +159,7 @@ DceCradleTestSuite::DceCradleTestSuite () {"sctp6", "ns3::LinuxSctp6SocketFactory", 20, false}, }; - // for the moment: not supported mptcp for freebsd -#ifdef LKL_LINUX - // ns3::LinuxSocketFdFactory is LKL std::string filePath = SearchExecFile ("DCE_PATH", "liblkl.so", 0); -#else - std::string filePath = SearchExecFile ("DCE_PATH", "liblinux.so", 0); -#endif for (unsigned int i = 0; i < sizeof(tests)/sizeof(testPair); i++) { if (filePath.length () <= 0) diff --git a/test/dce-manager-test.cc b/test/dce-manager-test.cc index 605447e5..063333f3 100644 --- a/test/dce-manager-test.cc +++ b/test/dce-manager-test.cc @@ -113,18 +113,6 @@ DceManagerTestCase::DoRun (void) stack.SetRoutingHelper (ipv4RoutingHelper); stack.Install (nodes); } - else if (m_netstack == "freebsd") - { - dceManager.SetNetworkStack ("ns3::FreeBSDSocketFdFactory", "Library", StringValue ("libfreebsd.so")); - dceManager.Install (nodes); - - dce.SetBinary ("freebsd-iproute"); - dce.SetStackSize (1 << 16); - dce.ResetArguments (); - dce.ParseArguments ("lo0 127.0.0.1 255.0.0.0"); - apps = dce.Install (nodes.Get (0)); - apps.Start (Seconds (2.0)); - } } else { @@ -161,9 +149,9 @@ static class DceManagerTestSuite : public TestSuite } g_processTests; // -#define NS3_STACK (1 << 0) -#define LINUX_STACK (1 << 1) -#define FREEBSD_STACK (1 << 2) +// The stacks a test runs on (LINUX_STACK is the build's macro). +#define NS3_MASK (1 << 0) +#define LINUX_MASK (1 << 1) DceManagerTestSuite::DceManagerTestSuite () : TestSuite ("dce-process-manager", Type::UNIT) @@ -179,44 +167,44 @@ DceManagerTestSuite::DceManagerTestSuite () } testPair; const testPair tests[] = { - { "test-empty", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-sleep", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-pthread", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-mutex", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-once", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-pthread-key", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-sem", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-malloc", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-malloc-2", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-fd-simple", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-strerror", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-stdio", 0, "/etc/passwd",false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-string", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-iostream", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-netdb", 3600, "", true, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-env", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-cond", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-timer-fd", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-stdlib", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-fork", 0, "", false, true, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-select", 3600, "", true, false, NS3_STACK|LINUX_STACK}, // freebsd-sim has no sock_poll - { "test-nanosleep", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-random", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-local-socket", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-poll", 3200, "", true, false, NS3_STACK|LINUX_STACK}, // freebsd-sim has no sock_poll - { "test-tcp-socket", 320, "", true, false, NS3_STACK|LINUX_STACK}, - { "test-exec", 0, "", false, true, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-raw-socket", 320, "", true, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-iperf", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-name", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-pipe", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-dirent", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-socket", 30, "", true, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-bug-multi-select", 30, "", true, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-tsearch", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-clock-gettime", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-gcc-builtin-apply", 0, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, - { "test-signal", 30, "", false, false, NS3_STACK|LINUX_STACK|FREEBSD_STACK}, + { "test-empty", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-sleep", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-pthread", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-mutex", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-once", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-pthread-key", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-sem", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-malloc", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-malloc-2", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-fd-simple", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-strerror", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-stdio", 0, "/etc/passwd",false, false, NS3_MASK|LINUX_MASK}, + { "test-string", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-iostream", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-netdb", 3600, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-env", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-cond", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-timer-fd", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-stdlib", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-fork", 0, "", false, true, NS3_MASK|LINUX_MASK}, + { "test-select", 3600, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-nanosleep", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-random", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-local-socket", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-poll", 3200, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-tcp-socket", 320, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-exec", 0, "", false, true, NS3_MASK|LINUX_MASK}, + { "test-raw-socket", 320, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-iperf", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-name", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-pipe", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-dirent", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-socket", 30, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-bug-multi-select", 30, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-tsearch", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-clock-gettime", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-gcc-builtin-apply", 0, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-signal", 30, "", false, false, NS3_MASK|LINUX_MASK}, }; // Prepare directories and files for test-stdio @@ -255,7 +243,7 @@ DceManagerTestSuite::DceManagerTestSuite () tests[i].stdinfile, tests[i].useNet, "ns3", - (tests[i].stackMask & NS3_STACK) ? + (tests[i].stackMask & NS3_MASK) ? (isUctxFiber ? tests[i].skipUctx : false) : true ), Duration::QUICK); @@ -264,12 +252,7 @@ DceManagerTestSuite::DceManagerTestSuite () // linux stack TypeId tid; bool kern_linux = TypeId::LookupByNameFailSafe ("ns3::LinuxSocketFdFactory", &tid); -#ifdef LKL_LINUX - // ns3::LinuxSocketFdFactory is LKL std::string filePath = SearchExecFile ("DCE_PATH", "liblkl.so", 0); -#else - std::string filePath = SearchExecFile ("DCE_PATH", "liblinux.so", 0); -#endif if (kern_linux && (filePath.length () > 0)) { for (unsigned int i = 0; i < sizeof(tests) / sizeof(testPair); i++) @@ -278,25 +261,7 @@ DceManagerTestSuite::DceManagerTestSuite () tests[i].stdinfile, tests[i].useNet, "linux", - (tests[i].stackMask & LINUX_STACK) ? - (isUctxFiber ? tests[i].skipUctx : false) : true - ), - Duration::QUICK); - } - } - - // FreeBSD - bool kern_freebsd = TypeId::LookupByNameFailSafe ("ns3::FreeBSDSocketFdFactory", &tid); - filePath = SearchExecFile ("DCE_PATH", "libfreebsd.so", 0); - if (kern_freebsd && (filePath.length () > 0)) - { - for (unsigned int i = 0; i < sizeof(tests) / sizeof(testPair); i++) - { - AddTestCase (new DceManagerTestCase (tests[i].name, Seconds (tests[i].duration), - tests[i].stdinfile, - tests[i].useNet, - "freebsd", - (tests[i].stackMask & FREEBSD_STACK) ? + (tests[i].stackMask & LINUX_MASK) ? (isUctxFiber ? tests[i].skipUctx : false) : true ), Duration::QUICK); diff --git a/test/dce-mptcp-test.cc b/test/dce-mptcp-test.cc index 0146e032..ff0222f8 100644 --- a/test/dce-mptcp-test.cc +++ b/test/dce-mptcp-test.cc @@ -89,7 +89,7 @@ SetMptcpEnabled (std::string key, std::string value) // Let each connection have more subflows and accept the addresses its // peer announces. The applications' TCP sockets are made MPTCP sockets -// (LklSocketFdFactory::Mptcp) at the start of each test. +// (KernelSocketFdFactory::Mptcp) at the start of each test. static void EnableMptcp (NodeContainer nodes) { @@ -105,7 +105,7 @@ DceMptcpTestCase::DoBasicRun (void) g_rcv0 = false; g_rcv1 = false; isMptcpEnabled = false; - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); uint32_t nRtrs = 2; std::string m_rate = "200Bps"; @@ -225,7 +225,7 @@ DceMptcpTestCase::DoBasicRun (void) " bytes"; std::cout << std::endl; Simulator::Destroy (); - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (false)); + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (false)); if (!isMptcpEnabled) { @@ -243,7 +243,7 @@ DceMptcpTestCase::DoAddrTestRun (void) g_rcv0 = false; g_rcv1 = false; isMptcpEnabled = false; - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (true)); + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (true)); double stopTime = 15.0; std::string p2pdelay = "10ms"; @@ -332,7 +332,7 @@ DceMptcpTestCase::DoAddrTestRun (void) Simulator::Stop (Seconds (stopTime)); Simulator::Run (); Simulator::Destroy (); - Config::SetDefaultFailSafe ("ns3::LklSocketFdFactory::Mptcp", BooleanValue (false)); + Config::SetDefault ("ns3::KernelSocketFdFactory::Mptcp", BooleanValue (false)); if (!isMptcpEnabled) { @@ -391,13 +391,7 @@ DceMptcpTestSuite::DceMptcpTestSuite () }; Packet::EnablePrinting (); - // for the moment: not supported dce cradle for freebsd -#ifdef LKL_LINUX - // ns3::LinuxSocketFdFactory is LKL std::string filePath = SearchExecFile ("DCE_PATH", "liblkl.so", 0); -#else - std::string filePath = SearchExecFile ("DCE_PATH", "liblinux.so", 0); -#endif for (unsigned int i = 0; i < sizeof(tests)/sizeof(testPair); i++) { if (filePath.length () <= 0) diff --git a/utils/build_kernel_deps.sh b/utils/build_kernel_deps.sh index 0f91e417..1b7c5d09 100755 --- a/utils/build_kernel_deps.sh +++ b/utils/build_kernel_deps.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Build everything DCE needs to run its kernel-stack tests and examples: -# - net-next-nuse (Linux 4.4 libos) -> liblinux.so, and the kernel tree -# used by "waf configure --enable-kernel-stack" -# - freebsd-sim -> libfreebsd.so +# Build the programs DCE's kernel-stack tests and examples run (the kernel +# itself is LKL, built by utils/build_lkl.sh): # - DCE-compatible (PIE) builds of ip, iperf, thttpd, wget, ping/ping6 # and the quagga routing daemons # - the ns-3-dce-quagga module sources, patched for current ns-3 @@ -11,13 +9,10 @@ # Usage: ./utils/build_kernel_deps.sh [deps_dir] # deps_dir: output directory (default: ../dce-kernel-deps) # JOBS: parallel make jobs (default: nproc) -# LIBOS: 0 skips net-next-nuse and freebsd-sim, for builds whose Linux -# stack is LKL (utils/build_lkl.sh) (default: 1) # # Outputs (sources and build trees stay in /src, which can be # deleted afterwards): # /bin_dce/ binaries and libraries for DCE_PATH -# /kernel/ argument for --enable-kernel-stack (LIBOS=1) # /ns-3-dce-quagga/ module to copy into myscripts/ # @@ -28,10 +23,7 @@ DEPS_DIR="$(mkdir -p "${1:-${DCE_DIR}/../dce-kernel-deps}" && cd "${1:-${DCE_DIR BIN_DCE="${DEPS_DIR}/bin_dce" SRC="${DEPS_DIR}/src" JOBS="${JOBS:-$(nproc)}" -LIBOS="${LIBOS:-1}" -NUSE_REV="libos-v4.4-fix1" -FREEBSD_REV="sim-ns3-dev-branch" IPROUTE2_REV="v6.12.0" # matches LKL's kernel; has "ip mptcp" IPUTILS_REV="s20101006" QUAGGA_MODULE_REV="b57e0f3184e34107c7e452b3c41807e0aff5b5ce" @@ -69,54 +61,6 @@ fetch_tar () { fi } -if [ "${LIBOS}" = 1 ]; then - echo "== net-next-nuse (Linux 4.4 libos)" - fetch_git https://github.com/libos-nuse/net-next-nuse.git "${NUSE_REV}" net-next-nuse - ( - cd "${SRC}/net-next-nuse" - make defconfig ARCH=lib > /dev/null - # Generate objs.mk serially: under -j, GNU Make >= 4.4 runs this rule - # twice concurrently and the output file gets interleaved. - make ARCH=lib arch/lib/objs.mk > /dev/null - # The NUSE/rump tools fail to build and are not used by DCE; only the - # libsim-linux library from arch/lib/tools is required. - make library ARCH=lib -j"${JOBS}" -k > build.log 2>&1 || true - test -f arch/lib/tools/libsim-linux-4.4.0.so || { tail -50 build.log; exit 1; } - ) - cp "${SRC}/net-next-nuse/arch/lib/tools/libsim-linux-4.4.0.so" "${BIN_DCE}/" - ln -sf libsim-linux-4.4.0.so "${BIN_DCE}/liblinux.so" - # DCE only needs the libos API headers from the kernel tree. - rm -rf "${DEPS_DIR}/kernel" - mkdir -p "${DEPS_DIR}/kernel/lib" - cp -a "${SRC}/net-next-nuse/arch/lib/include" "${DEPS_DIR}/kernel/lib/" - - echo "== freebsd-sim" - fetch_git https://github.com/direct-code-execution/freebsd-sim.git "${FREEBSD_REV}" freebsd-sim - ( - cd "${SRC}/freebsd-sim" - git apply --check "${DCE_DIR}/utils/freebsd-sim-dce.patch" 2> /dev/null \ - && git apply "${DCE_DIR}/utils/freebsd-sim-dce.patch" - cd sys/sim - # The kernel config tool and generated headers must exist before the - # parallel kernel build; the Makefile does not order this correctly. - make -f Makefile.config clean > /dev/null - make -f Makefile.config CC="gcc ${LEGACY_CFLAGS}" > /dev/null - rm -rf compile && mkdir -p compile - (cd conf && ../config -C DCE > /dev/null) - ( - cd compile/DCE - for m in device_if cpufreq_if bus_if linker_if clock_if; do - awk -f ../../../tools/makeobjops.awk ../../../kern/$m.m -h - done - awk -f ../../../tools/vnode_if.awk ../../../kern/vnode_if.src -q -p -h -c - ) - touch config - make buildkernel -j"${JOBS}" CC="gcc ${LEGACY_CFLAGS}" > build.log 2>&1 || { tail -50 build.log; exit 1; } - ) - cp "${SRC}/freebsd-sim/libsim-freebsd.git.so" "${BIN_DCE}/" - ln -sf libsim-freebsd.git.so "${BIN_DCE}/libfreebsd.so" -fi - echo "== iproute2 ${IPROUTE2_REV}" fetch_git https://git.kernel.org/pub/scm/network/iproute2/iproute2.git "${IPROUTE2_REV}" iproute2 ( diff --git a/utils/freebsd-sim-dce.patch b/utils/freebsd-sim-dce.patch deleted file mode 100644 index 5a655935..00000000 --- a/utils/freebsd-sim-dce.patch +++ /dev/null @@ -1,52 +0,0 @@ ---- a/sys/sim/generate-linker-script.py -+++ b/sys/sim/generate-linker-script.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/env python3 - - import re - ---- a/usr.sbin/config/main.c -+++ b/usr.sbin/config/main.c -@@ -121,7 +121,7 @@ - int * - __error(void) - { -- int test = 0; -+ static int test = 0; - return (&test); - } - ---- a/sys/sim/sim-device.c -+++ b/sys/sim/sim-device.c -@@ -525,7 +525,7 @@ - - static const u_char etherbroadcastaddr[ETHER_ADDR_LEN] = - { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; --struct SimDevice *sim_dev_create (void *priv, enum SimDevFlags flags) -+struct SimDevice *sim_dev_create (const char *ifname, void *priv, enum SimDevFlags flags) - { - /* FIXME!! */ - static uint32_t devnum = 0; ---- a/sys/sim/sim.c -+++ b/sys/sim/sim.c -@@ -79,7 +79,7 @@ - extern void sim_task_destroy (struct SimTask *task); - extern void *sim_task_get_private (struct SimTask *task); - --FORWARDER2(sim_dev_create, nvoid, struct SimDevice *, void *, enum SimDevFlags); -+FORWARDER3(sim_dev_create, nvoid, struct SimDevice *, const char *, void *, enum SimDevFlags); - FORWARDER1(sim_dev_destroy, void, void, struct SimDevice *); - FORWARDER2(sim_dev_set_address, void, void, struct SimDevice *, unsigned char *); - FORWARDER2(sim_dev_set_mtu, void, void, struct SimDevice *, int); ---- a/sys/sim/include/sim-init.h -+++ b/sys/sim/include/sim-init.h -@@ -38,7 +38,7 @@ - void (*sock_poll) (struct SimSocket *socket, void *ret); - void (*sock_pollfreewait) (void *polltable); - -- struct SimDevice *(*dev_create) (void *priv, enum SimDevFlags flags); -+ struct SimDevice *(*dev_create) (const char *ifname, void *priv, enum SimDevFlags flags); - void (*dev_destroy) (struct SimDevice *dev); - void *(*dev_get_private) (struct SimDevice *task); - void (*dev_set_address) (struct SimDevice *dev, unsigned char buffer[6]); diff --git a/utils/mptcp-build.sh b/utils/mptcp-build.sh deleted file mode 100644 index 311ec149..00000000 --- a/utils/mptcp-build.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -ex -# See dce-user-kernel.rst for more information -# You should run this from the mptcp repository on tag 0.89 -git remote add dce git://github.com/thehajime/net-next-sim.git || echo "ignore" -git fetch dce -git merge dce/sim-ns3-3.14.0-branch --no-commit - -cat >> arch/sim/defconfig < Date: Sat, 10 Oct 2026 11:58:08 +0200 Subject: [PATCH 10/71] Fail the process tests whose program does not end A process still running when the simulation stops is deleted with it, and its finished callback then reported success: a program that hung passed. The programs that do not end today are listed explicitly. Co-Authored-By: Claude Opus 5.5 --- test/dce-manager-test.cc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/dce-manager-test.cc b/test/dce-manager-test.cc index 063333f3..638466b2 100644 --- a/test/dce-manager-test.cc +++ b/test/dce-manager-test.cc @@ -53,10 +53,24 @@ DceManagerTestCase::DceManagerTestCase (std::string filename, Time maxDuration, { // mtrace (); } +// False once the simulation stopped: the processes that end then did not +// end, they are deleted with the simulation. +static bool g_running = false; + void DceManagerTestCase::Finished (int *pstatus, uint16_t pid, int status) { - *pstatus = status; + *pstatus = g_running ? status : -1; +} + +// Programs that do not end before the simulation stops: they pass if they +// did not fail until then. (They did not end before this was checked.) +static bool +MayNotEnd (std::string name, std::string stack) +{ + return name == "test-timer-fd" || name == "test-local-socket" + || name == "test-bug-multi-select" + || (name == "test-socket" && stack == "linux"); } void DceManagerTestCase::DoRun (void) @@ -135,7 +149,9 @@ DceManagerTestCase::DoRun (void) { Simulator::Stop (m_maxDuration); } + g_running = true; Simulator::Run (); + g_running = MayNotEnd (m_filename, m_netstack); Simulator::Destroy (); NS_TEST_ASSERT_MSG_EQ (status, 0, "Process did not return successfully: " << g_testError); From 3af2f35402981a6665b4a2dcc9452852d65d0419 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 11:58:08 +0200 Subject: [PATCH 11/71] Wake sigwait for pthread_kill, and consume one pending signal pthread_kill only woke a thread that did not block the signal, while sigwait callers block the signals they wait for: the thread slept forever. sigwait also consumed both the thread's and the process' pending instance of a signal, losing one. Co-Authored-By: Claude Opus 5.5 --- model/dce-pthread.cc | 5 +-- model/dce-signal.cc | 11 ++++-- test/test-signal.cc | 79 ++++++++++++++++++++++++++++++++++++++++---- wscript | 2 +- 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/model/dce-pthread.cc b/model/dce-pthread.cc index fa08dd31..e0be82cc 100644 --- a/model/dce-pthread.cc +++ b/model/dce-pthread.cc @@ -392,9 +392,10 @@ int dce_pthread_kill (pthread_t th, int sig) } sigaddset (&thread->pendingSignals, sig); - if (sigismember (&thread->signalMask, sig) == 0) + if (sigismember (&thread->signalMask, sig) == 0 + || sigismember (&thread->sigwaitSet, sig) == 1) { - // signal not blocked by thread. + // signal not blocked by thread, or the thread waits for it in sigwait. if (thread->task->IsBlocked ()) { thread->process->manager->Wakeup (thread); diff --git a/model/dce-signal.cc b/model/dce-signal.cc index b7bfe277..fe1939db 100644 --- a/model/dce-signal.cc +++ b/model/dce-signal.cc @@ -117,8 +117,15 @@ int dce_sigwait (const sigset_t *set, int *sig) if (sigismember (¤t->pendingSignals, s) == 1 || sigismember (¤t->process->pendingSignals, s) == 1) { - sigdelset (¤t->pendingSignals, s); - sigdelset (¤t->process->pendingSignals, s); + // Consume one occurrence: the thread-directed one first. + if (sigismember (¤t->pendingSignals, s) == 1) + { + sigdelset (¤t->pendingSignals, s); + } + else + { + sigdelset (¤t->process->pendingSignals, s); + } sigemptyset (¤t->sigwaitSet); *sig = s; return 0; diff --git a/test/test-signal.cc b/test/test-signal.cc index 4f04a3b2..9a8e4f62 100644 --- a/test/test-signal.cc +++ b/test/test-signal.cc @@ -4,23 +4,88 @@ #include #include #include +#include +#include + +static void +Block (int signum) +{ + sigset_t set; + sigemptyset (&set); + sigaddset (&set, signum); + TEST_ASSERT_EQUAL (sigprocmask (SIG_BLOCK, &set, 0), 0); +} + +static int +Sigwait (int signum) +{ + sigset_t set; + int sig = 0; + sigemptyset (&set); + sigaddset (&set, signum); + TEST_ASSERT_EQUAL (sigwait (&set, &sig), 0); + return sig; +} + +static void +SetTimer (int sec, int usec) +{ + struct itimerval it; + memset (&it, 0, sizeof (it)); + it.it_value.tv_sec = sec; + it.it_value.tv_usec = usec; + setitimer (ITIMER_REAL, &it, NULL); +} + +static volatile bool g_sent = false; + +static void * +Waiter (void *arg) +{ + Block (SIGUSR1); + Block (SIGALRM); + // Woken by a signal sent to this thread. + TEST_ASSERT_EQUAL (Sigwait (SIGUSR1), SIGUSR1); + // One occurrence sent to this thread and one sent to the process: two + // signals. + while (!g_sent) + { + usleep (100000); + } + TEST_ASSERT_EQUAL (Sigwait (SIGALRM), SIGALRM); + TEST_ASSERT_EQUAL (Sigwait (SIGALRM), SIGALRM); + return 0; +} + +static void +TestThreadSignals (void) +{ + Block (SIGUSR1); + Block (SIGALRM); + pthread_t thread; + TEST_ASSERT_EQUAL (pthread_create (&thread, 0, &Waiter, 0), 0); + sleep (1); + TEST_ASSERT_EQUAL (pthread_kill (thread, SIGUSR1), 0); + SetTimer (0, 500000); + sleep (1); + TEST_ASSERT_EQUAL (pthread_kill (thread, SIGALRM), 0); + g_sent = true; + TEST_ASSERT_EQUAL (pthread_join (thread, 0), 0); + printf ("thread signals rcvd\n"); +} int main (int argc, char *argv[]) { + TestThreadSignals (); + sigset_t sigcatch; int signum; sigemptyset (&sigcatch); sigaddset (&sigcatch, SIGALRM); { - struct itimerval it; - memset (&it, 0, sizeof (it)); - it.it_value.tv_sec = 1; - it.it_value.tv_usec = 500000; - - - setitimer (ITIMER_REAL, &it, NULL); + SetTimer (1, 500000); while (1) { sigwait (&sigcatch, &signum); diff --git a/wscript b/wscript index 945897b0..7f3f4d88 100755 --- a/wscript +++ b/wscript @@ -315,7 +315,7 @@ def build_dce_tests(module, bld): ['test-socket', []], ['test-bug-multi-select', []], ['test-tsearch', []], - ['test-signal', []], + ['test-signal', ['PTHREAD']], ['test-clock-gettime', []], ['test-gcc-builtin-apply', []], ['test-iostream', []], From f3fc2594ef751d24a2346e26328cede9e59903da Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 11:58:17 +0200 Subject: [PATCH 12/71] LKL: schedule kernel timers from the simulator's thread The timer host operation runs in the node's tasks, which run on threads of their own with PthreadFiberManager, the default: Simulator::Schedule must not be called there (ns-3 asserts it in debug builds). Use TaskManager::ScheduleMain, like the other events DCE tasks schedule. Co-Authored-By: Claude Opus 5.5 --- model/lkl/lkl-kernel.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/model/lkl/lkl-kernel.cc b/model/lkl/lkl-kernel.cc index 52417509..4a32c560 100644 --- a/model/lkl/lkl-kernel.cc +++ b/model/lkl/lkl-kernel.cc @@ -8,6 +8,7 @@ #include "ns3/simulator.h" #include "ns3/string.h" #include "ns3/event-impl.h" +#include "ns3/make-event.h" #include #include #include @@ -361,8 +362,10 @@ TimerSetOneshot (void *t, unsigned long delta) { LklTimer *timer = (LklTimer *)t; timer->event.Cancel (); - // Called from the node's own tasks, so the event keeps the node context. - timer->event = Simulator::Schedule (NanoSeconds (delta), &TimerExpire, timer); + // Called from the node's own tasks, which may run on threads of their own + // (PthreadFiberManager): schedule from the simulator's thread. The event + // keeps the node context. + timer->event = Manager ()->ScheduleMain (NanoSeconds (delta), MakeEvent (&TimerExpire, timer)); return 0; } From 2f0b9bd6ab9b1d6e519f936c9c3c96a6988bd96f Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 11:58:24 +0200 Subject: [PATCH 13/71] LKL: do not block in the kernel, so that DCE can stop processes DCE deletes the tasks of a process it stops (e.g. ApplicationContainer Stop) at once, wherever they are. A task blocked in an LKL system call leaves the kernel referring to it: the entries on its stack in the socket's wait queues, its LKL wait lists. The next wake-up of the socket crashed the simulation (dce-lkl-p2p --stop). Blocking socket calls (read, recvmsg, sendmsg, accept, connect) are now made non-blocking, and wait in DCE for the socket's events, honoring SO_RCVTIMEO and SO_SNDTIMEO and DCE signals; a blocking send still returns once all its data is queued. The LKL wait lists forget stopped tasks, as do the thread identities of the tasks the kernel did not create, which leaked. The files of a stopped process are released without being closed: release their kernel socket then. Co-Authored-By: Claude Opus 5.5 --- example/dce-lkl-p2p.cc | 36 ++++-- example/examples-to-run.py | 2 + model/kernel-socket-fd.cc | 230 +++++++++++++++++++++++++++++++++++-- model/kernel-socket-fd.h | 13 +++ model/lkl/lkl-kernel.cc | 25 +++- 5 files changed, 287 insertions(+), 19 deletions(-) diff --git a/example/dce-lkl-p2p.cc b/example/dce-lkl-p2p.cc index 88fcdf87..c4035352 100644 --- a/example/dce-lkl-p2p.cc +++ b/example/dce-lkl-p2p.cc @@ -7,6 +7,10 @@ * * With --loopback, a single node runs the TCP client and server over its * loopback device instead (like dce-linux-simple). + * + * With --stop, the servers are stopped while they wait for the clients, + * which then send to them: the kernel must keep working (ip runs on the + * server node afterwards). */ #include "ns3/core-module.h" #include "ns3/network-module.h" @@ -44,7 +48,7 @@ Finished (std::string name, uint32_t node, uint16_t pid, int status) } } -static void +static ApplicationContainer Run (Ptr node, Time at, std::string binary, std::string args, bool check = true) { DceApplicationHelper dce; @@ -62,15 +66,18 @@ Run (Ptr node, Time at, std::string binary, std::string args, bool check = } ApplicationContainer apps = dce.Install (node); apps.Start (at); + return apps; } int main (int argc, char *argv[]) { bool loopback = false; + bool stop = false; bool pcap = false; CommandLine cmd; cmd.AddValue ("loopback", "Use one node and its loopback device", loopback); + cmd.AddValue ("stop", "Stop the servers while they wait for the clients", stop); cmd.AddValue ("pcap", "Write pcap traces of the link", pcap); cmd.Parse (argc, argv); @@ -107,16 +114,27 @@ main (int argc, char *argv[]) Ptr server = nodes.Get (nodes.GetN () - 1); std::string serverAddr = loopback ? "127.0.0.1" : "10.0.0.2"; - Run (server, Seconds (1.0), "tcp-server", ""); - Run (nodes.Get (0), Seconds (1.5), "tcp-client", serverAddr); - if (!loopback) + if (stop) { - Run (server, Seconds (1.0), "udp-server", ""); - Run (nodes.Get (0), Seconds (1.5), "udp-client", serverAddr); + Run (server, Seconds (1.0), "tcp-server", "", false).Stop (Seconds (1.2)); + Run (server, Seconds (1.0), "udp-server", "", false).Stop (Seconds (1.2)); + Run (nodes.Get (0), Seconds (1.5), "tcp-client", serverAddr, false); + Run (nodes.Get (0), Seconds (1.5), "udp-client", serverAddr, false); + Run (server, Seconds (5.0), "ip", "addr show"); + Simulator::Stop (Seconds (10)); + } + else + { + Run (server, Seconds (1.0), "tcp-server", ""); + Run (nodes.Get (0), Seconds (1.5), "tcp-client", serverAddr); + if (!loopback) + { + Run (server, Seconds (1.0), "udp-server", ""); + Run (nodes.Get (0), Seconds (1.5), "udp-client", serverAddr); + } + // udp-client sends one datagram per second, 1000 times. + Simulator::Stop (Seconds (1100)); } - - // udp-client sends one datagram per second, 1000 times. - Simulator::Stop (Seconds (1100)); Simulator::Run (); Simulator::Destroy (); diff --git a/example/examples-to-run.py b/example/examples-to-run.py index 9228e99d..77856d53 100644 --- a/example/examples-to-run.py +++ b/example/examples-to-run.py @@ -92,6 +92,8 @@ ("dce-lkl-boot", "True", "True"), ("dce-lkl-p2p --loopback", "True", "True"), ("dce-lkl-p2p", "True", "True"), + ("dce-lkl-p2p --stop", "True", "True"), + ("dce-lkl-p2p --loopback --stop", "True", "True"), ] # A list of Python examples to run in order to ensure that they remain diff --git a/model/kernel-socket-fd.cc b/model/kernel-socket-fd.cc index 8f9dfee0..e997ca17 100644 --- a/model/kernel-socket-fd.cc +++ b/model/kernel-socket-fd.cc @@ -2,11 +2,13 @@ #include "kernel-socket-fd.h" #include "kernel-socket-fd-factory.h" #include "lkl-kernel.h" +#include "task-manager.h" #include "process.h" #include "utils.h" #include "file-usage.h" #include "wait-queue.h" #include "ns3/log.h" +#include "ns3/simulator.h" #include #include #include @@ -14,6 +16,9 @@ #include #include #include +#include +#include +#include NS_LOG_COMPONENT_DEFINE ("DceKernelSocketFd"); @@ -29,6 +34,18 @@ KernelSocketFd::KernelSocketFd (Ptr factory, int fd) KernelSocketFd::~KernelSocketFd () { + // Not closed: the files of a process DCE stops are released, not closed. + if (m_fd >= 0) + { + m_factory->Unregister (this); + // Not when the simulation is destroyed, outside of the node. + Ptr kernel = m_factory->GetKernel (); + if (kernel && kernel->GetTaskManager () + && kernel->GetTaskManager () == TaskManager::Current ()) + { + RawCall (__lkl__NR_close, m_fd); + } + } } int @@ -44,10 +61,15 @@ KernelSocketFd::NotifyEvents (short events) } long -KernelSocketFd::Call (long no, long a0, long a1, long a2, long a3, long a4, long a5) const +KernelSocketFd::RawCall (long no, long a0, long a1, long a2, long a3, long a4, long a5) const { long params[6] = { a0, a1, a2, a3, a4, a5 }; - long ret = m_factory->GetKernel ()->Syscall (no, params); + return m_factory->GetKernel ()->Syscall (no, params); +} + +long +KernelSocketFd::Result (long ret) const +{ if (ret < 0) { Current ()->err = -ret; @@ -56,36 +78,182 @@ KernelSocketFd::Call (long no, long a0, long a1, long a2, long a3, long a4, long return ret; } +long +KernelSocketFd::Call (long no, long a0, long a1, long a2, long a3, long a4, long a5) const +{ + return Result (RawCall (no, a0, a1, a2, a3, a4, a5)); +} + +Time +KernelSocketFd::GetTimeout (int option) const +{ + struct timeval tv; + socklen_t len = sizeof (tv); + if (RawCall (__lkl__NR_getsockopt, m_fd, SOL_SOCKET, option, (long)&tv, (long)&len) < 0) + { + return Seconds (0); + } + return Seconds (tv.tv_sec) + MicroSeconds (tv.tv_usec); +} + +int +KernelSocketFd::WaitEvents (short events, Time timeout) +{ + // Watched first: the probe below then sees the events that came before, + // and the watcher wakes this task for the later ones. + m_factory->Watch (this); + struct pollfd pfd; + pfd.fd = m_fd; + pfd.events = events; + pfd.revents = 0; + struct + { + long long tv_sec; + long long tv_nsec; + } zero = { 0, 0 }; + // Ready although the call would block (e.g. DCCP reports a socket with a + // full queue writable): wait for the next event, or retry in 1 ms. + bool ready = RawCall (__lkl__NR_ppoll, (long)&pfd, 1, (long)&zero, 0, 8) > 0; + bool retry = ready && (timeout.IsZero () || timeout > MilliSeconds (1)); + WaitQueueEntryTimeout *wq = new WaitQueueEntryTimeout (events | POLLERR | POLLHUP, + retry ? MilliSeconds (1) : timeout); + AddWaitQueue (wq, true); + PollTable::Result res = wq->Wait (); + RemoveWaitQueue (wq, true); + delete wq; + return retry && res == PollTable::TIMEOUT ? PollTable::OK : res; +} + +// DCE deletes the tasks of the processes it stops, at any time, while the +// kernel keeps references to a task sleeping in it (e.g. the wait queue +// entries on its stack): so tasks never block in the kernel, but in DCE. +long +KernelSocketFd::BlockingCall (short events, int timeoutOption, + long no, long a0, long a1, long a2, long a3, long a4, long a5) +{ + long flags = RawCall (__lkl__NR_fcntl, m_fd, F_GETFL); + if (flags < 0 || (flags & O_NONBLOCK)) + { + return RawCall (no, a0, a1, a2, a3, a4, a5); + } + Time end = Simulator::Now () + GetTimeout (timeoutOption); + bool timeout = end != Simulator::Now (); + while (true) + { + RawCall (__lkl__NR_fcntl, m_fd, F_SETFL, flags | O_NONBLOCK); + long ret = RawCall (no, a0, a1, a2, a3, a4, a5); + RawCall (__lkl__NR_fcntl, m_fd, F_SETFL, flags); + if (ret != -EAGAIN) + { + return ret; + } + if (timeout && Simulator::Now () >= end) + { + return -EAGAIN; + } + switch (WaitEvents (events, timeout ? end - Simulator::Now () : Seconds (0))) + { + case PollTable::INTERRUPTED: + return -EINTR; + case PollTable::TIMEOUT: + return -EAGAIN; + default: + break; + } + } +} + int KernelSocketFd::Close (void) { NS_LOG_FUNCTION (this); m_factory->Unregister (this); - return Call (__lkl__NR_close, m_fd); + int fd = m_fd; + m_fd = -1; + return Call (__lkl__NR_close, fd); } ssize_t KernelSocketFd::Write (const void *buf, size_t count) { - return Call (__lkl__NR_write, m_fd, (long)buf, count); + // A blocking write returns once all the data is queued. + size_t sent = 0; + while (true) + { + long ret = BlockingCall (POLLOUT, SO_SNDTIMEO, __lkl__NR_write, m_fd, + (long)((const uint8_t *)buf + sent), count - sent); + if (ret < 0) + { + return sent > 0 ? (ssize_t)sent : Result (ret); + } + sent += ret; + if (sent >= count || ret == 0) + { + return sent; + } + } } ssize_t KernelSocketFd::Read (void *buf, size_t count) { - return Call (__lkl__NR_read, m_fd, (long)buf, count); + return Result (BlockingCall (POLLIN, SO_RCVTIMEO, __lkl__NR_read, m_fd, (long)buf, count)); } ssize_t KernelSocketFd::Recvmsg (struct msghdr *msg, int flags) { - return Call (__lkl__NR_recvmsg, m_fd, (long)msg, flags); + if (flags & MSG_DONTWAIT) + { + return Call (__lkl__NR_recvmsg, m_fd, (long)msg, flags); + } + return Result (BlockingCall (POLLIN, SO_RCVTIMEO, __lkl__NR_recvmsg, m_fd, (long)msg, flags)); } ssize_t KernelSocketFd::Sendmsg (const struct msghdr *msg, int flags) { - return Call (__lkl__NR_sendmsg, m_fd, (long)msg, flags); + if (flags & MSG_DONTWAIT) + { + return Call (__lkl__NR_sendmsg, m_fd, (long)msg, flags); + } + // A blocking send returns once all the data is queued: send the rest of + // a partial send. + std::vector iov (msg->msg_iov, msg->msg_iov + msg->msg_iovlen); + struct msghdr rest = *msg; + rest.msg_iov = iov.empty () ? 0 : &iov[0]; + size_t total = 0; + for (size_t i = 0; i < iov.size (); i++) + { + total += iov[i].iov_len; + } + size_t sent = 0; + while (true) + { + long ret = BlockingCall (POLLOUT, SO_SNDTIMEO, __lkl__NR_sendmsg, m_fd, (long)&rest, flags); + if (ret < 0) + { + return sent > 0 ? (ssize_t)sent : Result (ret); + } + sent += ret; + if (sent >= total || ret == 0) + { + return sent; + } + // Skip what was sent. + size_t skip = ret; + while (rest.msg_iovlen > 0 && skip >= rest.msg_iov->iov_len) + { + skip -= rest.msg_iov->iov_len; + rest.msg_iov++; + rest.msg_iovlen--; + } + if (rest.msg_iovlen > 0) + { + rest.msg_iov->iov_base = (uint8_t *)rest.msg_iov->iov_base + skip; + rest.msg_iov->iov_len -= skip; + } + } } bool @@ -133,7 +301,50 @@ KernelSocketFd::Bind (const struct sockaddr *my_addr, socklen_t addrlen) int KernelSocketFd::Connect (const struct sockaddr *my_addr, socklen_t addrlen) { - return Call (__lkl__NR_connect, m_fd, (long)my_addr, addrlen); + long flags = RawCall (__lkl__NR_fcntl, m_fd, F_GETFL); + if (flags < 0 || (flags & O_NONBLOCK)) + { + return Call (__lkl__NR_connect, m_fd, (long)my_addr, addrlen); + } + // Without blocking in the kernel (see BlockingCall): connect, and wait in + // DCE until connecting again tells the outcome (and, on failure, makes the + // socket unconnected again, like a blocking connect). + Time end = Simulator::Now () + GetTimeout (SO_SNDTIMEO); + bool timeout = end != Simulator::Now (); + bool started = false; + while (true) + { + RawCall (__lkl__NR_fcntl, m_fd, F_SETFL, flags | O_NONBLOCK); + long ret = RawCall (__lkl__NR_connect, m_fd, (long)my_addr, addrlen); + RawCall (__lkl__NR_fcntl, m_fd, F_SETFL, flags); + if (started && (ret == -EISCONN || ret == -EEXIST)) + { + // Connected, for protocols that do not report it by connecting + // again (SCTP). + int error = 0; + socklen_t len = sizeof (error); + ret = RawCall (__lkl__NR_getsockopt, m_fd, SOL_SOCKET, SO_ERROR, (long)&error, (long)&len); + return Result (ret < 0 ? ret : -error); + } + if (ret != -EINPROGRESS && ret != -EALREADY) + { + return Result (ret); + } + started = true; + if (timeout && Simulator::Now () >= end) + { + return Result (-EINPROGRESS); + } + switch (WaitEvents (POLLOUT, timeout ? end - Simulator::Now () : Seconds (0))) + { + case PollTable::INTERRUPTED: + return Result (-EINTR); + case PollTable::TIMEOUT: + return Result (-EINPROGRESS); + default: + break; + } + } } int @@ -152,7 +363,8 @@ int KernelSocketFd::Accept (struct sockaddr *my_addr, socklen_t *addrlen) { NS_LOG_FUNCTION (this); - long kernelFd = Call (__lkl__NR_accept4, m_fd, (long)my_addr, (long)addrlen, 0); + long kernelFd = Result (BlockingCall (POLLIN, SO_RCVTIMEO, __lkl__NR_accept4, + m_fd, (long)my_addr, (long)addrlen, 0)); if (kernelFd < 0) { return -1; diff --git a/model/kernel-socket-fd.h b/model/kernel-socket-fd.h index cd685d6e..7cf1de0c 100644 --- a/model/kernel-socket-fd.h +++ b/model/kernel-socket-fd.h @@ -4,6 +4,7 @@ #include "unix-fd.h" #include "ns3/ptr.h" +#include "ns3/nstime.h" namespace ns3 { @@ -59,6 +60,18 @@ class KernelSocketFd : public UnixFd // Kernel system call; on error, sets the current thread's errno. long Call (long no, long a0 = 0, long a1 = 0, long a2 = 0, long a3 = 0, long a4 = 0, long a5 = 0) const; + // Kernel system call; returns -errno on error. + long RawCall (long no, long a0 = 0, long a1 = 0, long a2 = 0, + long a3 = 0, long a4 = 0, long a5 = 0) const; + // A system call that may block, made without blocking in the kernel; + // waits in DCE for the events instead. Returns -errno on error. + long BlockingCall (short events, int timeoutOption, long no, long a0 = 0, long a1 = 0, + long a2 = 0, long a3 = 0, long a4 = 0, long a5 = 0); + // Waits in DCE for one of the events or the timeout. + int WaitEvents (short events, Time timeout); + Time GetTimeout (int option) const; + // Sets errno from a RawCall result. + long Result (long ret) const; Ptr m_factory; int m_fd; diff --git a/model/lkl/lkl-kernel.cc b/model/lkl/lkl-kernel.cc index 4a32c560..74936919 100644 --- a/model/lkl/lkl-kernel.cc +++ b/model/lkl/lkl-kernel.cc @@ -166,13 +166,20 @@ ThreadEqual (lkl_thread_t a, lkl_thread_t b) /* Semaphores and mutexes -------------------------------------------------- */ +struct LklWaitList; +// The wait list each blocked task is on, to forget the tasks DCE stops. +std::map g_waiting; + struct LklWaitList { std::list waiters; void Wait (void) { - waiters.push_back (Manager ()->RunningTask ()); + Task *task = Manager ()->RunningTask (); + waiters.push_back (task); + g_waiting[task] = this; Manager ()->Sleep (); + g_waiting.erase (task); } void WakeOne (void) { @@ -180,6 +187,7 @@ struct LklWaitList { Task *task = waiters.front (); waiters.pop_front (); + g_waiting.erase (task); Manager ()->Wakeup (task); } } @@ -689,6 +697,21 @@ void LklKernel::TaskEnd (Task *task, bool running, void *context) { LklKernel *kernel = (LklKernel *)context; + // A stopped task may be waiting: it must not be woken once deleted. + std::map::iterator w = g_waiting.find (task); + if (w != g_waiting.end ()) + { + w->second->waiters.remove (task); + g_waiting.erase (w); + } + // The identity given to a task the kernel did not create (kernel threads + // drop theirs when they finish). + std::map::iterator t = g_threads.find (task); + if (t != g_threads.end () && t->second->fn == 0) + { + delete t->second; + g_threads.erase (t); + } TlsDestructors *destructors = new TlsDestructors (); for (std::set::iterator k = kernel->m_tlsKeys.begin (); k != kernel->m_tlsKeys.end (); ++k) { From 8a125a173ce0285502fd784fe2db444f3d0e2228 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:00:17 +0200 Subject: [PATCH 14/71] LKL: receive from LTE devices, and from devices without Mac48 addresses LTE devices do not support promiscuous mode, so the promiscuous protocol handler never got their packets: an LTE UE received nothing (the libos stack registered a non-promiscuous handler for them). The destination address of their frames was also converted from the device's Mac64Address; use the address of the kernel device instead. Co-Authored-By: Claude Opus 5.5 --- model/kernel-socket-fd-factory.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/model/kernel-socket-fd-factory.cc b/model/kernel-socket-fd-factory.cc index 5ed82811..06f3ec34 100644 --- a/model/kernel-socket-fd-factory.cc +++ b/model/kernel-socket-fd-factory.cc @@ -71,6 +71,7 @@ struct LklNetDevice struct lkl_netdev nd; // must stay first: LKL passes &nd to the operations KernelSocketFdFactory *factory; Ptr device; + Mac48Address mac; // the kernel device's address std::deque > rx; // received Ethernet frames Task *poller; bool hup; @@ -469,6 +470,7 @@ KernelSocketFdFactory::AddDevice (Ptr device) Mac48Address mac48 = Mac48Address::IsMatchingType (address) ? Mac48Address::ConvertFrom (address) : Mac48Address::Allocate (); mac48.CopyTo (mac); + dev->mac = mac48; struct lkl_netdev_args args; memset (&args, 0, sizeof (args)); args.mac = mac; @@ -518,8 +520,13 @@ KernelSocketFdFactory::AddDevice (Ptr device) } Call (__lkl__NR_close, s); + // LTE devices do not support promiscuous mode: they deliver received + // packets only to the non-promiscuous handlers. + TypeId lte; + bool promiscuous = !(TypeId::LookupByNameFailSafe ("ns3::LteNetDevice", <e) + && device->GetInstanceTypeId ().IsChildOf (lte)); GetObject ()->RegisterProtocolHandler (MakeCallback (&KernelSocketFdFactory::RxFromDevice, this), - 0, device, true); + 0, device, promiscuous); } void @@ -541,8 +548,10 @@ KernelSocketFdFactory::RxFromDevice (Ptr device, Ptr p, } // Rebuild the Ethernet frame the kernel expects. std::vector frame (ETH_HEADER + p->GetSize ()); + // Devices with other address types (e.g. LTE's Mac64Address) have a kernel + // device address of their own. Mac48Address dest = Mac48Address::IsMatchingType (to) - ? Mac48Address::ConvertFrom (to) : Mac48Address::ConvertFrom (device->GetAddress ()); + ? Mac48Address::ConvertFrom (to) : dev->mac; Mac48Address source = Mac48Address::IsMatchingType (from) ? Mac48Address::ConvertFrom (from) : Mac48Address ("00:00:00:00:00:00"); dest.CopyTo (&frame[0]); From 01a03675829018824da8234a3ca8a654c26b8de7 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:00:17 +0200 Subject: [PATCH 15/71] LKL: enable IPv4 forwarding by default with LinuxSocketFdFactory The libos stack's LinuxSocketFdFactory enabled it, and scripts written for it rely on it: routers that set no sysctl did not forward (e.g. dce-mptcp-lte-wifi-v6). Co-Authored-By: Claude Opus 5.5 --- model/linux-socket-fd-factory.cc | 7 +++++++ model/linux-socket-fd-factory.h | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/model/linux-socket-fd-factory.cc b/model/linux-socket-fd-factory.cc index db53c9b2..94453146 100644 --- a/model/linux-socket-fd-factory.cc +++ b/model/linux-socket-fd-factory.cc @@ -15,4 +15,11 @@ LinuxSocketFdFactory::GetTypeId (void) return tid; } +LinuxSocketFdFactory::LinuxSocketFdFactory () +{ + // The defaults of the libos stack, which scripts written for it rely on: + // applied when the kernel boots, before the scripts' own sysctls. + Set (".net.ipv4.conf.all.forwarding", "1"); +} + } // namespace ns3 diff --git a/model/linux-socket-fd-factory.h b/model/linux-socket-fd-factory.h index 917aaace..8361d427 100644 --- a/model/linux-socket-fd-factory.h +++ b/model/linux-socket-fd-factory.h @@ -7,12 +7,14 @@ namespace ns3 { /** * The Linux network stack: the Linux kernel as a library (LKL); see - * KernelSocketFdFactory. + * KernelSocketFdFactory. Like the libos stack it replaces, it enables IPv4 + * forwarding by default. */ class LinuxSocketFdFactory : public KernelSocketFdFactory { public: static TypeId GetTypeId (void); + LinuxSocketFdFactory (); }; } // namespace ns3 From 4bd06d312514e4b3fbed789ac1dac36ebdfcf6eb Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:00:17 +0200 Subject: [PATCH 16/71] Fix the MPTCP LTE+WiFi examples, and check that every path carries data dce-mptcp-lte-wifi: the WiFi router was out of range (ns-3 no longer detects frames below -82 dBm), its route to node 1 pointed at its own address, and node 1 had no route back to the WiFi network for the subflow it gets on its LTE-side address. dce-mptcp-lte-wifi-v6: the mobile node's addresses were hardcoded from an older MAC address allocation; compute them from the devices. The connection starts over IPv4, so the IPv6 subflow needs an IPv6 socket (the IPv4-mapped server address) and the server's IPv6 address, which it now announces; the mobile node's address is configured before the announcement arrives. Both examples now fail unless the server receives TCP payload from every address of the client, and run with test.py. Co-Authored-By: Claude Opus 5.5 --- example/dce-mptcp-lte-wifi-v6.cc | 100 +++++++++++++++++++++++++++---- example/dce-mptcp-lte-wifi.cc | 84 +++++++++++++++++++++++++- example/examples-to-run.py | 5 +- 3 files changed, 173 insertions(+), 16 deletions(-) diff --git a/example/dce-mptcp-lte-wifi-v6.cc b/example/dce-mptcp-lte-wifi-v6.cc index fd81211c..6712d33d 100644 --- a/example/dce-mptcp-lte-wifi-v6.cc +++ b/example/dce-mptcp-lte-wifi-v6.cc @@ -34,6 +34,7 @@ #include "ns3/internet-module.h" #include "ns3/dce-module.h" #include "ns3/point-to-point-helper.h" +#include "ns3/ppp-header.h" #include "ns3/wifi-module.h" #include "ns3/lte-module.h" #include "ns3/yans-wifi-helper.h" @@ -44,6 +45,59 @@ using namespace ns3; NS_LOG_COMPONENT_DEFINE ("DceMptcpLteWifiV6"); +// TCP payload bytes the server received from each address of the client. +static std::map g_received; + +static void +ServerRx (Ptr p) +{ + Ptr packet = p->Copy (); + PppHeader ppp; + packet->RemoveHeader (ppp); + std::ostringstream source; + if (ppp.GetProtocol () == 0x0021) // IPv4 + { + Ipv4Header ip; + packet->RemoveHeader (ip); + if (ip.GetProtocol () != 6) + { + return; + } + source << ip.GetSource (); + } + else if (ppp.GetProtocol () == 0x0057) // IPv6 + { + Ipv6Header ip; + packet->RemoveHeader (ip); + if (ip.GetNextHeader () != 6) + { + return; + } + source << ip.GetSource (); + } + else + { + return; + } + TcpHeader tcp; + packet->RemoveHeader (tcp); + g_received[source.str ()] += packet->GetSize (); +} + +// With MPTCP, every path carries data. +static int +CheckPaths (std::vector addresses) +{ + bool ok = true; + for (uint32_t i = 0; i < addresses.size (); i++) + { + uint64_t bytes = g_received[addresses[i]]; + std::cout << "received from " << addresses[i] << ": " << bytes << " bytes" << std::endl; + ok = ok && bytes > 0; + } + return ok ? 0 : 1; +} + static void AddAddress (Ptr node, Time at, const char *name, const char *address) { @@ -150,11 +204,18 @@ int main (int argc, char *argv[]) mac.SetType ("ns3::StaWifiMac", "Ssid", SsidValue (ssid1), "ActiveProbing", BooleanValue (false)); - wifi.Install (phy, mac, mn); + NetDeviceContainer mnWifi = wifi.Install (phy, mac, mn); // setup ap. mac.SetType ("ns3::ApWifiMac", "Ssid", SsidValue (ssid1)); - wifi.Install (phy, mac, ar.Get (0)); + NetDeviceContainer arWifi = wifi.Install (phy, mac, ar.Get (0)); + + // The addresses MN gets from AR1's router advertisements (EUI-64), and + // AR1's link-local address: MN's default router. + std::ostringstream mnAddr, arLinkLocal; + mnAddr << Ipv6Address::MakeAutoconfiguredAddress (mnWifi.Get (0)->GetAddress (), + Ipv6Address ("2001:1:2:4::")); + arLinkLocal << Ipv6Address::MakeAutoconfiguredLinkLocalAddress (arWifi.Get (0)->GetAddress ()); DceManagerHelper dceMng; @@ -223,6 +284,8 @@ int main (int argc, char *argv[]) // Assign ip addresses Ipv4InterfaceContainer if1; if1 = epcHelper->AssignUeIpv4Address (NetDeviceContainer (ueLteDevs)); + std::ostringstream lteAddr; + lteAddr << if1.GetAddress (0, 0); lteHelper->Attach (ueLteDevs.Get(0), enbLteDevs.Get(0)); // setup ip routes @@ -263,9 +326,12 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.11), "link set lo up"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.11), "link set sim0 up"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.11), "link set sim1 up"); - LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "-6 rule add from 2001:1:2:4:200:ff:fe00:a table 2"); - LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "route add default via fe80::200:ff:fe00:b dev sim1 table 2"); - LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "route add default via fe80::200:ff:fe00:b dev sim1"); + // The address is also configured statically, so that MN has it when the + // server announces its IPv6 address, before AR1's first advertisement. + AddAddress (mn.Get (0), Seconds (0.20), "sim1", (mnAddr.str () + "/64").c_str ()); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "-6 rule add from " + mnAddr.str () + " table 2"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "route add default via " + arLinkLocal.str () + " dev sim1 table 2"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "route add default via " + arLinkLocal.str () + " dev sim1"); // disable default injection from ra stack.SysctlSet (mn, ".net.ipv6.conf.sim1.accept_ra_defrtr", "0"); @@ -280,14 +346,15 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (mn.Get (0), Seconds (49.0), "route show table all"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (49.0), "-6 rule show"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.0), "route show table all"); - LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.1), "route get 2001:1:2:3::1 from 2001:1:2:4:200:ff:fe00:a"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (10.1), "route get 2001:1:2:3::1 from " + mnAddr.str ()); - // MPTCP: a subflow from the address the mobile node gets on its second - // interface (Linux MPTCP does not open subflows from new addresses by - // itself). + // MPTCP: an IPv6 subflow over Wi-Fi (Linux MPTCP does not open subflows + // from new addresses by itself). The connection starts over IPv4 (LTE): + // the server announces its IPv6 address, the peer of that subflow. LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); LinuxStackHelper::RunIp (sv.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); - LinuxStackHelper::RunIp (mn.Get (0), Seconds (9.0), "mptcp endpoint add 2001:1:2:4:200:ff:fe00:a dev sim1 subflow"); + LinuxStackHelper::RunIp (sv.Get (0), Seconds (0.5), "mptcp endpoint add 2001:1:2:3::1 dev sim0 signal"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (9.0), "mptcp endpoint add " + mnAddr.str () + " dev sim1 subflow"); { ApplicationContainer apps; @@ -313,9 +380,11 @@ int main (int argc, char *argv[]) dce.ResetEnvironment (); dce.AddArgument ("-V"); dce.AddArgument ("-c"); + // The IPv4-mapped IPv6 address: an IPv6 MPTCP socket, which can have + // IPv6 subflows besides the initial IPv4 one. std::string sv_addr = sv_sim0_v4; sv_addr.replace (sv_addr.find ("/"), 3, "\0 "); - dce.AddArgument (sv_addr); + dce.AddArgument ("::ffff:" + sv_addr); dce.AddArgument ("-i"); dce.AddArgument ("1"); dce.AddArgument ("--time"); @@ -332,9 +401,16 @@ int main (int argc, char *argv[]) phy.EnablePcapAll ("dce-mptcp-lte-wifi-v6"); p2p.EnablePcapAll ("dce-mptcp-lte-wifi-v6"); + Config::ConnectWithoutContext ("/NodeList/" + std::to_string (sv.Get (0)->GetId ()) + + "/DeviceList/*/$ns3::PointToPointNetDevice/PhyRxEnd", + MakeCallback (&ServerRx)); + Simulator::Stop (Seconds (50.0)); Simulator::Run (); Simulator::Destroy (); - return 0; + std::vector paths; + paths.push_back (lteAddr.str ()); + paths.push_back (mnAddr.str ()); + return CheckPaths (paths); } diff --git a/example/dce-mptcp-lte-wifi.cc b/example/dce-mptcp-lte-wifi.cc index 00175541..49f76416 100644 --- a/example/dce-mptcp-lte-wifi.cc +++ b/example/dce-mptcp-lte-wifi.cc @@ -37,10 +37,72 @@ #include "ns3/netanim-module.h" #include "ns3/constant-position-mobility-model.h" #include "ns3/config-store-module.h" +#include "ns3/ppp-header.h" using namespace ns3; NS_LOG_COMPONENT_DEFINE ("DceMptcpLteWifi"); +// TCP payload bytes the server received from each address of the client. +static std::map g_received; + +static void +ServerRx (Ptr p) +{ + Ptr packet = p->Copy (); + PppHeader ppp; + packet->RemoveHeader (ppp); + std::ostringstream source; + if (ppp.GetProtocol () == 0x0021) // IPv4 + { + Ipv4Header ip; + packet->RemoveHeader (ip); + if (ip.GetProtocol () != 6) + { + return; + } + source << ip.GetSource (); + } + else if (ppp.GetProtocol () == 0x0057) // IPv6 + { + Ipv6Header ip; + packet->RemoveHeader (ip); + if (ip.GetNextHeader () != 6) + { + return; + } + source << ip.GetSource (); + } + else + { + return; + } + TcpHeader tcp; + packet->RemoveHeader (tcp); + g_received[source.str ()] += packet->GetSize (); +} + +static std::string +AddressString (Ipv4Address address) +{ + std::ostringstream oss; + oss << address; + return oss.str (); +} + +// With MPTCP, every path carries data. +static int +CheckPaths (std::vector addresses) +{ + bool ok = true; + for (uint32_t i = 0; i < addresses.size (); i++) + { + uint64_t bytes = g_received[addresses[i]]; + std::cout << "received from " << addresses[i] << ": " << bytes << " bytes" << std::endl; + ok = ok && bytes > 0; + } + return ok ? 0 : 1; +} + void setPos (Ptr n, int x, int y, int z) { Ptr loc = CreateObject (); @@ -106,6 +168,7 @@ int main (int argc, char *argv[]) address1.SetBase ("10.1.0.0", "255.255.255.0"); address2.SetBase ("10.2.0.0", "255.255.255.0"); Ipv4InterfaceContainer if1, if2; + std::vector paths; // the client's addresses pointToPoint.SetDeviceAttribute ("DataRate", StringValue ("2Mbps")); pointToPoint.SetChannelAttribute ("Delay", StringValue (p2pdelay)); Ptr em1 = @@ -132,6 +195,7 @@ int main (int argc, char *argv[]) // Assign ip addresses if1 = epcHelper->AssignUeIpv4Address (NetDeviceContainer (ueLteDevs)); + paths.push_back (AddressString (if1.GetAddress (0, 0))); lteHelper->Attach (ueLteDevs.Get(0), enbLteDevs.Get(0)); // setup ip routes @@ -171,6 +235,7 @@ int main (int argc, char *argv[]) devices1 = wifi.Install (phy, mac, NodeContainer (nodes.Get (0), routers.Get (0))); // Assign ip addresses if1 = address1.Assign (devices1); + paths.push_back (AddressString (if1.GetAddress (0, 0))); address1.NewNetwork (); // setup ip routes cmd_oss.str (""); @@ -229,10 +294,19 @@ int main (int argc, char *argv[]) cmd_oss.str (""); cmd_oss << "route add 10.1.0.0/16 via " << if2.GetAddress (1, 0) << " dev sim" << 1 << " table " << (2); LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.1), cmd_oss.str ().c_str ()); + // The WiFi subflow reaches node 1's LTE-side address (10.2.0.1), + // and node 1 replies to it from that address: route the WiFi + // network back through the WiFi router. cmd_oss.str (""); - cmd_oss << "route add 10.2.0.0/16 via " << if2.GetAddress (1, 0) << " dev sim1"; + cmd_oss << "route add 10.1.0.0/16 via " << if2.GetAddress (1, 0) << " dev sim" + << devices2.Get (0)->GetIfIndex (); + LinuxStackHelper::RunIp (nodes.Get (1), Seconds (0.1), cmd_oss.str ().c_str ()); + cmd_oss.str (""); + cmd_oss << "route add 10.2.0.0/16 via " << if2.GetAddress (0, 0) << " dev sim1"; LinuxStackHelper::RunIp (routers.Get (0), Seconds (0.2), cmd_oss.str ().c_str ()); - setPos (routers.Get (0), 70, 30, 0); + // Within WiFi range of node 0: ns-3 no longer detects frames below + // -82 dBm, which the farther position (91 m) gave. + setPos (routers.Get (0), 0, 30, 0); } // default route @@ -345,9 +419,13 @@ int main (int argc, char *argv[]) outputConfig2.ConfigureDefaults (); outputConfig2.ConfigureAttributes (); + Config::ConnectWithoutContext ("/NodeList/" + std::to_string (nodes.Get (1)->GetId ()) + + "/DeviceList/*/$ns3::PointToPointNetDevice/PhyRxEnd", + MakeCallback (&ServerRx)); + Simulator::Stop (Seconds (stopTime)); Simulator::Run (); Simulator::Destroy (); - return 0; + return CheckPaths (paths); } diff --git a/example/examples-to-run.py b/example/examples-to-run.py index 77856d53..3efd48cc 100644 --- a/example/examples-to-run.py +++ b/example/examples-to-run.py @@ -84,7 +84,10 @@ ("dce-mptcp-handoff-v4v6 --v6Primary=1 --errRate=0.4", "True", "True"), ("dce-mptcp-handoff-v4v6 --v6Primary=1 --errRate=0.5", "True", "True"), ("dce-mptcp-handoff-v4v6 --v6Primary=1 --errRate=0.8", "True", "True"), -# ("dce-mptcp-lte-wifi", "True", "True"), + ("dce-mptcp-lte-wifi", "True", "True"), + ("dce-mptcp-lte-wifi --disLte=1", "True", "True"), + ("dce-mptcp-lte-wifi --disWifi=1", "True", "True"), + ("dce-mptcp-lte-wifi-v6", "True", "True"), ("dce-httpd", "True", "True"), # ("dce-wifi-ccnx", "True", "True"), ("dce-sctp-simple", "True", "True"), From 272cb23a8cc7fd158557cd24c5e1b987bf3d0cea Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:13:58 +0200 Subject: [PATCH 17/71] Use NetDevice index + 1 for interface indexes with the ns-3 stack if_nametoindex returned the IPv4 interface index, while netlink messages and socket options now use NetDevice index + 1, like Linux with its loopback device first: a route added with "dev" could go to another device. Link messages also reported the previous device as the parent link (IFLA_LINK) of each device. Co-Authored-By: Claude Opus 5.5 --- model/dce.cc | 8 +++----- netlink/netlink-socket.cc | 3 ++- test/dce-manager-test.cc | 5 +++++ test/netlink-socket-test.cc | 15 +++++++++++++++ test/test-ifindex.cc | 12 ++++++++++++ wscript | 1 + 6 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 test/test-ifindex.cc diff --git a/model/dce.cc b/model/dce.cc index 4d304ac0..bbc5a978 100644 --- a/model/dce.cc +++ b/model/dce.cc @@ -717,17 +717,15 @@ unsigned dce_if_nametoindex (const char *ifname) } else { - int index = 0; + // Interface indexes are NetDevice indexes + 1, as in netlink + // messages and socket options. Ptr node = Current ()->process->manager->GetObject (); - Ptr ipv4 = node->GetObject (); - for (uint32_t i = 0; i < node->GetNDevices (); ++i) { Ptr dev = node->GetDevice (i); if (ifname == Names::FindName (dev)) { - index = ipv4->GetInterfaceForDevice (dev); - return index; + return dev->GetIfIndex () + 1; } } return 0; diff --git a/netlink/netlink-socket.cc b/netlink/netlink-socket.cc index ec216a0f..20714144 100644 --- a/netlink/netlink-socket.cc +++ b/netlink/netlink-socket.cc @@ -953,7 +953,8 @@ NetlinkSocket::BuildInterfaceInfoDumpMessage (uint32_t interface_num, uint32_t s ifinfomsg.AppendAttribute (NetlinkAttribute (InterfaceInfoMessage::IFL_A_ADDRESS, ADDRESS, mac)); ifinfomsg.AppendAttribute (NetlinkAttribute (InterfaceInfoMessage::IFL_A_BROADCAST, ADDRESS, bcast)); ifinfomsg.AppendAttribute (NetlinkAttribute (InterfaceInfoMessage::IFL_A_MTU, U32, mtu)); - ifinfomsg.AppendAttribute (NetlinkAttribute (InterfaceInfoMessage::IFL_A_LINK, U32, interface_num)); + // The device itself (no parent device), like Linux for unstacked devices. + ifinfomsg.AppendAttribute (NetlinkAttribute (InterfaceInfoMessage::IFL_A_LINK, U32, interface_num + 1)); //ifinfomsg.AppendAttribute (NetlinkAttribute (InterfaceInfoMessage::IFL_A_QDISC, STRING, "")); //ifinfomsg.AppendAttribute (NetlinkAttribute (InterfaceInfoMessage::IFL_A_MASTER, U32, 0)); //ifinfomsg.AppendAttribute (NetlinkAttribute (InterfaceInfoMessage::IFL_A_STATS, UNSPEC, 0)); diff --git a/test/dce-manager-test.cc b/test/dce-manager-test.cc index 638466b2..09fd5df5 100644 --- a/test/dce-manager-test.cc +++ b/test/dce-manager-test.cc @@ -4,6 +4,7 @@ #include "ns3/uinteger.h" #include "ns3/boolean.h" #include "ns3/string.h" +#include "ns3/names.h" #include "ns3/internet-stack-helper.h" #include "ns3/dce-module.h" #include "ns3/ipv4-dce-routing-helper.h" @@ -126,6 +127,8 @@ DceManagerTestCase::DoRun (void) Ipv4DceRoutingHelper ipv4RoutingHelper; stack.SetRoutingHelper (ipv4RoutingHelper); stack.Install (nodes); + // Named like the kernel's loopback device. + Names::Add ("lo", nodes.Get (0)->GetDevice (0)); } } else @@ -153,6 +156,7 @@ DceManagerTestCase::DoRun (void) Simulator::Run (); g_running = MayNotEnd (m_filename, m_netstack); Simulator::Destroy (); + Names::Clear (); NS_TEST_ASSERT_MSG_EQ (status, 0, "Process did not return successfully: " << g_testError); } @@ -221,6 +225,7 @@ DceManagerTestSuite::DceManagerTestSuite () { "test-clock-gettime", 0, "", false, false, NS3_MASK|LINUX_MASK}, { "test-gcc-builtin-apply", 0, "", false, false, NS3_MASK|LINUX_MASK}, { "test-signal", 30, "", false, false, NS3_MASK|LINUX_MASK}, + { "test-ifindex", 30, "", true, false, NS3_MASK|LINUX_MASK}, }; // Prepare directories and files for test-stdio diff --git a/test/netlink-socket-test.cc b/test/netlink-socket-test.cc index 8aae86ba..d25c566d 100644 --- a/test/netlink-socket-test.cc +++ b/test/netlink-socket-test.cc @@ -369,6 +369,21 @@ NetlinkSocketTestCase::TestInferfaceInfoMessage () NS_TEST_ASSERT_MSG_EQ ((multinlmsg.GetNMessages () > 1) && (multinlmsg.GetMessage (0).GetMsgType () == NETLINK_RTM_NEWLINK), true, "msg might be incorrect"); + + // Linux interface indexes start at 1; devices without a parent device + // are their own link. + for (uint32_t i = 0; i < multinlmsg.GetNMessages (); i++) + { + if (multinlmsg.GetMessage (i).GetMsgType () != NETLINK_RTM_NEWLINK) + { + continue; + } + InterfaceInfoMessage ifi = multinlmsg.GetMessage (i).GetInterfaceInfoMessage (); + NS_TEST_ASSERT_MSG_EQ (ifi.GetInterfaceIndex (), (int32_t)(i + 1), "interface index"); + NetlinkAttribute link; + NS_TEST_ASSERT_MSG_EQ (ifi.GetAttributeByType (link, InterfaceInfoMessage::IFL_A_LINK), true, "IFLA_LINK"); + NS_TEST_ASSERT_MSG_EQ (link.GetAttrPayload ().GetU32 (), (uint32_t)ifi.GetInterfaceIndex (), "IFLA_LINK"); + } } void diff --git a/test/test-ifindex.cc b/test/test-ifindex.cc new file mode 100644 index 00000000..3920a3d5 --- /dev/null +++ b/test/test-ifindex.cc @@ -0,0 +1,12 @@ +#include +#include "test-macros.h" + +// Interface indexes start at 1 (lo): with the ns-3 stack, NetDevice index +// + 1, as in netlink messages. +int +main (int argc, char *argv[]) +{ + TEST_ASSERT_EQUAL (if_nametoindex ("lo"), 1); + TEST_ASSERT_EQUAL (if_nametoindex ("nosuchdevice"), 0); + return 0; +} diff --git a/wscript b/wscript index 7f3f4d88..0779f37b 100755 --- a/wscript +++ b/wscript @@ -316,6 +316,7 @@ def build_dce_tests(module, bld): ['test-bug-multi-select', []], ['test-tsearch', []], ['test-signal', ['PTHREAD']], + ['test-ifindex', []], ['test-clock-gettime', []], ['test-gcc-builtin-apply', []], ['test-iostream', []], From bf097fdef9069b4903a38735df36d5c40b526d0b Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:15:03 +0200 Subject: [PATCH 18/71] waf: keep the module header task generator's name With a list of targets, its name became their comma-joined list, so the use of it by name did not resolve. (Headers were still generated first: their group is built before the modules'.) Co-Authored-By: Claude Opus 5.5 --- ns3waf/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ns3waf/__init__.py b/ns3waf/__init__.py index 43386009..66621de6 100644 --- a/ns3waf/__init__.py +++ b/ns3waf/__init__.py @@ -372,7 +372,7 @@ def run(task): # when a header changes. copies = [os.path.join(out_relpath, 'include', 'ns3', os.path.basename(h)) for h in headers] - bld(rule=run, source=headers, target=[target] + copies) + bld(rule=run, source=headers, target=[target] + copies, name=target) bld(use=[target], target='NS3_HEADERS_%s' % name.upper(), export_includes=['include']) bld.install_files(os.path.join('${PREFIX}', 'include', 'ns3'), headers + [target]) From ca50cc86c20b97d918fe0920e94ffc8acccecc6d Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:23:57 +0200 Subject: [PATCH 19/71] build_kernel_deps.sh: verify downloads, and stop on patches that fail The tarballs are checked against pinned SHA-256 digests, and thttpd is downloaded over HTTPS. The iperf and ping DCE patches were applied with "|| true", which also accepted a patch that does not apply, building programs without the changes DCE needs; a patch is now skipped only if already applied. The quagga module patch is skipped the same way, which made a second run abort. A source tree from an earlier run must be at the requested revision (e.g. not a libos-era iproute2 without "ip mptcp"). Co-Authored-By: Claude Opus 5.5 --- utils/build_kernel_deps.sh | 41 +++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/utils/build_kernel_deps.sh b/utils/build_kernel_deps.sh index 1b7c5d09..42aef108 100755 --- a/utils/build_kernel_deps.sh +++ b/utils/build_kernel_deps.sh @@ -50,17 +50,36 @@ fetch_git () { git -C "${SRC}/$3" -c advice.detachedHead=false checkout -q "$2" fi fi + # A source tree from an earlier run must be at the same revision. + if [ "$(git -C "${SRC}/$3" rev-parse HEAD)" != \ + "$(git -C "${SRC}/$3" rev-parse -q --verify "$2^{commit}")" ]; then + echo "${SRC}/$3 is not at $2: delete it to fetch it again." >&2 + exit 1 + fi } fetch_tar () { - # fetch_tar + # fetch_tar if [ ! -d "${SRC}/$2" ]; then wget -q -O "${SRC}/$2.tar" "$1" + if ! echo "$3 ${SRC}/$2.tar" | sha256sum -c --quiet - > /dev/null 2>&1; then + echo "Wrong checksum for $1" >&2 + rm -f "${SRC}/$2.tar" + exit 1 + fi tar xf "${SRC}/$2.tar" -C "${SRC}" rm -f "${SRC}/$2.tar" fi } +apply_patch () { + # apply_patch , in the current directory: once. + if patch -p1 -R -s -f --dry-run < "$1" > /dev/null 2>&1; then + return # already applied + fi + patch -p1 -s < "$1" +} + echo "== iproute2 ${IPROUTE2_REV}" fetch_git https://git.kernel.org/pub/scm/network/iproute2/iproute2.git "${IPROUTE2_REV}" iproute2 ( @@ -74,10 +93,11 @@ fetch_git https://git.kernel.org/pub/scm/network/iproute2/iproute2.git "${IPROUT cp "${SRC}/iproute2/ip/ip" "${BIN_DCE}/" echo "== iperf ${IPERF_VERSION}" -fetch_tar "https://sourceforge.net/projects/iperf/files/iperf-${IPERF_VERSION}.tar.gz/download" "iperf-${IPERF_VERSION}" +fetch_tar "https://sourceforge.net/projects/iperf/files/iperf-${IPERF_VERSION}.tar.gz/download" "iperf-${IPERF_VERSION}" \ + 636b4eff0431cea80667ea85a67ce4c68698760a9837e1e9d13096d20362265b ( cd "${SRC}/iperf-${IPERF_VERSION}" - patch -p1 -N -s < "${DCE_DIR}/utils/iperf_4_dce.patch" || true + apply_patch "${DCE_DIR}/utils/iperf_4_dce.patch" ./configure -q CFLAGS="-g -fPIC ${LEGACY_CFLAGS}" CXXFLAGS="-g -fPIC -U_FORTIFY_SOURCE" \ LDFLAGS="-pie -rdynamic" > /dev/null make -j"${JOBS}" > /dev/null @@ -85,7 +105,8 @@ fetch_tar "https://sourceforge.net/projects/iperf/files/iperf-${IPERF_VERSION}.t cp "${SRC}/iperf-${IPERF_VERSION}/src/iperf" "${BIN_DCE}/" echo "== thttpd ${THTTPD_VERSION}" -fetch_tar "http://www.acme.com/software/thttpd/thttpd-${THTTPD_VERSION}.tar.gz" "thttpd-${THTTPD_VERSION}" +fetch_tar "https://www.acme.com/software/thttpd/thttpd-${THTTPD_VERSION}.tar.gz" "thttpd-${THTTPD_VERSION}" \ + 07719b08b1cff6a21c08697a7bcb4395425b07ee753106262fb62a03a7d32360 ( cd "${SRC}/thttpd-${THTTPD_VERSION}" sed -i "s/rm conftest.c/rm -f conftests.c/" configure @@ -101,7 +122,8 @@ fetch_tar "http://www.acme.com/software/thttpd/thttpd-${THTTPD_VERSION}.tar.gz" cp "${SRC}/thttpd-${THTTPD_VERSION}/thttpd" "${BIN_DCE}/" echo "== wget ${WGET_VERSION}" -fetch_tar "https://ftp.gnu.org/gnu/wget/wget-${WGET_VERSION}.tar.gz" "wget-${WGET_VERSION}" +fetch_tar "https://ftp.gnu.org/gnu/wget/wget-${WGET_VERSION}.tar.gz" "wget-${WGET_VERSION}" \ + 52126be8cf1bddd7536886e74c053ad7d0ed2aa89b4b630f76785bac21695fcd ( cd "${SRC}/wget-${WGET_VERSION}" CFLAGS="-fPIC -g ${LEGACY_CFLAGS}" LDFLAGS="-pie -rdynamic" ./configure -q \ @@ -117,13 +139,14 @@ echo "== iputils ${IPUTILS_REV}" fetch_git https://github.com/iputils/iputils.git "${IPUTILS_REV}" iputils ( cd "${SRC}/iputils" - patch -p1 -N -s < "${DCE_DIR}/utils/iputils-ping6.patch" || true + apply_patch "${DCE_DIR}/utils/iputils-ping6.patch" make CFLAGS="-fpic -D_GNU_SOURCE -g ${LEGACY_CFLAGS}" LDFLAGS="-pie -rdynamic" ping ping6 > /dev/null ) cp "${SRC}/iputils/ping" "${SRC}/iputils/ping6" "${BIN_DCE}/" echo "== quagga ${QUAGGA_VERSION}" -fetch_tar "https://src.fedoraproject.org/repo/pkgs/quagga/quagga-${QUAGGA_VERSION}.tar.gz/64cc29394eb8a4e24649d19dac868f64/quagga-${QUAGGA_VERSION}.tar.gz" "quagga-${QUAGGA_VERSION}" +fetch_tar "https://src.fedoraproject.org/repo/pkgs/quagga/quagga-${QUAGGA_VERSION}.tar.gz/64cc29394eb8a4e24649d19dac868f64/quagga-${QUAGGA_VERSION}.tar.gz" "quagga-${QUAGGA_VERSION}" \ + b7a98cc6b022bb0cb405557b3d920cf513150f64384dbd0a2248b5bd248df58b ( cd "${SRC}/quagga-${QUAGGA_VERSION}" CFLAGS="-fPIC -g ${LEGACY_CFLAGS}" LDFLAGS="-pie -rdynamic" ./configure -q \ @@ -139,8 +162,8 @@ echo "== ns-3-dce-quagga module" fetch_git https://github.com/direct-code-execution/ns-3-dce-quagga.git "${QUAGGA_MODULE_REV}" ns-3-dce-quagga ( cd "${SRC}/ns-3-dce-quagga" - git apply --check "${DCE_DIR}/utils/ns-3-dce-quagga-ns3.patch" 2> /dev/null \ - && git apply "${DCE_DIR}/utils/ns-3-dce-quagga-ns3.patch" + git apply --reverse --check "${DCE_DIR}/utils/ns-3-dce-quagga-ns3.patch" 2> /dev/null \ + || git apply "${DCE_DIR}/utils/ns-3-dce-quagga-ns3.patch" ) rm -rf "${DEPS_DIR}/ns-3-dce-quagga" cp -a "${SRC}/ns-3-dce-quagga" "${DEPS_DIR}/ns-3-dce-quagga" From 50f3f13cb78f8dc5df9e9fc555607e016922c94b Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:23:57 +0200 Subject: [PATCH 20/71] build_lkl.sh: require the LKL tree to be at the pinned revision Co-Authored-By: Claude Opus 5.5 --- utils/build_lkl.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/build_lkl.sh b/utils/build_lkl.sh index 10e2f3f9..fcd02c3d 100755 --- a/utils/build_lkl.sh +++ b/utils/build_lkl.sh @@ -35,6 +35,10 @@ if [ ! -d "${LKL_DIR}" ]; then git -C "${LKL_DIR}" -c advice.detachedHead=false checkout -q FETCH_HEAD fi LKL_DIR="$(cd "${LKL_DIR}" && pwd)" +if [ "$(git -C "${LKL_DIR}" rev-parse HEAD)" != "${LKL_REV}" ]; then + echo "${LKL_DIR} is not at ${LKL_REV}: delete it to fetch it again." >&2 + exit 1 +fi for patch in "${DCE_DIR}"/utils/lkl-patches/*.patch; do if git -C "${LKL_DIR}" apply --reverse --check "${patch}" 2> /dev/null; then From 4640a1756b239ea29205155f25443f22284d6342 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:23:57 +0200 Subject: [PATCH 21/71] LKL patch: give up on a stuck clock only after 1024 stuck measurements Breaking out of jent_gen_entropy() on the repetition count test's intermittent failure also ended entropy generation early on a real clock, without reporting it outside of FIPS mode, and for good, since the failure flag stays set. Count consecutive stuck measurements instead, with a limit far above the test's cutoffs. Co-Authored-By: Claude Opus 5.5 --- ...-not-loop-forever-on-a-clock-that-do.patch | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/utils/lkl-patches/0003-crypto-jitter-do-not-loop-forever-on-a-clock-that-do.patch b/utils/lkl-patches/0003-crypto-jitter-do-not-loop-forever-on-a-clock-that-do.patch index 4c19a16a..b14f585f 100644 --- a/utils/lkl-patches/0003-crypto-jitter-do-not-loop-forever-on-a-clock-that-do.patch +++ b/utils/lkl-patches/0003-crypto-jitter-do-not-loop-forever-on-a-clock-that-do.patch @@ -11,25 +11,34 @@ jent_gen_entropy() repeats the measurement until the health tests fail, but outside of FIPS mode jent_health_failure() never reports a failure, so the kernel hangs while initializing the jitterentropy RNG. -Stop when the repetition count test failed. Only its intermittent -failure can be seen here: the count restarts after it, so it never -reaches the permanent failure cutoff. The time tests of -jent_entropy_init() then reject the clock, and the jitterentropy RNG is -not registered, as for any other unsuitable clock. +Give up after many consecutive stuck measurements, far more than the +repetition count test's cutoffs, so that only a clock that does not +advance at all is given up on. The time tests of jent_entropy_init() +then reject the clock, and the jitterentropy RNG is not registered, as +for any other unsuitable clock. This happens with LKL in DCE, where the kernel runs on simulated time, which does not advance while the kernel initializes. TOUPSTREAM --- - crypto/jitterentropy.c | 12 +++++++++++- - 1 file changed, 11 insertions(+), 1 deletion(-) + crypto/jitterentropy.c | 16 ++++++++++++++-- + 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 3b390bd6c..9025437f9 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c -@@ -566,8 +566,18 @@ static void jent_gen_entropy(struct rand_data *ec) +@@ -556,7 +556,7 @@ static int jent_measure_jitter(struct rand_data *ec, __u64 *ret_current_delta) + */ + static void jent_gen_entropy(struct rand_data *ec) + { +- unsigned int k = 0, safety_factor = 0; ++ unsigned int k = 0, safety_factor = 0, stuck = 0; + + if (fips_enabled) + safety_factor = JENT_ENTROPY_SAFETY_FACTOR; +@@ -566,8 +566,19 @@ static void jent_gen_entropy(struct rand_data *ec) while (!jent_health_failure(ec)) { /* If a stuck measurement is received, repeat measurement */ @@ -37,15 +46,16 @@ index 3b390bd6c..9025437f9 100644 + if (jent_measure_jitter(ec, NULL)) { + /* + * Outside of FIPS mode, the health tests do not end -+ * the loop: give up on a clock that does not advance. -+ * The count restarts after an intermittent failure, -+ * so a permanent failure is never reported. ++ * the loop: give up on a clock that does not advance, ++ * which jent_entropy_init() rejects. The limit is far ++ * above the cutoffs of the repetition count test, ++ * which a usable clock does not reach. + */ -+ if (ec->health_failure & -+ (JENT_RCT_FAILURE | JENT_RCT_FAILURE_PERMANENT)) ++ if (++stuck >= 1024) + break; continue; + } ++ stuck = 0; /* * We multiply the loop value with ->osr to obtain the From 53f2e90f002552a180663f4e33717f94704689f7 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:24:11 +0200 Subject: [PATCH 22/71] Tighten the checks of dce-lkl-boot and dce-lkl-p2p dce-lkl-boot: reject --nodes=0, count a failed uname, fail if the sleep took host time, and run long enough for all the nodes. dce-lkl-p2p: check that tcp-server read all that tcp-client sent, not only that it stopped reading. Co-Authored-By: Claude Opus 5.5 --- example/dce-lkl-boot.cc | 17 ++++++++++++++--- example/dce-lkl-p2p.cc | 6 +++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/example/dce-lkl-boot.cc b/example/dce-lkl-boot.cc index f6a991f3..4a220994 100644 --- a/example/dce-lkl-boot.cc +++ b/example/dce-lkl-boot.cc @@ -31,6 +31,10 @@ CheckNode (Ptr node) long ret = kernel->Syscall (__lkl__NR_uname, params); std::cout << "node " << node->GetId () << ": uname=" << ret << " " << uts[0] << " " << uts[2] << std::endl; + if (ret != 0) + { + g_failures++; + } struct { @@ -45,8 +49,9 @@ CheckNode (Ptr node) double wall = std::chrono::duration (std::chrono::steady_clock::now () - wallBefore).count (); std::cout << "node " << node->GetId () << ": nanosleep(10s)=" << ret << " simulated " << slept.GetNanoSeconds () << " ns, wall " << wall << " s" << std::endl; - // Linux adds the task's timer slack (50 us by default) to the sleep. - if (ret != 0 || slept < Seconds (10) || slept > Seconds (10) + MilliSeconds (1)) + // Linux adds the task's timer slack (50 us by default) to the sleep. The + // host must not sleep. + if (ret != 0 || slept < Seconds (10) || slept > Seconds (10) + MilliSeconds (1) || wall > 5) { g_failures++; } @@ -73,6 +78,11 @@ main (int argc, char *argv[]) CommandLine cmd; cmd.AddValue ("nodes", "Number of nodes", nNodes); cmd.Parse (argc, argv); + if (nNodes == 0) + { + std::cerr << "--nodes must be at least 1" << std::endl; + return 1; + } NodeContainer nodes; nodes.Create (nNodes); @@ -85,7 +95,8 @@ main (int argc, char *argv[]) Simulator::ScheduleWithContext (i, Seconds (1.0 + 0.5 * i), &StartCheck, nodes.Get (i)); } - Simulator::Stop (Seconds (30)); + // The last node starts at 1 + 0.5 (n - 1) s, and sleeps 10 s. + Simulator::Stop (Seconds (15 + 0.5 * nNodes)); Simulator::Run (); Simulator::Destroy (); diff --git a/example/dce-lkl-p2p.cc b/example/dce-lkl-p2p.cc index c4035352..097548a3 100644 --- a/example/dce-lkl-p2p.cc +++ b/example/dce-lkl-p2p.cc @@ -38,7 +38,11 @@ Finished (std::string name, uint32_t node, uint16_t pid, int status) std::ifstream file (path.str ()); std::stringstream out; out << file.rdbuf (); - ok = out.str ().find ("did read all buffers") != std::string::npos; + // tcp-server reads until the end of the connection: check that it + // got the 1000 kB tcp-client sends. udp-server reads 1000 datagrams. + std::string done = name.find ("tcp") != std::string::npos + ? "did read all buffers tot:1024000\n" : "did read all buffers"; + ok = out.str ().find (done) != std::string::npos; } std::cout << Simulator::Now ().GetSeconds () << "s " << name << " exited with " << status << (ok ? "" : " FAILED") << std::endl; From bb91d0a23d559fa1138f8bf4a944dcbe5600f175 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:37:29 +0200 Subject: [PATCH 23/71] MPTCP examples: count TCP payload, and fit the subflow limits to nRtrs The check that every path carries data counted TCP headers, so a path with only a handshake or ACKs passed. The subflow limits were fixed at 4, while the fullmesh endpoints need nRtrs * nRtrs - 1 subflows: derive them, and reject the router counts Linux MPTCP's 8 subflows cannot cover (more than 3). Co-Authored-By: Claude Opus 5.5 --- example/dce-cradle-mptcp.cc | 24 ++++++++++++++++++++---- example/dce-iperf-mptcp.cc | 24 ++++++++++++++++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/example/dce-cradle-mptcp.cc b/example/dce-cradle-mptcp.cc index d0f396ab..86a63662 100644 --- a/example/dce-cradle-mptcp.cc +++ b/example/dce-cradle-mptcp.cc @@ -9,7 +9,7 @@ using namespace ns3; -// Bytes the server received from each address of the client. +// TCP payload bytes the server received from each address of the client. static std::map g_received; static void @@ -22,6 +22,12 @@ ServerRx (Ptr p) if (ppp.GetProtocol () == 0x0021) // IPv4 { packet->RemoveHeader (ip); + if (ip.GetProtocol () != 6) + { + return; + } + TcpHeader tcp; + packet->RemoveHeader (tcp); g_received[ip.GetSource ()] += packet->GetSize (); } } @@ -40,6 +46,14 @@ int main (int argc, char *argv[]) CommandLine cmd; cmd.AddValue ("nRtrs", "Number of routers. Default 2", nRtrs); cmd.Parse (argc, argv); + // The client opens subflows between all its addresses and all the + // server's (fullmesh): nRtrs * nRtrs - 1 additional subflows, and Linux + // MPTCP allows at most 8. + if (nRtrs < 1 || nRtrs > 3) + { + std::cerr << "nRtrs must be from 1 to 3" << std::endl; + return 1; + } NodeContainer nodes, routers; nodes.Create (2); @@ -122,11 +136,13 @@ int main (int argc, char *argv[]) stack.SysctlSet (routers, ".net.ipv4.conf.all.forwarding", "1"); - // MPTCP: up to 4 subflows per connection; the client opens subflows from - // its other addresses, the server announces its other addresses. + // MPTCP: the client opens subflows from its other addresses, the server + // announces its other addresses. + cmd_oss.str (""); + cmd_oss << "mptcp limits set subflows " << nRtrs * nRtrs - 1 << " add_addr_accepted " << nRtrs - 1; for (uint32_t n = 0; n < 2; n++) { - LinuxStackHelper::RunIp (nodes.Get (n), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (nodes.Get (n), Seconds (0.5), cmd_oss.str ().c_str ()); } for (uint32_t i = 1; i < nRtrs; i++) { diff --git a/example/dce-iperf-mptcp.cc b/example/dce-iperf-mptcp.cc index 3e8eb2e8..8c6e60e0 100644 --- a/example/dce-iperf-mptcp.cc +++ b/example/dce-iperf-mptcp.cc @@ -9,7 +9,7 @@ using namespace ns3; -// Bytes the server received from each address of the client. +// TCP payload bytes the server received from each address of the client. static std::map g_received; static void @@ -22,6 +22,12 @@ ServerRx (Ptr p) if (ppp.GetProtocol () == 0x0021) // IPv4 { packet->RemoveHeader (ip); + if (ip.GetProtocol () != 6) + { + return; + } + TcpHeader tcp; + packet->RemoveHeader (tcp); g_received[ip.GetSource ()] += packet->GetSize (); } } @@ -40,6 +46,14 @@ int main (int argc, char *argv[]) CommandLine cmd; cmd.AddValue ("nRtrs", "Number of routers. Default 2", nRtrs); cmd.Parse (argc, argv); + // The client opens subflows between all its addresses and all the + // server's (fullmesh): nRtrs * nRtrs - 1 additional subflows, and Linux + // MPTCP allows at most 8. + if (nRtrs < 1 || nRtrs > 3) + { + std::cerr << "nRtrs must be from 1 to 3" << std::endl; + return 1; + } NodeContainer nodes, routers; nodes.Create (2); @@ -122,11 +136,13 @@ int main (int argc, char *argv[]) stack.SysctlSet (routers, ".net.ipv4.conf.all.forwarding", "1"); - // MPTCP: up to 4 subflows per connection; the client opens subflows from - // its other addresses, the server announces its other addresses. + // MPTCP: the client opens subflows from its other addresses, the server + // announces its other addresses. + cmd_oss.str (""); + cmd_oss << "mptcp limits set subflows " << nRtrs * nRtrs - 1 << " add_addr_accepted " << nRtrs - 1; for (uint32_t n = 0; n < 2; n++) { - LinuxStackHelper::RunIp (nodes.Get (n), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); + LinuxStackHelper::RunIp (nodes.Get (n), Seconds (0.5), cmd_oss.str ().c_str ()); } for (uint32_t i = 1; i < nRtrs; i++) { From 99117d76a98f91d8fedf687dc3dbadde92bb337a Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:39:57 +0200 Subject: [PATCH 24/71] dce-mptcp-handoff-v6: open the subflows over the second access point The mobile node was out of the second access point's range (ns-3 no longer detects frames below -82 dBm): place it within reach of both. Its table 3 default route was added before sim1 was up, and failed; the address added at 15.2 s had no source routing rule, and its MPTCP endpoint was added while the address was still tentative, so no subflow was opened from it. Co-Authored-By: Claude Opus 5.5 --- example/dce-mptcp-handoff-v6.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/example/dce-mptcp-handoff-v6.cc b/example/dce-mptcp-handoff-v6.cc index e7a98519..1e3105bf 100644 --- a/example/dce-mptcp-handoff-v6.cc +++ b/example/dce-mptcp-handoff-v6.cc @@ -96,7 +96,7 @@ int main (int argc, char *argv[]) positionAlloc->Add (Vector (50.0, -75.0, 0.0)); // SV positionAlloc->Add (Vector (50.0, -50.0, 0.0)); // R positionAlloc->Add (Vector (0.0, 10.0, 0.0)); // AR1 - positionAlloc->Add (Vector (100.0, 10.0, 0.0)); // AR2 + positionAlloc->Add (Vector (80.0, 10.0, 0.0)); // AR2 mobility.SetPositionAllocator (positionAlloc); mobility.SetMobilityModel ("ns3::ConstantPositionMobilityModel"); mobility.Install (sv); @@ -118,6 +118,9 @@ int main (int argc, char *argv[]) "Pause", StringValue ("ns3::ConstantRandomVariable[Constant=0.2]")); mobility.SetMobilityModel ("ns3::ConstantPositionMobilityModel"); mobility.Install (mn); + // Within reach of both access points (ns-3 no longer detects frames + // below -82 dBm, about 50 m away here). + mn.Get (0)->GetObject ()->SetPosition (Vector (40.0, 10.0, 0.0)); // backend @@ -232,7 +235,9 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (mn.Get (0), Seconds (2.0), "link set sim1 up"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "-6 rule add from 2001:1:2:7:200:ff:fe00:9 table 3"); // LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "route add default via fe80::200:ff:fe00:a dev sim1"); - LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.20), "route add default via fe80::200:ff:fe00:a dev sim1 table 3"); + // Once sim1 is up. + LinuxStackHelper::RunIp (mn.Get (0), Seconds (2.10), "route add default via fe80::200:ff:fe00:a dev sim1 table 3"); + LinuxStackHelper::RunIp (mn.Get (0), Seconds (15.20), "-6 rule add from 2001:1:2:7::3939 table 3"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (15.20), "-6 addr change 2001:1:2:7::3939/64 dev sim1"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (18.20), "-6 addr flush dev sim1"); // disable default injection from ra @@ -257,7 +262,8 @@ int main (int argc, char *argv[]) LinuxStackHelper::RunIp (mn.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); LinuxStackHelper::RunIp (sv.Get (0), Seconds (0.5), "mptcp limits set subflows 4 add_addr_accepted 4"); LinuxStackHelper::RunIp (mn.Get (0), Seconds (9.0), "mptcp endpoint add 2001:1:2:7:200:ff:fe00:9 dev sim1 subflow"); - LinuxStackHelper::RunIp (mn.Get (0), Seconds (15.3), "mptcp endpoint add 2001:1:2:7::3939 dev sim1 subflow"); + // Once duplicate address detection is over (about 1 s). + LinuxStackHelper::RunIp (mn.Get (0), Seconds (17.0), "mptcp endpoint add 2001:1:2:7::3939 dev sim1 subflow"); Config::ConnectWithoutContext ("/NodeList/" + std::to_string (sv.Get (0)->GetId ()) + "/DeviceList/0/$ns3::PointToPointNetDevice/PhyRxEnd", MakeCallback (&ServerRx)); From 658ed0623db6d2424be319d42794bf98e6e85b02 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:41:50 +0200 Subject: [PATCH 25/71] Ipv4DceRouting: deliver multicast locally when it is also forwarded With a static multicast route, Ipv4StaticRouting only forwards the packet, so raw sockets (e.g. a routing daemon's for 224.0.0.5) stopped receiving the group. Co-Authored-By: Claude Opus 5.5 --- model/ipv4-dce-routing.cc | 14 ++--- test/ipv4-dce-routing-test.cc | 111 ++++++++++++++++++++++++++++++++++ wscript | 1 + 3 files changed, 118 insertions(+), 8 deletions(-) create mode 100644 test/ipv4-dce-routing-test.cc diff --git a/model/ipv4-dce-routing.cc b/model/ipv4-dce-routing.cc index f26206e4..1c74aac1 100644 --- a/model/ipv4-dce-routing.cc +++ b/model/ipv4-dce-routing.cc @@ -70,20 +70,18 @@ Ipv4DceRouting::RouteInput (Ptr p, const Ipv4Header &header, PtrGetInterfaceForDevice (idev)); + Ipv4StaticRouting::RouteInput (p, header, idev, ucb, mcb, lcb, ecb); return true; } - return false; + return Ipv4StaticRouting::RouteInput (p, header, idev, ucb, mcb, lcb, ecb); } void diff --git a/test/ipv4-dce-routing-test.cc b/test/ipv4-dce-routing-test.cc new file mode 100644 index 00000000..b478d98d --- /dev/null +++ b/test/ipv4-dce-routing-test.cc @@ -0,0 +1,111 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +#include "ns3/test.h" +#include "ns3/node.h" +#include "ns3/packet.h" +#include "ns3/simple-net-device.h" +#include "ns3/internet-stack-helper.h" +#include "ns3/ipv4.h" +#include "ns3/ipv4-header.h" +#include "ns3/ipv4-route.h" +#include "ns3/ipv4-dce-routing.h" +#include "ns3/ipv4-dce-routing-helper.h" + +using namespace ns3; + +namespace { + +int g_local = 0; +int g_forwarded = 0; + +void +Unicast (Ptr route, Ptr p, const Ipv4Header &header) +{ +} + +void +Multicast (Ptr route, Ptr p, const Ipv4Header &header) +{ + g_forwarded++; +} + +void +Local (Ptr p, const Ipv4Header &header, uint32_t iif) +{ + g_local++; +} + +void +Error (Ptr p, const Ipv4Header &header, Socket::SocketErrno error) +{ +} + +} // namespace + +/** + * Multicast packets are delivered locally (to raw sockets, e.g. of routing + * daemons), with or without a multicast route to forward them. + */ +class Ipv4DceRoutingMulticastTestCase : public TestCase +{ +public: + Ipv4DceRoutingMulticastTestCase () + : TestCase ("multicast local delivery") + { + } + +private: + virtual void + DoRun (void) + { + Ptr node = CreateObject (); + Ptr in = CreateObject (); + Ptr out = CreateObject (); + in->SetAddress (Mac48Address::Allocate ()); + out->SetAddress (Mac48Address::Allocate ()); + node->AddDevice (in); + node->AddDevice (out); + InternetStackHelper stack; + Ipv4DceRoutingHelper routingHelper; + stack.SetRoutingHelper (routingHelper); + stack.SetIpv6StackInstall (false); + stack.Install (node); + Ptr ipv4 = node->GetObject (); + uint32_t inIf = ipv4->AddInterface (in); + uint32_t outIf = ipv4->AddInterface (out); + ipv4->AddAddress (inIf, Ipv4InterfaceAddress ("10.0.0.1", "255.255.255.0")); + ipv4->AddAddress (outIf, Ipv4InterfaceAddress ("10.0.1.1", "255.255.255.0")); + ipv4->SetUp (inIf); + ipv4->SetUp (outIf); + Ptr routing = DynamicCast (ipv4->GetRoutingProtocol ()); + NS_TEST_ASSERT_MSG_EQ (bool (routing), true, "Ipv4DceRouting"); + + Ipv4Header header; + header.SetSource ("10.0.0.2"); + header.SetDestination ("224.0.0.5"); + Ptr p = Create (10); + + g_local = g_forwarded = 0; + routing->RouteInput (p, header, in, MakeCallback (&Unicast), MakeCallback (&Multicast), + MakeCallback (&Local), MakeCallback (&Error)); + NS_TEST_EXPECT_MSG_EQ (g_local, 1, "delivered without a multicast route"); + NS_TEST_EXPECT_MSG_EQ (g_forwarded, 0, "nothing to forward without a multicast route"); + + std::vector outputs (1, outIf); + routing->AddMulticastRoute ("10.0.0.2", "224.0.0.5", inIf, outputs); + g_local = g_forwarded = 0; + routing->RouteInput (p, header, in, MakeCallback (&Unicast), MakeCallback (&Multicast), + MakeCallback (&Local), MakeCallback (&Error)); + NS_TEST_EXPECT_MSG_EQ (g_local, 1, "delivered with a multicast route"); + NS_TEST_EXPECT_MSG_EQ (g_forwarded, 1, "forwarded with a multicast route"); + } +}; + +static class Ipv4DceRoutingTestSuite : public TestSuite +{ +public: + Ipv4DceRoutingTestSuite () + : TestSuite ("dce-ipv4-routing", Type::UNIT) + { + AddTestCase (new Ipv4DceRoutingMulticastTestCase (), TestCase::Duration::QUICK); + } +} g_ipv4DceRoutingTestSuite; diff --git a/wscript b/wscript index 0779f37b..fbe2f5ff 100755 --- a/wscript +++ b/wscript @@ -258,6 +258,7 @@ def dce_kw(**kw): def build_dce_tests(module, bld): tests_source = [ 'test/dce-manager-test.cc', + 'test/ipv4-dce-routing-test.cc', ] if bld.env['LINUX_STACK']: tests_source += [ From ed1ce26557aa572c8b449549809e6ea4f93ec274 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:44:55 +0200 Subject: [PATCH 26/71] Release the socket that socket() cannot return for lack of descriptors On EMFILE, dce_socket leaked the socket it had created: with LKL, its kernel socket, so that the kernel soon ran out of descriptors too. Co-Authored-By: Claude Opus 5.5 --- model/dce-fd.cc | 6 ++---- test/dce-manager-test.cc | 1 + test/test-emfile.cc | 37 +++++++++++++++++++++++++++++++++++++ wscript | 1 + 4 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 test/test-emfile.cc diff --git a/model/dce-fd.cc b/model/dce-fd.cc index 9db7d5bc..f9d86267 100644 --- a/model/dce-fd.cc +++ b/model/dce-fd.cc @@ -438,13 +438,11 @@ int dce_socket (int domain, int type, int protocol) int fd = UtilsAllocateFd (); if (fd == -1) { + socket->Close (); + socket->Unref (); current->err = EMFILE; return -1; } - if (!socket) - { - return -1; - } socket->IncFdCount (); current->process->openFiles[fd] = new FileUsage (fd, socket); diff --git a/test/dce-manager-test.cc b/test/dce-manager-test.cc index 09fd5df5..37c7976c 100644 --- a/test/dce-manager-test.cc +++ b/test/dce-manager-test.cc @@ -226,6 +226,7 @@ DceManagerTestSuite::DceManagerTestSuite () { "test-gcc-builtin-apply", 0, "", false, false, NS3_MASK|LINUX_MASK}, { "test-signal", 30, "", false, false, NS3_MASK|LINUX_MASK}, { "test-ifindex", 30, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-emfile", 30, "", true, false, NS3_MASK|LINUX_MASK}, }; // Prepare directories and files for test-stdio diff --git a/test/test-emfile.cc b/test/test-emfile.cc new file mode 100644 index 00000000..63718e13 --- /dev/null +++ b/test/test-emfile.cc @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include +#include "test-macros.h" + +// Running out of file descriptors: socket() keeps failing with EMFILE, and +// does not leak the sockets it could not return. +int +main (int argc, char *argv[]) +{ + std::vector fds; + while (true) + { + int fd = socket (AF_INET, SOCK_DGRAM, 0); + if (fd < 0) + { + TEST_ASSERT_EQUAL (errno, EMFILE); + break; + } + fds.push_back (fd); + } + for (int i = 0; i < 100; i++) + { + TEST_ASSERT_EQUAL (socket (AF_INET, SOCK_DGRAM, 0), -1); + TEST_ASSERT_EQUAL (errno, EMFILE); + } + for (size_t i = 0; i < fds.size (); i++) + { + TEST_ASSERT_EQUAL (close (fds[i]), 0); + } + int fd = socket (AF_INET, SOCK_DGRAM, 0); + TEST_ASSERT (fd >= 0); + TEST_ASSERT_EQUAL (close (fd), 0); + return 0; +} diff --git a/wscript b/wscript index fbe2f5ff..62957782 100755 --- a/wscript +++ b/wscript @@ -318,6 +318,7 @@ def build_dce_tests(module, bld): ['test-tsearch', []], ['test-signal', ['PTHREAD']], ['test-ifindex', []], + ['test-emfile', []], ['test-clock-gettime', []], ['test-gcc-builtin-apply', []], ['test-iostream', []], From 4fefd5a14207f137332f9adee6760bdf0bc0bdf6 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:48:10 +0200 Subject: [PATCH 27/71] ns-3 stack: apply IPV6_PKTINFO on connected sockets, and reject bad ones The interface selected with IPV6_PKTINFO was ignored when sending on a connected socket. Like Linux, sendmsg now fails with ENODEV for an interface that does not exist, instead of using the default route, and with EINVAL for a control message longer than the control buffer, instead of reading past it. Co-Authored-By: Claude Opus 5.5 --- model/unix-datagram-socket-fd.cc | 25 +++++++++-- model/unix-datagram-socket-fd.h | 2 +- test/dce-manager-test.cc | 1 + test/test-ipv6-pktinfo.cc | 71 ++++++++++++++++++++++++++++++++ wscript | 1 + 5 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 test/test-ipv6-pktinfo.cc diff --git a/model/unix-datagram-socket-fd.cc b/model/unix-datagram-socket-fd.cc index 5dce831c..c2f914f8 100644 --- a/model/unix-datagram-socket-fd.cc +++ b/model/unix-datagram-socket-fd.cc @@ -297,12 +297,23 @@ UnixDatagramSocketFd::DoSendmsg (const struct msghdr *msg, int flags) Ptr outDev = 0; for (struct cmsghdr *c = CMSG_FIRSTHDR (msg); c != 0; c = CMSG_NXTHDR ((struct msghdr *)msg, c)) { + if (c->cmsg_len < sizeof (struct cmsghdr) + || (uint8_t *)c + c->cmsg_len > (uint8_t *)msg->msg_control + msg->msg_controllen) + { + current->err = EINVAL; + return -1; + } if (c->cmsg_level == SOL_IPV6 && c->cmsg_type == IPV6_PKTINFO && c->cmsg_len >= CMSG_LEN (sizeof (struct in6_pktinfo))) { struct in6_pktinfo *pkt6 = (struct in6_pktinfo *)CMSG_DATA (c); Ptr node = current->process->manager->GetObject (); - if (pkt6->ipi6_ifindex >= 1 && pkt6->ipi6_ifindex <= node->GetNDevices ()) + if (pkt6->ipi6_ifindex > node->GetNDevices ()) + { + current->err = ENODEV; + return -1; + } + if (pkt6->ipi6_ifindex >= 1) { outDev = node->GetDevice (pkt6->ipi6_ifindex - 1); } @@ -390,7 +401,7 @@ UnixDatagramSocketFd::DoSendmsg (const struct msghdr *msg, int flags) TaskManager *manager = TaskManager::Current (); result = -1; manager->ExecOnMain (MakeEvent (&UnixDatagramSocketFd::MainSend, - this, &result, packet)); + this, &result, packet, outDev)); } if (result == -1) { @@ -483,8 +494,16 @@ UnixDatagramSocketFd::MainSendTo (int *r, Ptr p, uint32_t f, Address ad, m_socket->BindToNetDevice (bound); } void -UnixDatagramSocketFd::MainSend (int *r, Ptr p) +UnixDatagramSocketFd::MainSend (int *r, Ptr p, Ptr dev) { + if (!dev) + { + *r = m_socket->Send (p); + return; + } + Ptr bound = m_socket->GetBoundNetDevice (); + m_socket->BindToNetDevice (dev); *r = m_socket->Send (p); + m_socket->BindToNetDevice (bound); } } // namespace ns3 diff --git a/model/unix-datagram-socket-fd.h b/model/unix-datagram-socket-fd.h index d6aa2400..78f6621b 100644 --- a/model/unix-datagram-socket-fd.h +++ b/model/unix-datagram-socket-fd.h @@ -33,7 +33,7 @@ class UnixDatagramSocketFd : public UnixSocketFd void QueueErr (sock_extended_err ee, struct sockaddr_in offender, uint8_t ttl); void CopyMacAddress (const Address &a, uint8_t* const buf); void MainSendTo (int *r, Ptr p, uint32_t f, Address ad, Ptr dev); - void MainSend (int *r, Ptr p); + void MainSend (int *r, Ptr p, Ptr dev); struct Error { diff --git a/test/dce-manager-test.cc b/test/dce-manager-test.cc index 37c7976c..2456b921 100644 --- a/test/dce-manager-test.cc +++ b/test/dce-manager-test.cc @@ -227,6 +227,7 @@ DceManagerTestSuite::DceManagerTestSuite () { "test-signal", 30, "", false, false, NS3_MASK|LINUX_MASK}, { "test-ifindex", 30, "", true, false, NS3_MASK|LINUX_MASK}, { "test-emfile", 30, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-ipv6-pktinfo", 30, "", true, false, NS3_MASK|LINUX_MASK}, }; // Prepare directories and files for test-stdio diff --git a/test/test-ipv6-pktinfo.cc b/test/test-ipv6-pktinfo.cc new file mode 100644 index 00000000..3fe59bd8 --- /dev/null +++ b/test/test-ipv6-pktinfo.cc @@ -0,0 +1,71 @@ +#define _GNU_SOURCE 1 +#include +#include +#include +#include +#include +#include +#include "test-macros.h" + +// Sends one datagram with an IPV6_PKTINFO control message selecting +// interface ifindex; controllen is the length of the control buffer given. +static ssize_t +SendWithPktinfo (int sock, struct sockaddr_in6 *to, unsigned ifindex, size_t controllen) +{ + char data[] = "hello"; + struct iovec iov; + iov.iov_base = data; + iov.iov_len = sizeof (data); + union + { + char buf[CMSG_SPACE (sizeof (struct in6_pktinfo))]; + struct cmsghdr align; + } control; + memset (&control, 0, sizeof (control)); + struct msghdr msg; + memset (&msg, 0, sizeof (msg)); + msg.msg_name = to; + msg.msg_namelen = to ? sizeof (*to) : 0; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = control.buf; + msg.msg_controllen = controllen; + struct cmsghdr *c = (struct cmsghdr *)control.buf; + c->cmsg_level = IPPROTO_IPV6; + c->cmsg_type = IPV6_PKTINFO; + c->cmsg_len = CMSG_LEN (sizeof (struct in6_pktinfo)); + struct in6_pktinfo *info = (struct in6_pktinfo *)CMSG_DATA (c); + info->ipi6_ifindex = ifindex; + return sendmsg (sock, &msg, 0); +} + +int +main (int argc, char *argv[]) +{ + int sock = socket (AF_INET6, SOCK_DGRAM, 0); + TEST_ASSERT (sock >= 0); + struct sockaddr_in6 to; + memset (&to, 0, sizeof (to)); + to.sin6_family = AF_INET6; + to.sin6_port = htons (9); + to.sin6_addr = in6addr_loopback; + size_t controllen = CMSG_SPACE (sizeof (struct in6_pktinfo)); + + // lo is interface 1. + TEST_ASSERT_EQUAL (SendWithPktinfo (sock, &to, 1, controllen), 6); + // No such interface. + TEST_ASSERT_EQUAL (SendWithPktinfo (sock, &to, 99, controllen), -1); + TEST_ASSERT_EQUAL (errno, ENODEV); + // A control message longer than the control buffer. + TEST_ASSERT_EQUAL (SendWithPktinfo (sock, &to, 1, CMSG_LEN (sizeof (struct in6_pktinfo)) - 4), -1); + TEST_ASSERT_EQUAL (errno, EINVAL); + + // The same on a connected socket. + TEST_ASSERT_EQUAL (connect (sock, (struct sockaddr *)&to, sizeof (to)), 0); + TEST_ASSERT_EQUAL (SendWithPktinfo (sock, 0, 1, controllen), 6); + TEST_ASSERT_EQUAL (SendWithPktinfo (sock, 0, 99, controllen), -1); + TEST_ASSERT_EQUAL (errno, ENODEV); + + TEST_ASSERT_EQUAL (close (sock), 0); + return 0; +} diff --git a/wscript b/wscript index 62957782..59857b6f 100755 --- a/wscript +++ b/wscript @@ -319,6 +319,7 @@ def build_dce_tests(module, bld): ['test-signal', ['PTHREAD']], ['test-ifindex', []], ['test-emfile', []], + ['test-ipv6-pktinfo', []], ['test-clock-gettime', []], ['test-gcc-builtin-apply', []], ['test-iostream', []], From 8774b3dbbc84f2161d9f9e2deffd021b2a133aca Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:51:02 +0200 Subject: [PATCH 28/71] ns-3 stack: apply ICMP6_FILTER to raw ICMPv6 sockets The option was accepted and ignored, so the socket received the ICMPv6 types it blocked. ns-3 does not expose its raw socket's filter: DCE keeps it, drops the blocked messages when receiving, and returns it from getsockopt. Co-Authored-By: Claude Opus 5.5 --- model/unix-datagram-socket-fd.cc | 8 ++++++ model/unix-socket-fd.cc | 32 ++++++++++++++++++---- model/unix-socket-fd.h | 4 +++ test/dce-manager-test.cc | 1 + test/test-icmp6-filter.cc | 47 ++++++++++++++++++++++++++++++++ wscript | 1 + 6 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 test/test-icmp6-filter.cc diff --git a/model/unix-datagram-socket-fd.cc b/model/unix-datagram-socket-fd.cc index c2f914f8..40564942 100644 --- a/model/unix-datagram-socket-fd.cc +++ b/model/unix-datagram-socket-fd.cc @@ -173,6 +173,7 @@ UnixDatagramSocketFd::DoRecvmsg (struct msghdr *msg, int flags) // msg->msg_controllen = 0; // why??? } +again: if (!WaitRecvDoSignal (flags & MSG_DONTWAIT)) { // current->err set by call above. @@ -198,6 +199,13 @@ UnixDatagramSocketFd::DoRecvmsg (struct msghdr *msg, int flags) { Ipv6Header ipv6Header; packet->RemoveHeader (ipv6Header); + // ICMP6_FILTER (the filter is set on ICMPv6 sockets only) + uint8_t type; + if (m_icmp6FilterSet && !(flags & MSG_PEEK) + && packet->CopyData (&type, 1) == 1 && ICMP6_FILTER_WILLBLOCK (type, &m_icmp6Filter)) + { + goto again; + } } if ((PacketSocketAddress::IsMatchingType (from))) { diff --git a/model/unix-socket-fd.cc b/model/unix-socket-fd.cc index 18a604d4..6b8c0c93 100644 --- a/model/unix-socket-fd.cc +++ b/model/unix-socket-fd.cc @@ -58,7 +58,8 @@ UnixSocketFd::UnixSocketFd (Ptr socket) : m_socket (socket), m_sendTimeout (Seconds (0.0)), m_recvTimeout (Seconds (0.0)), - m_peekedData (0) + m_peekedData (0), + m_icmp6FilterSet (false) { m_socket->SetRecvCallback (MakeCallback (&UnixSocketFd::RecvSocketData, this)); m_socket->SetSendCallback (MakeCallback (&UnixSocketFd::SendSocketData, this)); @@ -519,15 +520,20 @@ UnixSocketFd::Setsockopt (int level, int optname, { case ICMP6_FILTER: { + if (m_socket->GetInstanceTypeId ().GetName () != "ns3::Ipv6RawSocketImpl") + { + current->err = ENOPROTOOPT; + return -1; + } if (optlen != sizeof (struct icmp6_filter)) { current->err = EINVAL; return -1; } - // ns-3 does not expose its ICMPv6 raw socket filter; accept the - // option and deliver all ICMPv6 types (applications such as - // zebra check the type of every message they read). - NS_LOG_WARN ("ICMP6_FILTER accepted but not applied"); + // ns-3 does not expose its ICMPv6 raw socket filter: applied + // when receiving. + memcpy (&m_icmp6Filter, optval, sizeof (m_icmp6Filter)); + m_icmp6FilterSet = true; } break; default: NS_LOG_WARN ("Unsupported setsockopt requested. level: IPPROTO_ICMPV6, optname: " << optname); @@ -553,6 +559,22 @@ UnixSocketFd::Getsockopt (int level, int optname, switch (level) { + case IPPROTO_ICMPV6: + if (optname != ICMP6_FILTER || *optlen < (socklen_t) sizeof (struct icmp6_filter)) + { + current->err = optname != ICMP6_FILTER ? ENOPROTOOPT : EINVAL; + return -1; + } + if (m_icmp6FilterSet) + { + memcpy (optval, &m_icmp6Filter, sizeof (m_icmp6Filter)); + } + else + { + ICMP6_FILTER_SETPASSALL ((struct icmp6_filter *)optval); + } + *optlen = sizeof (struct icmp6_filter); + break; case SOL_RAW: switch (optname) { diff --git a/model/unix-socket-fd.h b/model/unix-socket-fd.h index f4f7b0f5..6f36bff5 100644 --- a/model/unix-socket-fd.h +++ b/model/unix-socket-fd.h @@ -28,6 +28,7 @@ #include #include #include +#include #include struct sockaddr_ll; @@ -100,6 +101,9 @@ class UnixSocketFd : public UnixFd Ptr m_peekedData; Address m_peekedAddress; + // ICMP6_FILTER: the ICMPv6 types a raw socket does not receive. + bool m_icmp6FilterSet; + struct icmp6_filter m_icmp6Filter; void AddPeekedData (const uint8_t *buf, uint32_t count, Address from); void AddPeekedData (Ptr p); diff --git a/test/dce-manager-test.cc b/test/dce-manager-test.cc index 2456b921..044a7ae7 100644 --- a/test/dce-manager-test.cc +++ b/test/dce-manager-test.cc @@ -228,6 +228,7 @@ DceManagerTestSuite::DceManagerTestSuite () { "test-ifindex", 30, "", true, false, NS3_MASK|LINUX_MASK}, { "test-emfile", 30, "", true, false, NS3_MASK|LINUX_MASK}, { "test-ipv6-pktinfo", 30, "", true, false, NS3_MASK|LINUX_MASK}, + { "test-icmp6-filter", 30, "", true, false, NS3_MASK|LINUX_MASK}, }; // Prepare directories and files for test-stdio diff --git a/test/test-icmp6-filter.cc b/test/test-icmp6-filter.cc new file mode 100644 index 00000000..839ebbbf --- /dev/null +++ b/test/test-icmp6-filter.cc @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include +#include +#include +#include "test-macros.h" + +// A raw ICMPv6 socket with ICMP6_FILTER only receives the types it lets +// pass: here the echo reply, not the echo request sent to ::1 before it. +int +main (int argc, char *argv[]) +{ + int sock = socket (AF_INET6, SOCK_RAW, IPPROTO_ICMPV6); + TEST_ASSERT (sock >= 0); + struct icmp6_filter filter; + ICMP6_FILTER_SETBLOCKALL (&filter); + ICMP6_FILTER_SETPASS (ICMP6_ECHO_REPLY, &filter); + TEST_ASSERT_EQUAL (setsockopt (sock, IPPROTO_ICMPV6, ICMP6_FILTER, &filter, sizeof (filter)), 0); + struct icmp6_filter got; + socklen_t len = sizeof (got); + TEST_ASSERT_EQUAL (getsockopt (sock, IPPROTO_ICMPV6, ICMP6_FILTER, &got, &len), 0); + TEST_ASSERT_EQUAL (memcmp (&got, &filter, sizeof (filter)), 0); + struct timeval timeout = { 1, 0 }; + TEST_ASSERT_EQUAL (setsockopt (sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof (timeout)), 0); + + struct sockaddr_in6 to; + memset (&to, 0, sizeof (to)); + to.sin6_family = AF_INET6; + to.sin6_addr = in6addr_loopback; + struct icmp6_hdr request; + memset (&request, 0, sizeof (request)); + request.icmp6_type = ICMP6_ECHO_REQUEST; + request.icmp6_id = htons (1); + request.icmp6_seq = htons (1); + TEST_ASSERT_EQUAL (sendto (sock, &request, sizeof (request), 0, (struct sockaddr *)&to, sizeof (to)), + (ssize_t)sizeof (request)); + + unsigned char buf[1500]; + ssize_t n = recv (sock, buf, sizeof (buf), 0); + TEST_ASSERT (n >= (ssize_t)sizeof (struct icmp6_hdr)); + TEST_ASSERT_EQUAL ((int)((struct icmp6_hdr *)buf)->icmp6_type, ICMP6_ECHO_REPLY); + + TEST_ASSERT_EQUAL (close (sock), 0); + return 0; +} diff --git a/wscript b/wscript index 59857b6f..5390a0b1 100755 --- a/wscript +++ b/wscript @@ -320,6 +320,7 @@ def build_dce_tests(module, bld): ['test-ifindex', []], ['test-emfile', []], ['test-ipv6-pktinfo', []], + ['test-icmp6-filter', []], ['test-clock-gettime', []], ['test-gcc-builtin-apply', []], ['test-iostream', []], From 44edf32d046c7e12e7b89f636ff14ac47423769f Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Sat, 10 Oct 2026 12:59:47 +0200 Subject: [PATCH 29/71] LinuxSocketImpl: retry a blocked send even if the socket does not wake After a send failed with EAGAIN, the poll loop notified the application again only once time went on, and then waited for the socket to wake: when the kernel had already reported the socket writable at the same time, nothing woke it, and the application stopped sending (DCCP bulk transfers ran at a fourth of their rate). Wait at most 1 ms then. dce-tcp-ns3-nsc-comparison now fails if a flow stalls or the flows use little of the bottleneck. Co-Authored-By: Claude Opus 5.5 --- example/dce-tcp-ns3-nsc-comparison.cc | 32 ++++++++++++++++++++++++++- model/linux/linux-socket-impl.cc | 8 ++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/example/dce-tcp-ns3-nsc-comparison.cc b/example/dce-tcp-ns3-nsc-comparison.cc index 60b56442..ce611dda 100644 --- a/example/dce-tcp-ns3-nsc-comparison.cc +++ b/example/dce-tcp-ns3-nsc-comparison.cc @@ -36,6 +36,15 @@ std::string sock_factory = "ns3::LinuxTcpSocketFactory"; int m_seed = 1; double startTime = 4.0; double stopTime = 20.0; + +// When each sink last received data: a flow that stalls stops early. +static std::vector