New DCE applications

.github/workflows/build-and-test.yml

@@ -2,13 +2,18 @@ name: Build and Test DCE
 
 on:
   push:
-    branches: [ sikabo_master ]
+    branches: [ master ]
+  # Run for every pull request (merge-request pipeline), whatever the target branch
   pull_request:
-    branches: [ sikabo_master ]
   workflow_dispatch:
 
+# Cancel superseded runs of the same branch/PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
-  NS3_VER: "3.46"
+  NS3_VER: "3.49"
 
 jobs:
   build:
@@ -18,8 +23,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - distro: ubuntu-22.04
-            image: ubuntu:22.04
           - distro: ubuntu-24.04
             image: ubuntu:24.04
           - distro: debian-trixie
@@ -44,7 +47,44 @@ jobs:
             libsqlite3-dev \
             libxml2-dev \
             wget \
-            make
+            make \
+            bc \
+            flex \
+            bison \
+            byacc \
+            gawk \
+            file \
+            xz-utils \
+            libssl-dev \
+            libdb-dev \
+            libsctp-dev \
+            libx11-dev \
+            libxext-dev \
+            libxv-dev \
+            libpulse-dev \
+            libsdl2-dev \
+            libfltk1.3-dev \
+            libpng-dev \
+            libjpeg-dev \
+            autoconf \
+            automake \
+            libtool \
+            meson \
+            libgtk-4-dev \
+            libuchardet-dev \
+            libpsl-dev \
+            libseccomp-dev \
+            libharfbuzz-dev \
+            libfribidi-dev \
+            libcairo2-dev \
+            libfontconfig-dev \
+            libfreetype-dev \
+            xvfb \
+            dbus \
+            libdav1d-dev \
+            libvpx-dev \
+            libopenh264-dev \
+            libopus-dev
 
       - name: Configure git safe directory
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE/ns-3-dce"
@@ -88,24 +128,82 @@ jobs:
         working-directory: ns-${{ env.NS3_VER }}
         run: ./ns3 install
 
+      # DCE builds of ip, iperf, thttpd, wget, ping and quagga, and the
+      # ns-3-dce-quagga module.
+      - name: Cache kernel stack dependencies
+        id: cache-kernel-deps
+        uses: actions/cache@v4
+        with:
+          path: |
+            deps/bin_dce
+            deps/ns-3-dce-quagga
+          key: kernel-deps-${{ matrix.distro }}-${{ hashFiles('ns-3-dce/utils/build_kernel_deps.sh', 'ns-3-dce/utils/*.patch') }}
+
+      - name: Build kernel stack dependencies
+        if: steps.cache-kernel-deps.outputs.cache-hit != 'true'
+        working-directory: ns-3-dce
+        run: |
+          ./utils/build_kernel_deps.sh "$GITHUB_WORKSPACE/deps"
+          rm -rf "$GITHUB_WORKSPACE/deps/src"
+
+      # Linux 6.12 kernel library (LKL) and its headers.
+      - name: Cache LKL
+        id: cache-lkl
+        uses: actions/cache@v4
+        with:
+          path: deps/lkl
+          key: lkl-${{ matrix.distro }}-${{ hashFiles('ns-3-dce/utils/build_lkl.sh', 'ns-3-dce/utils/lkl-patches/*.patch', 'ns-3-dce/utils/lkl-dce.config') }}
+
+      - name: Build LKL
+        if: steps.cache-lkl.outputs.cache-hit != 'true'
+        working-directory: ns-3-dce
+        run: |
+          ./utils/build_lkl.sh "$GITHUB_WORKSPACE/lkl-src" "$GITHUB_WORKSPACE/deps/lkl"
+          rm -rf "$GITHUB_WORKSPACE/lkl-src" "$GITHUB_WORKSPACE/deps/lkl/liblkl.so.debug"
+
+      - name: Add Quagga module
+        run: cp -a deps/ns-3-dce-quagga ns-3-dce/myscripts/
+
       - name: Configure DCE
         working-directory: ns-3-dce
         run: |
           ./waf configure \
             --with-ns3="$GITHUB_WORKSPACE/build" \
             --prefix="$GITHUB_WORKSPACE/build" \
+            --with-lkl="$GITHUB_WORKSPACE/deps/lkl" \
             --enable-opt
 
       - name: Build DCE
         working-directory: ns-3-dce
         run: ./waf build
 
-      - name: Build bash for DCE
+      - name: Install DCE binaries
         working-directory: ns-3-dce
-        run: ./utils/build_bash.sh build/bin_dce
+        run: |
+          cp -a "$GITHUB_WORKSPACE/deps/bin_dce/." build/bin_dce/
+          cp "$GITHUB_WORKSPACE/deps/lkl/liblkl.so" build/bin_dce/
+          ./utils/build_bash.sh build/bin_dce
 
       - name: Run DCE tests
         working-directory: ns-3-dce
         env:
           LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib:${{ github.workspace }}/build/lib64
-        run: python3 test.py
+        # xvfb-run provides the X server used by dce-x11-hello and the
+        # browsers; dbus-run-session the session bus GApplication (Northstar)
+        # registers on.
+        run: xvfb-run -a dbus-run-session -- python3 test.py -r
+
+      # Sockets cannot be uploaded; symlinks would be followed (dce-browser
+      # links its browser's /usr and /etc to the host's).
+      - name: Remove sockets and symlinks from test output
+        if: always()
+        working-directory: ns-3-dce
+        run: if [ -d testpy-output ]; then find testpy-output \( -type s -o -type l \) -delete; fi
+
+      - name: Upload test output
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: testpy-output-${{ matrix.distro }}
+          path: ns-3-dce/testpy-output
+          if-no-files-found: ignore

.gitignore

@@ -13,3 +13,4 @@ GRTAGS
 GTAGS
 myscripts/ns-3-dce-quagga/
 myscripts/ns-3-dce-umip/
+testpy-output/

RELEASE_NOTES

@@ -5,6 +5,78 @@ This file contains Direct Code Execution release notes (most recent releases fir
 
 Please refer the page https://www.nsnam.org/projects/direct-code-execution/ for detailed instructions
 
+Changes since dce-1.12 (development, ns-3.49)
+==============================================
+
+New examples
+------------
+- dce-wifi-video: real ffmpeg streaming video over simulated 802.11n
+  (station - access point - station) on the Linux kernel stack or the ns-3
+  stack, watched live with a host player or in a window drawn from inside
+  the simulation; as a test, the received streams must be identical to the
+  sent ones. utils/build_kernel_deps.sh builds the DCE version of ffmpeg.
+- x11-hello, gtk4-hello and dce-x11-hello: X11 and GTK4 clients built for
+  DCE draw their windows on the host display.
+- dce-browser: a web browser on a Wi-Fi station browses a site served by
+  thttpd on another station, or the Internet through a host tap device
+  (utils/dce-tap-setup.sh): Dillo and Northstar (GTK4, JavaScript, sound)
+  built for DCE by utils/build_kernel_deps.sh, or unmodified prebuilt
+  Chromium and Brave, which play YouTube with sound. The Wi-Fi capture and
+  the browser's TLS key log let Wireshark decrypt the traffic. See
+  doc/source/dce-x11.rst.
+- build-and-run.sh builds everything in a container and runs the demos.
+
+New DCE features
+----------------
+- Host socket passthrough: the DceHostUnixSocketPaths global value lists
+  AF_UNIX path prefixes for which connect() reaches a real host socket.
+  DceX11Helper uses it for the host's X server, D-Bus session bus and
+  PulseAudio server.
+- eventfd(2), epoll(7), pipe2(2), futex(2) (through syscall(2)), SOCK_SEQPACKET,
+  connected socketpair(), descriptor passing (SCM_RIGHTS), FIONREAD/FIONBIO,
+  memfd_create, shm_open, gettid, /proc/self/exe, and several hundred more
+  libc entry points for ffmpeg, the X11 libraries, GLib/GIO/GTK4, Dillo,
+  Northstar and Chromium.
+- Programs built as executables (PIE), such as a prebuilt Chromium, run
+  unmodified: main() is found from the entry point, their thread local
+  variables get their own room (DCE_RESERVE_EXECUTABLE_TLS), their own
+  allocator is used by their libraries, and their references to old glibc
+  symbol versions bind to DCE.
+- TaskManager::PerFiberTls gives every task thread local storage of its
+  own (UcontextFiberManager, x86_64).
+- A program which crashes (SIGSEGV, SIGBUS, SIGILL, SIGFPE) ends its
+  process, as the kernel would, instead of the simulation.
+- DcePcapCheck reads ns-3 captures back (802.11 with A-MPDU, Ethernet, raw
+  IP) for examples that check their traffic.
+- Performance: libc functions are forwarded with a tail jump, the heap no
+  longer walks all its chunks, mutexes and condition variables are found
+  by id, and a zero-timeout poll() or epoll_wait() lets the simulated clock
+  catch up with the wall clock instead of spinning.
+- malloc() and friends return 16 byte aligned memory, like glibc.
+
+Bugs fixed
+----------
+- LKL: the kernel's re-armed timer events piled up in the simulator's
+  queue (gigabytes per minute with a busy kernel), and frames larger than
+  1500 bytes were truncated (every full-size Wi-Fi TCP segment).
+- pthread_cond_wait() could miss a wake-up, and a timed-out waiter could
+  take the wake-up of another thread: programs froze.
+- socketpair() did not connect its two ends; shutdown() did not wake a
+  blocked reader; a write to a closed peer returned 0 instead of EPIPE; a
+  blocking read waited for the whole buffer; sendmsg()/recvmsg() mishandled
+  their control data and errors.
+- fork() returned 0 in the parent with UcontextFiberManager; it now fails
+  with ENOSYS. PthreadFiberManager's switches aborted under _FORTIFY_SOURCE.
+- openat() relative to a directory ran a shell; F_DUPFD_CLOEXEC returned a
+  host descriptor; dlerror() was the host's; SIG_DFL and SIG_IGN were
+  called as handlers; exit() ran thread specific data destructors.
+- open64() passed garbage as the file creation mode; socket() with
+  SOCK_NONBLOCK/SOCK_CLOEXEC and setsockopt(SO_SNDBUF) on ns-3 UDP failed;
+  closing a pipe end whose peer was closed failed; realpath() leaked host
+  paths; the fortified __sprintf_chk() took the buffer size as the format
+  string, and wrote one character into a buffer of unknown size; a task
+  could be woken up after its process had exited.
+
 Release dce-1.12
 ================
 In order to use DCE, please use the bake tool to install DCE.  DCE 1.12

bindings/python/ns3modulegen_core_customizations.py

@@ -449,18 +449,16 @@ def add_ipv4_address_tp_hash(module):
 
 def post_register_types(root_module):
     remove_classes = [
-        # These are classes which get introduced when KERNEL_STACK is enabled , so should be removed for non-linux DCE builds
+        # The Linux stack (LKL) is not part of the Python bindings
         'KernelSocketFdFactory',
         'LinuxSocketImpl',
         'KernelSocketFd',
         'LinuxSocketFdFactory',
-        'FreeBSDSocketFdFactory'
     ]
 
-    if "KERNEL_STACK" not in os.environ:
-        for classname in remove_classes:
-            try:
-                root_module.classes.remove(root_module['ns3::%s' % classname])
-            except:
-                pass
+    for classname in remove_classes:
+        try:
+            root_module.classes.remove(root_module['ns3::%s' % classname])
+        except:
+            pass
 

bindings/python/wscript

@@ -584,9 +584,6 @@ def build(bld):
     if not env['ENABLE_PYTHON_BINDINGS']:
         return 
 
-    if bld.env['KERNEL_STACK']:
-        os.environ['KERNEL_STACK'] = bld.env['KERNEL_STACK']
-
 
     dce_generate_python_bindings(bld)