From 6d060fffa35b667000b7ac2b92c6b28bd17cbd10 Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Sat, 14 Nov 2026 12:54:30 -0800 Subject: [PATCH 1/2] Use a Unix domain socketpair for TriggerFd on non-Linux POSIX TriggerFd used a pipe on every platform without eventfd. On QNX, pipes are served by the single-threaded pipe resource manager, which makes them slow under load. A socketpair is pollable, nonblocking, and can be passed over SCM_RIGHTS, so it is a drop-in replacement for the pipe on macOS, QNX, and other POSIX systems. Linux keeps using eventfd. Add triggerfd_test covering create, trigger/clear, repeated triggers, non-blocking behaviour when the buffer is full, and move semantics. --- README.md | 2 +- toolbelt/BUILD.bazel | 12 +++++++ toolbelt/triggerfd.cc | 38 ++++++++++++---------- toolbelt/triggerfd.h | 2 +- toolbelt/triggerfd_test.cc | 66 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 19 deletions(-) create mode 100644 toolbelt/triggerfd_test.cc diff --git a/README.md b/README.md index 7254b91..9fbe136 100644 --- a/README.md +++ b/README.md @@ -913,7 +913,7 @@ int value = resizable->VectorGet(&vec_hdr, 0); // Returns 42 ### TriggerFd -A file descriptor that can be used to trigger events. Implemented as an `eventfd` on Linux or a pipe on other systems. +A file descriptor that can be used to trigger events. Implemented as an `eventfd` on Linux and a Unix domain `socketpair` on other POSIX systems, including macOS and QNX. **Header:** `toolbelt/triggerfd.h` diff --git a/toolbelt/BUILD.bazel b/toolbelt/BUILD.bazel index 0c9da20..f4786a9 100644 --- a/toolbelt/BUILD.bazel +++ b/toolbelt/BUILD.bazel @@ -81,6 +81,18 @@ cc_test( "@googletest//:gtest_main", ], ) +cc_test( + name = "triggerfd_test", + size = "small", + srcs = ["triggerfd_test.cc"], + deps = [ + ":toolbelt", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@googletest//:gtest_main", + ], +) + cc_test( name = "pipe_test", size = "medium", diff --git a/toolbelt/triggerfd.cc b/toolbelt/triggerfd.cc index fd9a835..034a788 100644 --- a/toolbelt/triggerfd.cc +++ b/toolbelt/triggerfd.cc @@ -5,11 +5,17 @@ #include "toolbelt/triggerfd.h" #include "absl/strings/str_format.h" +// Linux has eventfd, which is the cheapest pollable trigger. Everywhere else +// (macOS, QNX, other POSIX) use a Unix domain socketpair. Pipes are a poor +// fit: on QNX they go through a single-threaded pipe service, and a socketpair +// is pollable, nonblocking, and passable over SCM_RIGHTS on every POSIX system +// we care about. +#include #if defined(__linux__) #include #else -#include -#include +#include +#include #endif namespace toolbelt { @@ -41,22 +47,20 @@ absl::Status TriggerFd::Open() { poll_fd_.SetFd(fd); trigger_fd_ = poll_fd_; #else - int pipes[2]; - if (pipe(pipes) == -1) { + int fds[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) { return absl::InternalError(absl::StrFormat( - "Unable to create trigger fd pipe: %s", strerror(errno))); + "Unable to create trigger fd socketpair: %s", strerror(errno))); } - // Both ends of the pipe are nonblocking. - if (fcntl(pipes[0], F_SETFL, (fcntl(pipes[0], F_GETFL) | O_NONBLOCK)) == -1) { - return absl::InternalError(absl::StrFormat( - "Unable to set non-blocking on read pipe: %s", strerror(errno))); + // Owned immediately so both ends are closed if setting nonblocking fails. + poll_fd_.SetFd(fds[0]); + trigger_fd_.SetFd(fds[1]); + if (absl::Status status = poll_fd_.SetNonBlocking(); !status.ok()) { + return status; } - if (fcntl(pipes[1], F_SETFL, (fcntl(pipes[1], F_GETFL) | O_NONBLOCK)) == -1) { - return absl::InternalError(absl::StrFormat( - "Unable to set non-blocking on write pipe: %s", strerror(errno))); + if (absl::Status status = trigger_fd_.SetNonBlocking(); !status.ok()) { + return status; } - poll_fd_.SetFd(pipes[0]); // Read end - trigger_fd_.SetFd(pipes[1]); // Write end. #endif return absl::OkStatus(); } @@ -67,7 +71,7 @@ void TriggerFd::Trigger() { int64_t val = 1; (void)::write(trigger_fd_.Fd(), &val, 8); #else - // Pipe. Write a single byte to it. + // Unix domain socket. Write a single byte to it. char val = 'x'; (void)::write(trigger_fd_.Fd(), &val, 1); #endif @@ -81,8 +85,8 @@ bool TriggerFd::Clear() { ssize_t n = ::read(poll_fd_.Fd(), &val, 8); return (n > 0); #else - // Pipe. Read all bytes from the pipe. The pipe is nonblocking so we will - // get an EAGAIN when we've cleared it. + // Unix domain socket. Read all pending bytes. The socket is nonblocking + // so we get EAGAIN once it has been drained. bool was_triggered = false; for (;;) { char buffer[256]; diff --git a/toolbelt/triggerfd.h b/toolbelt/triggerfd.h index b248e7d..d99a111 100644 --- a/toolbelt/triggerfd.h +++ b/toolbelt/triggerfd.h @@ -13,7 +13,7 @@ namespace toolbelt { // A TriggerFd is a file descriptor that can be used to trigger // an event. If the facility is available, it is implemented -// as an eventfd, otherwise it's a pipe. +// as an eventfd on Linux, otherwise a Unix domain socketpair. class TriggerFd { public: TriggerFd() = default; diff --git a/toolbelt/triggerfd_test.cc b/toolbelt/triggerfd_test.cc new file mode 100644 index 0000000..95f3d90 --- /dev/null +++ b/toolbelt/triggerfd_test.cc @@ -0,0 +1,66 @@ +// Copyright 2023 David Allison +// All Rights Reserved +// See LICENSE file for licensing information. + +#include "toolbelt/triggerfd.h" +#include +#include + +namespace { + +bool Readable(toolbelt::TriggerFd &t) { + struct pollfd pfd = t.GetPollFd().GetPollFd(); + return ::poll(&pfd, 1, 0) == 1 && (pfd.revents & POLLIN) != 0; +} + +TEST(TriggerFdTest, CreateAndClearUntriggered) { + auto t = toolbelt::TriggerFd::Create(); + ASSERT_TRUE(t.ok()); + ASSERT_TRUE(t->GetPollFd().Valid()); + ASSERT_TRUE(t->GetTriggerFd().Valid()); + ASSERT_FALSE(Readable(*t)); + ASSERT_FALSE(t->Clear()); +} + +TEST(TriggerFdTest, TriggerThenClear) { + auto t = toolbelt::TriggerFd::Create(); + ASSERT_TRUE(t.ok()); + t->Trigger(); + ASSERT_TRUE(Readable(*t)); + ASSERT_TRUE(t->Clear()); + ASSERT_FALSE(Readable(*t)); + ASSERT_FALSE(t->Clear()); +} + +TEST(TriggerFdTest, MultipleTriggersClearedAtOnce) { + auto t = toolbelt::TriggerFd::Create(); + ASSERT_TRUE(t.ok()); + for (int i = 0; i < 10; i++) { + t->Trigger(); + } + ASSERT_TRUE(t->Clear()); + ASSERT_FALSE(Readable(*t)); +} + +TEST(TriggerFdTest, TriggerNeverBlocksWhenFull) { + auto t = toolbelt::TriggerFd::Create(); + ASSERT_TRUE(t.ok()); + // Far more triggers than any pipe/socket buffer can hold. The trigger fd is + // nonblocking so this must not hang. + for (int i = 0; i < 1000000; i++) { + t->Trigger(); + } + ASSERT_TRUE(t->Clear()); + ASSERT_FALSE(Readable(*t)); +} + +TEST(TriggerFdTest, Move) { + auto t = toolbelt::TriggerFd::Create(); + ASSERT_TRUE(t.ok()); + toolbelt::TriggerFd moved = std::move(*t); + ASSERT_FALSE(t->GetPollFd().Valid()); + moved.Trigger(); + ASSERT_TRUE(moved.Clear()); +} + +} // namespace From 10e2b042cb0e4602b8362d50612abf3c8b681ecc Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Sat, 14 Nov 2026 23:16:26 -0800 Subject: [PATCH 2/2] Use a socketpair for TriggerFd only on QNX. QNX pipes go through a single-threaded pipe service. On macOS a socketpair was slower than a pipe, especially when a publish wakes another thread, and its read end always reports POLLOUT. Linux stays on eventfd and other POSIX systems stay on a pipe. --- README.md | 2 +- toolbelt/triggerfd.cc | 38 ++++++++++++++++++++++++++++---------- toolbelt/triggerfd.h | 3 ++- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9fbe136..1b3d7ee 100644 --- a/README.md +++ b/README.md @@ -913,7 +913,7 @@ int value = resizable->VectorGet(&vec_hdr, 0); // Returns 42 ### TriggerFd -A file descriptor that can be used to trigger events. Implemented as an `eventfd` on Linux and a Unix domain `socketpair` on other POSIX systems, including macOS and QNX. +A file descriptor that can be used to trigger events. Implemented as an `eventfd` on Linux, a Unix domain `socketpair` on QNX, and a pipe on other POSIX systems, including macOS. QNX pipes go through a single-threaded pipe service, and a socketpair is slower than a pipe on macOS. **Header:** `toolbelt/triggerfd.h` diff --git a/toolbelt/triggerfd.cc b/toolbelt/triggerfd.cc index 034a788..4150551 100644 --- a/toolbelt/triggerfd.cc +++ b/toolbelt/triggerfd.cc @@ -5,17 +5,18 @@ #include "toolbelt/triggerfd.h" #include "absl/strings/str_format.h" -// Linux has eventfd, which is the cheapest pollable trigger. Everywhere else -// (macOS, QNX, other POSIX) use a Unix domain socketpair. Pipes are a poor -// fit: on QNX they go through a single-threaded pipe service, and a socketpair -// is pollable, nonblocking, and passable over SCM_RIGHTS on every POSIX system -// we care about. +// Linux has eventfd, which is the cheapest pollable trigger. QNX pipes go +// through a single-threaded pipe service, so QNX uses a Unix domain socketpair. +// Every other POSIX system, including macOS, keeps the pipe. A socketpair is +// slower there, and its read end always reports POLLOUT. #include #if defined(__linux__) #include -#else +#elif defined(__QNX__) || defined(__QNXNTO__) #include #include +#else +#include #endif namespace toolbelt { @@ -46,7 +47,7 @@ absl::Status TriggerFd::Open() { } poll_fd_.SetFd(fd); trigger_fd_ = poll_fd_; -#else +#elif defined(__QNX__) || defined(__QNXNTO__) int fds[2]; if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) { return absl::InternalError(absl::StrFormat( @@ -61,6 +62,23 @@ absl::Status TriggerFd::Open() { if (absl::Status status = trigger_fd_.SetNonBlocking(); !status.ok()) { return status; } +#else + int pipes[2]; + if (pipe(pipes) == -1) { + return absl::InternalError(absl::StrFormat( + "Unable to create trigger fd pipe: %s", strerror(errno))); + } + // Both ends of the pipe are nonblocking. + if (fcntl(pipes[0], F_SETFL, (fcntl(pipes[0], F_GETFL) | O_NONBLOCK)) == -1) { + return absl::InternalError(absl::StrFormat( + "Unable to set non-blocking on read pipe: %s", strerror(errno))); + } + if (fcntl(pipes[1], F_SETFL, (fcntl(pipes[1], F_GETFL) | O_NONBLOCK)) == -1) { + return absl::InternalError(absl::StrFormat( + "Unable to set non-blocking on write pipe: %s", strerror(errno))); + } + poll_fd_.SetFd(pipes[0]); // Read end + trigger_fd_.SetFd(pipes[1]); // Write end. #endif return absl::OkStatus(); } @@ -71,7 +89,7 @@ void TriggerFd::Trigger() { int64_t val = 1; (void)::write(trigger_fd_.Fd(), &val, 8); #else - // Unix domain socket. Write a single byte to it. + // Pipe, or a socketpair on QNX. Write a single byte to it. char val = 'x'; (void)::write(trigger_fd_.Fd(), &val, 1); #endif @@ -85,8 +103,8 @@ bool TriggerFd::Clear() { ssize_t n = ::read(poll_fd_.Fd(), &val, 8); return (n > 0); #else - // Unix domain socket. Read all pending bytes. The socket is nonblocking - // so we get EAGAIN once it has been drained. + // Pipe, or a socketpair on QNX. Read all pending bytes. The fd is + // nonblocking, so we get EAGAIN once it has been drained. bool was_triggered = false; for (;;) { char buffer[256]; diff --git a/toolbelt/triggerfd.h b/toolbelt/triggerfd.h index d99a111..aefced1 100644 --- a/toolbelt/triggerfd.h +++ b/toolbelt/triggerfd.h @@ -13,7 +13,8 @@ namespace toolbelt { // A TriggerFd is a file descriptor that can be used to trigger // an event. If the facility is available, it is implemented -// as an eventfd on Linux, otherwise a Unix domain socketpair. +// as an eventfd on Linux, a Unix domain socketpair on QNX, and a pipe +// everywhere else. class TriggerFd { public: TriggerFd() = default;