diff --git a/backend/cmd/config.toml b/backend/cmd/config.toml index a54ecbb6e..fde463245 100644 --- a/backend/cmd/config.toml +++ b/backend/cmd/config.toml @@ -10,7 +10,7 @@ # Vehicle Configuration [vehicle] -boards = ["HVBMS","LCU", "PCU", "VCU", "BLCU"] +boards = ["HVBMS","LCU", "PCU", "VCU"] # ADJ (Architecture Description JSON) Configuration [adj] @@ -20,7 +20,6 @@ validate = true # Execute ADJ validator before starting backend # Transport Configuration [transport] propagate_fault = true - # TCP Configuration # These settings control how the backend reconnects to boards when connections are lost [tcp] diff --git a/backend/cmd/dev-config.toml b/backend/cmd/dev-config.toml index 47fcacad5..c236c5dab 100644 --- a/backend/cmd/dev-config.toml +++ b/backend/cmd/dev-config.toml @@ -13,7 +13,6 @@ # Vehicle Configuration [vehicle] boards = ["HVSCU", "HVSCU-Cabinet", "PCU", "LCU", "BCU", "BMSL"] - # ADJ (Architecture Description JSON) Configuration [adj] branch = "software" # Leave blank when using ADJ as a submodule (like this: "") diff --git a/backend/cmd/setup_transport.go b/backend/cmd/setup_transport.go index 22f10add7..9611b7cc1 100644 --- a/backend/cmd/setup_transport.go +++ b/backend/cmd/setup_transport.go @@ -147,7 +147,15 @@ func configureUDPServerTransport( if !ok { board = "unknown" } - transp.ReportError(fmt.Errorf("UDP keep-alive timeout: no packets received from board %s (%s) for %dms, fault sent", board, ip, config.UDP.KeepAliveTimeoutMs)) + err := fmt.Errorf("UDP keep-alive timeout: no packets received from board %s (%s) for %dms, fault sent", board, ip, config.UDP.KeepAliveTimeoutMs) + transp.ReportError(err) + + // Close the board's TCP connection ourselves: the fault we just + // broadcast leaves unacked data on a dead peer, which suppresses the + // TCP keep-alive and would delay disconnect detection by minutes. + if ok { + transp.DisconnectTarget(board, err) + } } udpServer := udp.NewServer( diff --git a/backend/pkg/transport/network/tcp/conn.go b/backend/pkg/transport/network/tcp/conn.go index 4786c9e5d..6dff6feb3 100644 --- a/backend/pkg/transport/network/tcp/conn.go +++ b/backend/pkg/transport/network/tcp/conn.go @@ -43,3 +43,17 @@ func (conn *connWithErr) Write(b []byte) (n int, err error) { func (conn *connWithErr) Close() error { return conn.Conn.Close() } + +// CloseWithError closes conn, delivering reason to its error channel first so +// the connection handler blocked on that channel wakes up. Closing a +// connection directly would not do this, since Read/Write filter out +// net.ErrClosed. +func CloseWithError(conn net.Conn, reason error) error { + if cwe, ok := conn.(*connWithErr); ok { + select { + case cwe.errors <- reason: + default: + } + } + return conn.Close() +} diff --git a/backend/pkg/transport/network/udp/server.go b/backend/pkg/transport/network/udp/server.go index 953d0a05d..7453e1278 100644 --- a/backend/pkg/transport/network/udp/server.go +++ b/backend/pkg/transport/network/udp/server.go @@ -182,7 +182,10 @@ func (s *Server) keepAliveLoop() { Dur("timeout", s.keepAliveTimeout). Msg("keep-alive timeout: no UDP packets received") if s.OnDisconnect != nil { - s.OnDisconnect(ip) + // Run the callback on its own goroutine so a slow + // handler (e.g. blocking TCP writes) cannot stall + // timeout detection for the remaining IPs + go s.OnDisconnect(ip) } } } diff --git a/backend/pkg/transport/transport.go b/backend/pkg/transport/transport.go index 1cc0b4f7c..7fb9c008c 100644 --- a/backend/pkg/transport/transport.go +++ b/backend/pkg/transport/transport.go @@ -224,6 +224,11 @@ func (transport *Transport) readLoopTCPConn(conn net.Conn, logger zerolog.Logger for { packet, err := transport.decoder.DecodeNext(conn) if err != nil { + // net.ErrClosed means we closed the connection on purpose + // (e.g. UDP keep-alive timeout), and a fault was already sent + if errors.Is(err, net.ErrClosed) { + return + } logger.Error().Stack().Err(err).Msg("decode") transport.errChan <- err transport.SendFault() @@ -268,6 +273,10 @@ func (transport *Transport) SendMessage(message abstraction.TransportMessage) er return err } +// faultWriteTimeout bounds each TCP write of the fault broadcast; boards on +// the vehicle LAN ack in milliseconds, so exceeding this means a dead peer +const faultWriteTimeout = time.Second + // handlePacketEvent is used to send an order to one of the connected boards func (transport *Transport) handlePacketEvent(message PacketMessage) error { eventLogger := transport.logger.With().Str("type", fmt.Sprintf("%T", message.Packet)).Uint16("id", uint16(message.Id())).Logger() @@ -287,17 +296,30 @@ func (transport *Transport) handlePacketEvent(message PacketMessage) error { defer transport.connectionsMx.RUnlock() for target, conn := range transport.connections { targetName := string(target) + + // Bound each write so a dead peer with a full send buffer cannot + // hold the connections lock (and the caller) for minutes + conn.SetWriteDeadline(time.Now().Add(faultWriteTimeout)) + var writeErr error totalWritten := 0 for totalWritten < len(data) { n, err := conn.Write(data[totalWritten:]) eventLogger.Trace().Str("target", targetName).Int("amount", n).Msg("written chunk") totalWritten += n if err != nil { - eventLogger.Error().Str("target", targetName).Stack().Err(err).Msg("write") - transport.errChan <- err - return err + writeErr = err + break } } + conn.SetWriteDeadline(time.Time{}) + + // Keep broadcasting to the remaining boards even if one write + // fails: the fault must reach every live board + if writeErr != nil { + eventLogger.Error().Str("target", targetName).Stack().Err(writeErr).Msg("write") + transport.errChan <- writeErr + continue + } eventLogger.Info().Str("target", targetName).Msg("sent") } return nil @@ -475,6 +497,22 @@ func (transport *Transport) TargetFromIp(ip string) (abstraction.TransportTarget return target, ok } +// DisconnectTarget forcefully closes the TCP connection to target, if any. +// The connection handler wakes up with reason, cleans up and notifies the +// disconnection, and the client reconnection loop takes over. +func (transport *Transport) DisconnectTarget(target abstraction.TransportTarget, reason error) bool { + transport.connectionsMx.RLock() + conn, ok := transport.connections[target] + transport.connectionsMx.RUnlock() + if !ok { + return false + } + + transport.logger.Warn().Str("target", string(target)).Err(reason).Msg("forcefully disconnecting target") + tcp.CloseWithError(conn, reason) + return true +} + func (transport *Transport) SendFault() { err := transport.SendMessage(NewPacketMessage(data.NewPacket(0))) if err != nil { diff --git a/electron-app/preload.js b/electron-app/preload.js index 3dce66219..0ed097df6 100644 --- a/electron-app/preload.js +++ b/electron-app/preload.js @@ -48,6 +48,8 @@ contextBridge.exposeInMainWorld("electronAPI", { restartBackend: () => ipcRenderer.invoke("restart-backend"), // Get the list of views available in this build getAvailableViews: () => ipcRenderer.invoke("get-available-views"), + // Run the kernel setup script (used by mode selector renderer) + setupKernel: () => ipcRenderer.invoke("setup-kernel"), // Set initial mode (used by mode selector renderer) setInitialMode: (mode) => { ipcRenderer.send("mode-selected", mode); diff --git a/electron-app/renderer/mode-selector/index.html b/electron-app/renderer/mode-selector/index.html index b7f2bebed..6fa83e8e2 100644 --- a/electron-app/renderer/mode-selector/index.html +++ b/electron-app/renderer/mode-selector/index.html @@ -178,6 +178,38 @@ box-shadow: inset 0 0 0 1px rgba(237, 238, 240, 0.04); } + .kernel-setup { + margin-top: 16px; + display: grid; + gap: 8px; + justify-items: center; + } + + .kernel-setup button { + min-height: 44px; + padding: 10px 24px; + } + + .kernel-setup button:disabled { + opacity: 0.5; + cursor: wait; + transform: none; + } + + .kernel-status { + font-size: 0.85rem; + min-height: 1.2em; + opacity: 0.85; + } + + .kernel-status.ok { + color: #7ee787; + } + + .kernel-status.error { + color: #ff8080; + } + @media (max-width: 420px) { .card { padding: 20px; @@ -209,6 +241,10 @@