diff --git a/lib/ex_unit/lib/ex_unit/cli_formatter.ex b/lib/ex_unit/lib/ex_unit/cli_formatter.ex
index 807d4c4aaff..dd03351f15c 100644
--- a/lib/ex_unit/lib/ex_unit/cli_formatter.ex
+++ b/lib/ex_unit/lib/ex_unit/cli_formatter.ex
@@ -436,7 +436,13 @@ defmodule ExUnit.CLIFormatter do
       true -> IO.puts("\n")
     end
 
-    IO.puts(formatted)
+    formatted
+    |> ensure_printable()
+    |> IO.puts()
+  end
+
+  defp ensure_printable(binary) do
+    if String.valid?(binary), do: binary, else: String.replace_invalid(binary)
   end
 
   defp format_type_counts(type_counter) do
diff --git a/lib/ex_unit/test/ex_unit_test.exs b/lib/ex_unit/test/ex_unit_test.exs
index 3acd3436e24..12915601352 100644
--- a/lib/ex_unit/test/ex_unit_test.exs
+++ b/lib/ex_unit/test/ex_unit_test.exs
@@ -1281,4 +1281,27 @@ defmodule ExUnitTest do
   defp max_failures_reached_msg() do
     "--max-failures reached, aborting test suite"
   end
+
+  test "reports failures with invalid UTF-8 in messages" do
+    defmodule InvalidUtf8Test do
+      use ExUnit.Case
+
+      test "poisoned" do
+        flunk("frame bytes: " <> <<0xC3, 0x28, 0xFF>>)
+      end
+
+      test "clean" do
+        flunk("never silenced")
+      end
+    end
+
+    output =
+      capture_io(fn ->
+        assert ExUnit.run() == %{total: 2, excluded: 0, failures: 2, skipped: 0}
+      end)
+
+    assert String.valid?(output)
+    assert output =~ "frame bytes:"
+    assert output =~ "never silenced"
+  end
 end