diff --git a/CHANGELOG.md b/CHANGELOG.md index 343699951..5270fd434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Add numerical stability to logsumexp function - Fix Mean Shift delta hyper-parameter - Fix MAPE calculation in Error Analysis report + - Fix Cross Entropy loss missing (1−t)·log(1−p) term - Fix SiLU derivative at zero - Remove spurious +1 in Relative Entropy gradient - SoftPlus derivative expressed in terms of input diff --git a/src/NeuralNet/CostFunctions/CrossEntropy.php b/src/NeuralNet/CostFunctions/CrossEntropy.php index 4e7527f1c..618dcff78 100644 --- a/src/NeuralNet/CostFunctions/CrossEntropy.php +++ b/src/NeuralNet/CostFunctions/CrossEntropy.php @@ -33,9 +33,14 @@ class CrossEntropy implements ClassificationLoss */ public function compute(Matrix $output, Matrix $target) : float { - $entropy = $output->clipLower(EPSILON)->log(); + $output = $output->clip(EPSILON, 1.0 - EPSILON); - return $target->negate()->multiply($entropy)->mean()->mean(); + $ones = Matrix::ones(...$output->shape()); + + $entropy = $output->log()->multiply($target) + ->add($ones->subtract($output)->log()->multiply($ones->subtract($target))); + + return $entropy->negate()->mean()->mean(); } /** diff --git a/tests/NeuralNet/CostFunctions/CrossEntropyTest.php b/tests/NeuralNet/CostFunctions/CrossEntropyTest.php index 1188b1c0f..bb80defd5 100644 --- a/tests/NeuralNet/CostFunctions/CrossEntropyTest.php +++ b/tests/NeuralNet/CostFunctions/CrossEntropyTest.php @@ -63,7 +63,7 @@ public function computeProvider() : Generator Matrix::quick([ [1.0, 0.0, 0.0], ]), - 0.00335011195116715, + 0.006700227235667666, ]; yield [ @@ -73,7 +73,7 @@ public function computeProvider() : Generator Matrix::quick([ [0.0, 1.0, 0.0], ]), - 0.3054302439580517, + 0.5500866356514519, ]; yield [ @@ -83,7 +83,7 @@ public function computeProvider() : Generator Matrix::quick([ [1.0, 0.0, 0.0], ]), - 6.140226914650789, + 6.94287545086808, ]; yield [ @@ -97,7 +97,7 @@ public function computeProvider() : Generator [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], ]), - 0.10809567592917217, + 0.20764012617655228, ]; }