MDL-66476 mlbackend: Use F1 as the main accuracy metric

lib/mlbackend/php/classes/processor.php

@@ -31,6 +31,7 @@ use Phpml\CrossValidation\RandomSplit;
 use Phpml\Dataset\ArrayDataset;
 use Phpml\ModelManager;
 use Phpml\Classification\Linear\LogisticRegression;
+use Phpml\Metric\ClassificationReport;
 
 /**
  * PHP predictions processor.
@@ -309,7 +310,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
             return $resultobj;
         }
 
-        $phis = array();
+        $scores = array();
 
         // Evaluate the model multiple times to confirm the results are not significantly random due to a short amount of data.
         for ($i = 0; $i < $niterations; $i++) {
@@ -322,39 +323,43 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
 
                 $classifier->train($data->getTrainSamples(), $data->getTrainLabels());
                 $predictedlabels = $classifier->predict($data->getTestSamples());
-                $phis[] = $this->get_phi($data->getTestLabels(), $predictedlabels);
+                $report = new ClassificationReport($data->getTestLabels(), $predictedlabels,
+                    ClassificationReport::WEIGHTED_AVERAGE);
             } else {
                 $predictedlabels = $classifier->predict($samples);
-                $phis[] = $this->get_phi($targets, $predictedlabels);
+                $report = new ClassificationReport($targets, $predictedlabels,
+                    ClassificationReport::WEIGHTED_AVERAGE);
             }
+            $averages = $report->getAverage();
+            $scores[] = $averages['f1score'];
         }
 
         // Let's fill the results changing the returned status code depending on the phi-related calculated metrics.
-        return $this->get_evaluation_result_object($dataset, $phis, $maxdeviation);
+        return $this->get_evaluation_result_object($dataset, $scores, $maxdeviation);
     }
 
     /**
      * Returns the results objects from all evaluations.
      *
      * @param \stored_file $dataset
-     * @param array $phis
+     * @param array $scores
      * @param float $maxdeviation
      * @return \stdClass
      */
-    protected function get_evaluation_result_object(\stored_file $dataset, $phis, $maxdeviation) {
+    protected function get_evaluation_result_object(\stored_file $dataset, $scores, $maxdeviation) {
 
-        // Average phi of all evaluations as final score.
-        if (count($phis) === 1) {
-            $avgphi = reset($phis);
+        // Average f1 score of all evaluations as final score.
+        if (count($scores) === 1) {
+            $avgscore = reset($scores);
         } else {
-            $avgphi = \Phpml\Math\Statistic\Mean::arithmetic($phis);
+            $avgscore = \Phpml\Math\Statistic\Mean::arithmetic($scores);
         }
 
         // Standard deviation should ideally be calculated against the area under the curve.
-        if (count($phis) === 1) {
+        if (count($scores) === 1) {
             $modeldev = 0;
         } else {
-            $modeldev = \Phpml\Math\Statistic\StandardDeviation::population($phis);
+            $modeldev = \Phpml\Math\Statistic\StandardDeviation::population($scores);
         }
 
         // Let's fill the results object.
@@ -363,9 +368,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
         // Zero is ok, now we add other bits if something is not right.
         $resultobj->status = \core_analytics\model::OK;
         $resultobj->info = array();
-
-        // Convert phi to a standard score (from -1 to 1 to a value between 0 and 1).
-        $resultobj->score = ($avgphi + 1) / 2;
+        $resultobj->score = $avgscore;
 
         // If each iteration results varied too much we need more data to confirm that this is a valid model.
         if ($modeldev > $maxdeviation) {
@@ -523,33 +526,6 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
         return $modeldir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME;
     }
 
-    /**
-     * Returns the Phi correlation coefficient.
-     *
-     * @param array $testlabels
-     * @param array $predictedlabels
-     * @return float
-     */
-    protected function get_phi($testlabels, $predictedlabels) {
-
-        // Binary here only as well.
-        $matrix = \Phpml\Metric\ConfusionMatrix::compute($testlabels, $predictedlabels, array(0, 1));
-
-        $tptn = $matrix[0][0] * $matrix[1][1];
-        $fpfn = $matrix[1][0] * $matrix[0][1];
-        $tpfp = $matrix[0][0] + $matrix[1][0];
-        $tpfn = $matrix[0][0] + $matrix[0][1];
-        $tnfp = $matrix[1][1] + $matrix[1][0];
-        $tnfn = $matrix[1][1] + $matrix[0][1];
-        if ($tpfp === 0 || $tpfn === 0 || $tnfp === 0 || $tnfn === 0) {
-            $phi = 0;
-        } else {
-            $phi = ( $tptn - $fpfn ) / sqrt( $tpfp * $tpfn * $tnfp * $tnfn);
-        }
-
-        return $phi;
-    }
-
     /**
      * Extracts metadata from the dataset file.
      *

lib/mlbackend/php/upgrade.txt

+This files describes API changes in the mlbackend_php code, the
+information provided here is intended especially for developers.
+
+=== 3.8 ===
+
+* The phi coefficient (Matthews' correlation coefficient) has been replaced by
+  the F1 score as the main accuracy metric. Therefore, \mlbackend_php\processor::get_phi
+  method has been removed.

lib/mlbackend/python/classes/processor.php

@@ -38,7 +38,7 @@ class processor implements  \core_analytics\classifier, \core_analytics\regresso
     /**
      * The required version of the python package that performs all calculations.
      */
-    const REQUIRED_PIP_PACKAGE_VERSION = '2.2.1';
+    const REQUIRED_PIP_PACKAGE_VERSION = '2.3.0';
 
     /**
      * The python package is installed in a server.

lib/mlbackend/python/upgrade.txt

+This files describes API changes in the mlbackend_python code, the
+information provided here is intended especially for developers.
+
+=== 3.8 ===
+
+* The phi coefficient (Matthews' correlation coefficient) has been replaced by
+  the F1 score as the main accuracy metric.