Commit f5c65ba3 authored by David Monllaó's avatar David Monllaó
Browse files

MDL-66476 mlbackend: Use F1 as the main accuracy metric

parent ebec727c
...@@ -31,6 +31,7 @@ use Phpml\CrossValidation\RandomSplit; ...@@ -31,6 +31,7 @@ use Phpml\CrossValidation\RandomSplit;
use Phpml\Dataset\ArrayDataset; use Phpml\Dataset\ArrayDataset;
use Phpml\ModelManager; use Phpml\ModelManager;
use Phpml\Classification\Linear\LogisticRegression; use Phpml\Classification\Linear\LogisticRegression;
use Phpml\Metric\ClassificationReport;
/** /**
* PHP predictions processor. * PHP predictions processor.
...@@ -309,7 +310,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor ...@@ -309,7 +310,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
return $resultobj; return $resultobj;
} }
$phis = array(); $scores = array();
// Evaluate the model multiple times to confirm the results are not significantly random due to a short amount of data. // Evaluate the model multiple times to confirm the results are not significantly random due to a short amount of data.
for ($i = 0; $i < $niterations; $i++) { for ($i = 0; $i < $niterations; $i++) {
...@@ -322,39 +323,43 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor ...@@ -322,39 +323,43 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
$classifier->train($data->getTrainSamples(), $data->getTrainLabels()); $classifier->train($data->getTrainSamples(), $data->getTrainLabels());
$predictedlabels = $classifier->predict($data->getTestSamples()); $predictedlabels = $classifier->predict($data->getTestSamples());
$phis[] = $this->get_phi($data->getTestLabels(), $predictedlabels); $report = new ClassificationReport($data->getTestLabels(), $predictedlabels,
ClassificationReport::WEIGHTED_AVERAGE);
} else { } else {
$predictedlabels = $classifier->predict($samples); $predictedlabels = $classifier->predict($samples);
$phis[] = $this->get_phi($targets, $predictedlabels); $report = new ClassificationReport($targets, $predictedlabels,
ClassificationReport::WEIGHTED_AVERAGE);
} }
$averages = $report->getAverage();
$scores[] = $averages['f1score'];
} }
// Let's fill the results changing the returned status code depending on the phi-related calculated metrics. // Let's fill the results changing the returned status code depending on the phi-related calculated metrics.
return $this->get_evaluation_result_object($dataset, $phis, $maxdeviation); return $this->get_evaluation_result_object($dataset, $scores, $maxdeviation);
} }
/** /**
* Returns the results objects from all evaluations. * Returns the results objects from all evaluations.
* *
* @param \stored_file $dataset * @param \stored_file $dataset
* @param array $phis * @param array $scores
* @param float $maxdeviation * @param float $maxdeviation
* @return \stdClass * @return \stdClass
*/ */
protected function get_evaluation_result_object(\stored_file $dataset, $phis, $maxdeviation) { protected function get_evaluation_result_object(\stored_file $dataset, $scores, $maxdeviation) {
// Average phi of all evaluations as final score. // Average f1 score of all evaluations as final score.
if (count($phis) === 1) { if (count($scores) === 1) {
$avgphi = reset($phis); $avgscore = reset($scores);
} else { } else {
$avgphi = \Phpml\Math\Statistic\Mean::arithmetic($phis); $avgscore = \Phpml\Math\Statistic\Mean::arithmetic($scores);
} }
// Standard deviation should ideally be calculated against the area under the curve. // Standard deviation should ideally be calculated against the area under the curve.
if (count($phis) === 1) { if (count($scores) === 1) {
$modeldev = 0; $modeldev = 0;
} else { } else {
$modeldev = \Phpml\Math\Statistic\StandardDeviation::population($phis); $modeldev = \Phpml\Math\Statistic\StandardDeviation::population($scores);
} }
// Let's fill the results object. // Let's fill the results object.
...@@ -363,9 +368,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor ...@@ -363,9 +368,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
// Zero is ok, now we add other bits if something is not right. // Zero is ok, now we add other bits if something is not right.
$resultobj->status = \core_analytics\model::OK; $resultobj->status = \core_analytics\model::OK;
$resultobj->info = array(); $resultobj->info = array();
$resultobj->score = $avgscore;
// Convert phi to a standard score (from -1 to 1 to a value between 0 and 1).
$resultobj->score = ($avgphi + 1) / 2;
// If each iteration results varied too much we need more data to confirm that this is a valid model. // If each iteration results varied too much we need more data to confirm that this is a valid model.
if ($modeldev > $maxdeviation) { if ($modeldev > $maxdeviation) {
...@@ -523,33 +526,6 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor ...@@ -523,33 +526,6 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
return $modeldir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME; return $modeldir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME;
} }
/**
* Returns the Phi correlation coefficient.
*
* @param array $testlabels
* @param array $predictedlabels
* @return float
*/
protected function get_phi($testlabels, $predictedlabels) {
// Binary here only as well.
$matrix = \Phpml\Metric\ConfusionMatrix::compute($testlabels, $predictedlabels, array(0, 1));
$tptn = $matrix[0][0] * $matrix[1][1];
$fpfn = $matrix[1][0] * $matrix[0][1];
$tpfp = $matrix[0][0] + $matrix[1][0];
$tpfn = $matrix[0][0] + $matrix[0][1];
$tnfp = $matrix[1][1] + $matrix[1][0];
$tnfn = $matrix[1][1] + $matrix[0][1];
if ($tpfp === 0 || $tpfn === 0 || $tnfp === 0 || $tnfn === 0) {
$phi = 0;
} else {
$phi = ( $tptn - $fpfn ) / sqrt( $tpfp * $tpfn * $tnfp * $tnfn);
}
return $phi;
}
/** /**
* Extracts metadata from the dataset file. * Extracts metadata from the dataset file.
* *
......
This files describes API changes in the mlbackend_php code, the
information provided here is intended especially for developers.
=== 3.8 ===
* The phi coefficient (Matthews' correlation coefficient) has been replaced by
the F1 score as the main accuracy metric. Therefore, \mlbackend_php\processor::get_phi
method has been removed.
...@@ -38,7 +38,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regresso ...@@ -38,7 +38,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regresso
/** /**
* The required version of the python package that performs all calculations. * The required version of the python package that performs all calculations.
*/ */
const REQUIRED_PIP_PACKAGE_VERSION = '2.2.1'; const REQUIRED_PIP_PACKAGE_VERSION = '2.3.0';
/** /**
* The python package is installed in a server. * The python package is installed in a server.
......
This files describes API changes in the mlbackend_python code, the
information provided here is intended especially for developers.
=== 3.8 ===
* The phi coefficient (Matthews' correlation coefficient) has been replaced by
the F1 score as the main accuracy metric.
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment