Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
moodle
moodle
Commits
f5c65ba3
Commit
f5c65ba3
authored
Oct 01, 2019
by
David Monllaó
Browse files
MDL-66476 mlbackend: Use F1 as the main accuracy metric
parent
ebec727c
Changes
4
Hide whitespace changes
Inline
Side-by-side
lib/mlbackend/php/classes/processor.php
View file @
f5c65ba3
...
...
@@ -31,6 +31,7 @@ use Phpml\CrossValidation\RandomSplit;
use
Phpml\Dataset\ArrayDataset
;
use
Phpml\ModelManager
;
use
Phpml\Classification\Linear\LogisticRegression
;
use
Phpml\Metric\ClassificationReport
;
/**
* PHP predictions processor.
...
...
@@ -309,7 +310,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
return
$resultobj
;
}
$
phi
s
=
array
();
$
score
s
=
array
();
// Evaluate the model multiple times to confirm the results are not significantly random due to a short amount of data.
for
(
$i
=
0
;
$i
<
$niterations
;
$i
++
)
{
...
...
@@ -322,39 +323,43 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
$classifier
->
train
(
$data
->
getTrainSamples
(),
$data
->
getTrainLabels
());
$predictedlabels
=
$classifier
->
predict
(
$data
->
getTestSamples
());
$phis
[]
=
$this
->
get_phi
(
$data
->
getTestLabels
(),
$predictedlabels
);
$report
=
new
ClassificationReport
(
$data
->
getTestLabels
(),
$predictedlabels
,
ClassificationReport
::
WEIGHTED_AVERAGE
);
}
else
{
$predictedlabels
=
$classifier
->
predict
(
$samples
);
$phis
[]
=
$this
->
get_phi
(
$targets
,
$predictedlabels
);
$report
=
new
ClassificationReport
(
$targets
,
$predictedlabels
,
ClassificationReport
::
WEIGHTED_AVERAGE
);
}
$averages
=
$report
->
getAverage
();
$scores
[]
=
$averages
[
'f1score'
];
}
// Let's fill the results changing the returned status code depending on the phi-related calculated metrics.
return
$this
->
get_evaluation_result_object
(
$dataset
,
$
phi
s
,
$maxdeviation
);
return
$this
->
get_evaluation_result_object
(
$dataset
,
$
score
s
,
$maxdeviation
);
}
/**
* Returns the results objects from all evaluations.
*
* @param \stored_file $dataset
* @param array $
phi
s
* @param array $
score
s
* @param float $maxdeviation
* @return \stdClass
*/
protected
function
get_evaluation_result_object
(
\
stored_file
$dataset
,
$
phi
s
,
$maxdeviation
)
{
protected
function
get_evaluation_result_object
(
\
stored_file
$dataset
,
$
score
s
,
$maxdeviation
)
{
// Average
phi
of all evaluations as final score.
if
(
count
(
$
phi
s
)
===
1
)
{
$avg
phi
=
reset
(
$
phi
s
);
// Average
f1 score
of all evaluations as final score.
if
(
count
(
$
score
s
)
===
1
)
{
$avg
score
=
reset
(
$
score
s
);
}
else
{
$avg
phi
=
\
Phpml\Math\Statistic\Mean
::
arithmetic
(
$
phi
s
);
$avg
score
=
\
Phpml\Math\Statistic\Mean
::
arithmetic
(
$
score
s
);
}
// Standard deviation should ideally be calculated against the area under the curve.
if
(
count
(
$
phi
s
)
===
1
)
{
if
(
count
(
$
score
s
)
===
1
)
{
$modeldev
=
0
;
}
else
{
$modeldev
=
\
Phpml\Math\Statistic\StandardDeviation
::
population
(
$
phi
s
);
$modeldev
=
\
Phpml\Math\Statistic\StandardDeviation
::
population
(
$
score
s
);
}
// Let's fill the results object.
...
...
@@ -363,9 +368,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
// Zero is ok, now we add other bits if something is not right.
$resultobj
->
status
=
\
core_analytics\model
::
OK
;
$resultobj
->
info
=
array
();
// Convert phi to a standard score (from -1 to 1 to a value between 0 and 1).
$resultobj
->
score
=
(
$avgphi
+
1
)
/
2
;
$resultobj
->
score
=
$avgscore
;
// If each iteration results varied too much we need more data to confirm that this is a valid model.
if
(
$modeldev
>
$maxdeviation
)
{
...
...
@@ -523,33 +526,6 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
return
$modeldir
.
DIRECTORY_SEPARATOR
.
self
::
MODEL_FILENAME
;
}
/**
* Returns the Phi correlation coefficient.
*
* @param array $testlabels
* @param array $predictedlabels
* @return float
*/
protected
function
get_phi
(
$testlabels
,
$predictedlabels
)
{
// Binary here only as well.
$matrix
=
\
Phpml\Metric\ConfusionMatrix
::
compute
(
$testlabels
,
$predictedlabels
,
array
(
0
,
1
));
$tptn
=
$matrix
[
0
][
0
]
*
$matrix
[
1
][
1
];
$fpfn
=
$matrix
[
1
][
0
]
*
$matrix
[
0
][
1
];
$tpfp
=
$matrix
[
0
][
0
]
+
$matrix
[
1
][
0
];
$tpfn
=
$matrix
[
0
][
0
]
+
$matrix
[
0
][
1
];
$tnfp
=
$matrix
[
1
][
1
]
+
$matrix
[
1
][
0
];
$tnfn
=
$matrix
[
1
][
1
]
+
$matrix
[
0
][
1
];
if
(
$tpfp
===
0
||
$tpfn
===
0
||
$tnfp
===
0
||
$tnfn
===
0
)
{
$phi
=
0
;
}
else
{
$phi
=
(
$tptn
-
$fpfn
)
/
sqrt
(
$tpfp
*
$tpfn
*
$tnfp
*
$tnfn
);
}
return
$phi
;
}
/**
* Extracts metadata from the dataset file.
*
...
...
lib/mlbackend/php/upgrade.txt
0 → 100644
View file @
f5c65ba3
This files describes API changes in the mlbackend_php code, the
information provided here is intended especially for developers.
=== 3.8 ===
* The phi coefficient (Matthews' correlation coefficient) has been replaced by
the F1 score as the main accuracy metric. Therefore, \mlbackend_php\processor::get_phi
method has been removed.
lib/mlbackend/python/classes/processor.php
View file @
f5c65ba3
...
...
@@ -38,7 +38,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regresso
/**
* The required version of the python package that performs all calculations.
*/
const
REQUIRED_PIP_PACKAGE_VERSION
=
'2.
2.1
'
;
const
REQUIRED_PIP_PACKAGE_VERSION
=
'2.
3.0
'
;
/**
* The python package is installed in a server.
...
...
lib/mlbackend/python/upgrade.txt
0 → 100644
View file @
f5c65ba3
This files describes API changes in the mlbackend_python code, the
information provided here is intended especially for developers.
=== 3.8 ===
* The phi coefficient (Matthews' correlation coefficient) has been replaced by
the F1 score as the main accuracy metric.
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment