Commit 5c5cb3ee authored by David Monllaó's avatar David Monllaó
Browse files

MDL-59265 analytics: Rename machine learning backend method

- Method names renamed to avoid interface changes once
  we support regression and unsupervised learning
- Adding regressor interface even if not implemente
- predictor interface comments expanded
- Differentiate model's required accuracy from predictions quality
- Add missing get_callback_boundary call
- Updated datasets' metadata to allow 3rd parties to code
  regressors themselves
- Add missing option to exception message
- Include target data into the dataset regardless of being a prediction
  dataset or a training dataset
- Explicit in_array and array_search non-strict calls
- Overwrite discrete should_be_displayed implementation with the binary one
- Overwrite no_teacher get_display_value as it would otherwise look
  wrong
- Other minor fixes
parent b8fe16cd
......@@ -469,6 +469,9 @@ abstract class base {
return $result;
}
// Add target metadata.
$this->add_target_metadata($data);
// Write all calculated data to a file.
$file = $dataset->store($data);
......@@ -636,4 +639,28 @@ abstract class base {
$DB->insert_record('analytics_predict_samples', $predictionrange);
}
}
/**
* Adds target metadata to the dataset.
*
* @param array $data
* @return void
*/
protected function add_target_metadata(&$data) {
$data[0][] = 'targetcolumn';
$data[1][] = $this->analysabletarget->get_id();
if ($this->analysabletarget->is_linear()) {
$data[0][] = 'targettype';
$data[1][] = 'linear';
$data[0][] = 'targetmin';
$data[1][] = $this->analysabletarget::get_min_value();
$data[0][] = 'targetmax';
$data[1][] = $this->analysabletarget::get_max_value();
} else {
$data[0][] = 'targettype';
$data[1][] = 'discrete';
$data[0][] = 'targetclasses';
$data[1][] = json_encode($this->analysabletarget::get_classes());
}
}
}
......@@ -46,6 +46,23 @@ abstract class binary extends discrete {
return array(0);
}
/**
* It should always be displayed.
*
* Binary values have no subtypes by default, please overwrite if
* your indicator is adding extra features.
*
* @param float $value
* @param string $subtype
* @return bool
*/
public function should_be_displayed($value, $subtype) {
if ($subtype != false) {
return false;
}
return true;
}
/**
* get_display_value
*
......
......@@ -85,7 +85,7 @@ abstract class discrete extends base {
*/
public function get_display_value($value, $subtype = false) {
$displayvalue = array_search($subtype, static::get_classes());
$displayvalue = array_search($subtype, static::get_classes(), false);
debugging('Please overwrite \core_analytics\local\indicator\discrete::get_display_value to show something ' .
'different than the default "' . $displayvalue . '"', DEBUG_DEVELOPER);
......
......@@ -63,7 +63,7 @@ abstract class linear extends base {
}
/**
* should_be_displayed
* Show only the main feature.
*
* @param float $value
* @param string $subtype
......
......@@ -231,7 +231,7 @@ abstract class base extends \core_analytics\calculable {
*/
protected function min_prediction_score() {
// The default minimum discards predictions with a low score.
return \core_analytics\model::MIN_SCORE;
return \core_analytics\model::PREDICTION_MIN_SCORE;
}
/**
......
......@@ -78,7 +78,7 @@ abstract class binary extends discrete {
throw new \moodle_exception('errorpredictionformat', 'analytics');
}
if (in_array($value, $this->ignored_predicted_classes())) {
if (in_array($value, $this->ignored_predicted_classes(), false)) {
// Just in case, if it is ignored the prediction should not even be recorded but if it would, it is ignored now,
// which should mean that is it nothing serious.
return self::OUTCOME_VERY_POSITIVE;
......
......@@ -42,17 +42,18 @@ abstract class discrete extends base {
*/
public function is_linear() {
// Not supported yet.
throw new \coding_exception('Sorry, this version\'s prediction processors only support targets with binary values.');
throw new \coding_exception('Sorry, this version\'s prediction processors only support targets with binary values.' .
' You can write your own and overwrite this method though.');
}
/**
* Is the provided class one of this target valid classes?
*
* @param string $class
* @param mixed $class
* @return bool
*/
protected static function is_a_class($class) {
return (in_array($class, static::get_classes()));
return (in_array($class, static::get_classes(), false));
}
/**
......@@ -99,7 +100,7 @@ abstract class discrete extends base {
throw new \moodle_exception('errorpredictionformat', 'analytics');
}
if (in_array($value, $this->ignored_predicted_classes())) {
if (in_array($value, $this->ignored_predicted_classes(), false)) {
// Just in case, if it is ignored the prediction should not even be recorded.
return self::OUTCOME_OK;
}
......@@ -138,15 +139,16 @@ abstract class discrete extends base {
* Returns the predicted classes that will be ignored.
*
* Better be keen to add more than less classes here, the callback is always able to discard some classes. As an example
* a target with classes 'grade 0-3', 'grade 3-6', 'grade 6-8' and 'grade 8-10' is interested in flagging both 'grade 0-3'
* and 'grade 3-6'. On the other hand, a target like dropout risk with classes 'yes', 'no' may just be interested in 'yes'.
* a target with classes 'grade 0-3', 'grade 3-6', 'grade 6-8' and 'grade 8-10' is interested in flagging both 'grade 6-8'
* and 'grade 8-10' as ignored. On the other hand, a target like dropout risk with classes 'yes', 'no' may just be
* interested in 'yes'.
*
* @return array List of values that will be ignored (array keys are ignored).
*/
protected function ignored_predicted_classes() {
// Coding exception as this will only be called if this target have non-linear values.
throw new \coding_exception('Overwrite ignored_predicted_classes() and return an array with the classes that triggers ' .
'the callback');
throw new \coding_exception('Overwrite ignored_predicted_classes() and return an array with the classes that should not ' .
'trigger the callback');
}
/**
......@@ -162,10 +164,8 @@ abstract class discrete extends base {
return false;
}
if (!$this->is_linear()) {
if (in_array($predictedvalue, $this->ignored_predicted_classes())) {
return false;
}
if (in_array($predictedvalue, $this->ignored_predicted_classes())) {
return false;
}
return true;
......
......@@ -42,7 +42,8 @@ abstract class linear extends base {
*/
public function is_linear() {
// Not supported yet.
throw new \coding_exception('Sorry, this version\'s prediction processors only support targets with binary values.');
throw new \coding_exception('Sorry, this version\'s prediction processors only support targets with binary values.' .
' You can write your own and overwrite this method though.');
}
/**
......@@ -52,7 +53,7 @@ abstract class linear extends base {
* @param string $ignoredsubtype
* @return int
*/
public function get_calculated_outcome($value, $ignoredsubtype = false) {
public function get_calculation_outcome($value, $ignoredsubtype = false) {
// This is very generic, targets will probably be interested in overwriting this.
$diff = static::get_max_value() - static::get_min_value();
......@@ -67,7 +68,7 @@ abstract class linear extends base {
*
* @return float
*/
protected static function get_max_value() {
public static function get_max_value() {
// Coding exception as this will only be called if this target have linear values.
throw new \coding_exception('Overwrite get_max_value() and return the target max value');
}
......@@ -77,11 +78,33 @@ abstract class linear extends base {
*
* @return float
*/
protected static function get_min_value() {
public static function get_min_value() {
// Coding exception as this will only be called if this target have linear values.
throw new \coding_exception('Overwrite get_min_value() and return the target min value');
}
/**
* Should the model callback be triggered?
*
* @param mixed $predictedvalue
* @param float $predictionscore
* @return bool
*/
public function triggers_callback($predictedvalue, $predictionscore) {
if (!parent::triggers_callback($predictedvalue, $predictionscore)) {
return false;
}
// People may not want to set a boundary.
$boundary = $this->get_callback_boundary();
if (!empty($boundary) && floatval($predictedvalue) < $boundary) {
return false;
}
return true;
}
/**
* Returns the minimum value that triggers the callback.
*
......
......@@ -371,12 +371,9 @@ abstract class base {
$metadata = array(
'timesplitting' => $this->get_id(),
// If no target the first column is the sampleid, if target the last column is the target.
// This will need to be updated when we support unsupervised learning models.
'nfeatures' => count(current($dataset)) - 1
);
if ($target) {
$metadata['targetclasses'] = json_encode($target::get_classes());
$metadata['targettype'] = ($target->is_linear()) ? 'linear' : 'discrete';
}
// The first 2 samples will be used to store metadata about the dataset.
$metadatacolumns = [];
......
......@@ -80,6 +80,11 @@ class model {
*/
const MIN_SCORE = 0.7;
/**
* Minimum prediction confidence (from 0 to 1) to accept a prediction as reliable enough.
*/
const PREDICTION_MIN_SCORE = 0.6;
/**
* Maximum standard deviation between different evaluation repetitions to consider that evaluation results are stable.
*/
......@@ -524,8 +529,13 @@ class model {
$outputdir = $this->get_output_dir(array('evaluation', $dashestimesplittingid));
// Evaluate the dataset, the deviation we accept in the results depends on the amount of iterations.
$predictorresult = $predictor->evaluate($this->model->id, self::ACCEPTED_DEVIATION,
if ($this->get_target()->is_linear()) {
$predictorresult = $predictor->evaluate_regression($this->get_unique_id(), self::ACCEPTED_DEVIATION,
self::EVALUATION_ITERATIONS, $dataset, $outputdir);
} else {
$predictorresult = $predictor->evaluate_classification($this->get_unique_id(), self::ACCEPTED_DEVIATION,
self::EVALUATION_ITERATIONS, $dataset, $outputdir);
}
$result->status = $predictorresult->status;
$result->info = $predictorresult->info;
......@@ -599,7 +609,11 @@ class model {
$samplesfile = $datasets[$this->model->timesplitting];
// Train using the dataset.
$predictorresult = $predictor->train($this->get_unique_id(), $samplesfile, $outputdir);
if ($this->get_target()->is_linear()) {
$predictorresult = $predictor->train_regression($this->get_unique_id(), $samplesfile, $outputdir);
} else {
$predictorresult = $predictor->train_classification($this->get_unique_id(), $samplesfile, $outputdir);
}
$result = new \stdClass();
$result->status = $predictorresult->status;
......@@ -678,8 +692,12 @@ class model {
$result->predictions = $this->get_static_predictions($indicatorcalculations);
} else {
// Prediction process runs on the machine learning backend.
$predictorresult = $predictor->predict($this->get_unique_id(), $samplesfile, $outputdir);
// Estimation and classification processes run on the machine learning backend side.
if ($this->get_target()->is_linear()) {
$predictorresult = $predictor->estimate($this->get_unique_id(), $samplesfile, $outputdir);
} else {
$predictorresult = $predictor->classify($this->get_unique_id(), $samplesfile, $outputdir);
}
$result->status = $predictorresult->status;
$result->info = $predictorresult->info;
$result->predictions = $this->format_predictor_predictions($predictorresult);
......
......@@ -43,34 +43,67 @@ interface predictor {
public function is_ready();
/**
* Train the provided dataset.
* Train this processor classification model using the provided supervised learning dataset.
*
* @param int $modelid
* @param string $uniqueid
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function train($modelid, \stored_file $dataset, $outputdir);
public function train_classification($uniqueid, \stored_file $dataset, $outputdir);
/**
* Predict the provided dataset samples.
* Classifies the provided dataset samples.
*
* @param int $modelid
* @param string $uniqueid
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function predict($modelid, \stored_file $dataset, $outputdir);
public function classify($uniqueid, \stored_file $dataset, $outputdir);
/**
* evaluate
* Evaluates this processor classification model using the provided supervised learning dataset.
*
* @param int $modelid
* @param string $uniqueid
* @param float $maxdeviation
* @param int $niterations
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function evaluate($modelid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir);
public function evaluate_classification($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir);
/**
* Train this processor regression model using the provided supervised learning dataset.
*
* @param string $uniqueid
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function train_regression($uniqueid, \stored_file $dataset, $outputdir);
/**
* Estimates linear values for the provided dataset samples.
*
* @param string $uniqueid
* @param \stored_file $dataset
* @param mixed $outputdir
* @return void
*/
public function estimate($uniqueid, \stored_file $dataset, $outputdir);
/**
* Evaluates this processor regression model using the provided supervised learning dataset.
*
* @param string $uniqueid
* @param float $maxdeviation
* @param int $niterations
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function evaluate_regression($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir);
}
......@@ -64,6 +64,23 @@ class no_teacher extends \core_analytics\local\indicator\binary {
return array('context', 'course');
}
/**
* Reversed because the indicator is in 'negative' and the max returned value means teacher present.
*
* @param float $value
* @param string $subtype
* @return string
*/
public function get_display_value($value, $subtype = false) {
// No subtypes for binary values by default.
if ($value == -1) {
return get_string('yes');
} else if ($value == 1) {
return get_string('no');
}
}
/**
* calculate_sample
*
......
......@@ -73,14 +73,14 @@ class processor implements \core_analytics\predictor {
}
/**
* Trains a machine learning algorithm with the provided training set.
* Train this processor classification model using the provided supervised learning dataset.
*
* @param string $uniqueid
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function train($uniqueid, \stored_file $dataset, $outputdir) {
public function train_classification($uniqueid, \stored_file $dataset, $outputdir) {
// Output directory is already unique to the model.
$modelfilepath = $outputdir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME;
......@@ -134,14 +134,14 @@ class processor implements \core_analytics\predictor {
}
/**
* Predicts the provided samples
* Classifies the provided dataset samples.
*
* @param string $uniqueid
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function predict($uniqueid, \stored_file $dataset, $outputdir) {
public function classify($uniqueid, \stored_file $dataset, $outputdir) {
// Output directory is already unique to the model.
$modelfilepath = $outputdir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME;
......@@ -199,7 +199,7 @@ class processor implements \core_analytics\predictor {
}
/**
* Evaluates the provided dataset.
* Evaluates this processor classification model using the provided supervised learning dataset.
*
* During evaluation we need to shuffle the evaluation dataset samples to detect deviated results,
* if the dataset is massive we can not load everything into memory. We know that 2GB is the
......@@ -216,7 +216,7 @@ class processor implements \core_analytics\predictor {
* @param string $outputdir
* @return \stdClass
*/
public function evaluate($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) {
public function evaluate_classification($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) {
$fh = $dataset->get_content_file_handle();
// The first lines are var names and the second one values.
......@@ -351,6 +351,47 @@ class processor implements \core_analytics\predictor {
return $resultobj;
}
/**
* Train this processor regression model using the provided supervised learning dataset.
*
* @throws new \coding_exception
* @param string $uniqueid
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function train_regression($uniqueid, \stored_file $dataset, $outputdir) {
throw new \coding_exception('This predictor does not support regression yet.');
}
/**
* Estimates linear values for the provided dataset samples.
*
* @throws new \coding_exception
* @param string $uniqueid
* @param \stored_file $dataset
* @param mixed $outputdir
* @return void
*/
public function estimate($uniqueid, \stored_file $dataset, $outputdir) {
throw new \coding_exception('This predictor does not support regression yet.');
}
/**
* Evaluates this processor regression model using the provided supervised learning dataset.
*
* @throws new \coding_exception
* @param string $uniqueid
* @param float $maxdeviation
* @param int $niterations
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function evaluate_regression($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) {
throw new \coding_exception('This predictor does not support regression yet.');
}
/**
* Returns the Phi correlation coefficient.
*
......
......@@ -79,7 +79,7 @@ class processor implements \core_analytics\predictor {
* @param string $outputdir
* @return \stdClass
*/
public function train($uniqueid, \stored_file $dataset, $outputdir) {
public function train_classification($uniqueid, \stored_file $dataset, $outputdir) {
// Obtain the physical route to the file.
$datasetpath = $this->get_file_path($dataset);
......@@ -113,14 +113,14 @@ class processor implements \core_analytics\predictor {
}
/**
* Returns predictions for the provided dataset samples.
* Classifies the provided dataset samples.
*
* @param string $uniqueid
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function predict($uniqueid, \stored_file $dataset, $outputdir) {
public function classify($uniqueid, \stored_file $dataset, $outputdir) {
// Obtain the physical route to the file.
$datasetpath = $this->get_file_path($dataset);
......@@ -154,7 +154,7 @@ class processor implements \core_analytics\predictor {
}
/**
* Evaluates the provided dataset.
* Evaluates this processor classification model using the provided supervised learning dataset.
*
* @param string $uniqueid
* @param float $maxdeviation
......@@ -163,7 +163,7 @@ class processor implements \core_analytics\predictor {
* @param string $outputdir
* @return \stdClass
*/
public function evaluate($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) {
public function evaluate_classification($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) {
// Obtain the physical route to the file.
$datasetpath = $this->get_file_path($dataset);
......@@ -195,6 +195,47 @@ class processor implements \core_analytics\predictor {
return $resultobj;
}
/**
* Train this processor regression model using the provided supervised learning dataset.
*
* @throws new \coding_exception
* @param string $uniqueid
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function train_regression($uniqueid, \stored_file $dataset, $outputdir) {
throw new \coding_exception('This predictor does not support regression yet.');
}
/**
* Estimates linear values for the provided dataset samples.
*
* @throws new \coding_exception
* @param string $uniqueid
* @param \stored_file $dataset
* @param mixed $outputdir
* @return void
*/
public function estimate($uniqueid, \stored_file $dataset, $outputdir) {
throw new \coding_exception('This predictor does not support regression yet.');
}
/**
* Evaluates this processor regression model using the provided supervised learning dataset.
*
* @throws new \coding_exception
* @param string $uniqueid
* @param float $maxdeviation
* @param int $niterations
* @param \stored_file $dataset
* @param string $outputdir
* @return \stdClass
*/
public function evaluate_regression($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) {
throw new \coding_exception('This predictor does not support regression yet.');
}
/**
* Returns the path to the dataset file.
*
......
......@@ -134,7 +134,7 @@ class insight implements \renderable, \templatable {
* Returns a CSS class from the calculated value outcome.
*
* @param \core_analytics\calculable $calculable
* @param mixed $value
* @param float $value
* @param string|false $subtype
* @return string
*/
......@@ -159,8 +159,8 @@ class insight implements \renderable, \templatable {
default:
throw new \coding_exception('The outcome returned by ' . get_class($calculable) . '::get_calculation_outcome is ' .
'not one of the accepted values. Please use \core_analytics\calculable::OUTCOME_VERY_POSITIVE, ' .
'\core_analytics\calculable::OUTCOME_OK, \core_analytics\calculable::OUTCOME_NEGATIVE or ' .
'\core_analytics\calculable::OUTCOME_VERY_NEGATIVE');
'\core_analytics\calculable::OUTCOME_OK, \core_analytics\calculable::OUTCOME_NEGATIVE, ' .
'\core_analytics\calculable::OUTCOME_VERY_NEGATIVE or \core_analytics\calculable::OUTCOME_NEUTRAL');
}
return $style;
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment